diff --git a/README.md b/README.md index 5a84d81a..34bc07ec 100644 --- a/README.md +++ b/README.md @@ -415,18 +415,30 @@ services: | Parameter | Description | Default | |:---------------------------------------- |:----------------------------------------------------------------------------- |:-------------- | | `alerting.pagerduty` | Configuration for alerts of type `pagerduty` | `{}` | -| `alerting.pagerduty.integration-key` | PagerDuty Events API v2 integration key. | Required `""` | +| `alerting.pagerduty.integration-key` | PagerDuty Events API v2 integration key. | `""` | | `alerting.pagerduty.default-alert` | Default alert configuration.
See [Setting a default alert](#setting-a-default-alert) | N/A | +| `alerting.pagerduty.integrations` | Pagerduty integrations per team configurations | `[]` | +| `alerting.pagerduty.integrations[].integration-key` | Pagerduty integrationkey for a perticular team | `""` | +| `alerting.pagerduty.integrations[].group` | the group that the integration key belongs to | `""` | It is highly recommended to set `services[].alerts[].send-on-resolved` to `true` for alerts of type `pagerduty`, because unlike other alerts, the operation resulting from setting said parameter to `true` will not create another incident, but mark the incident as resolved on PagerDuty instead. +Behavior: +- Team integration have priority over the general integration +- If no team integration is provided it will defaults to the general pagerduty integration +- If no team integration and no general integration were provided it defaults to the first team integration provided + + ```yaml alerting: pagerduty: integration-key: "********************************" + intergrations: + - integration-key: "********************************" + group: "core" services: - name: website @@ -443,6 +455,20 @@ services: success-threshold: 5 send-on-resolved: true description: "healthcheck failed" + - name: back-end + group: core + url: "https://example.org/" + interval: 5m + conditions: + - "[STATUS] == 200" + - "[CERTIFICATE_EXPIRATION] > 48h" + alerts: + - type: pagerduty + enabled: true + failure-threshold: 3 + success-threshold: 5 + send-on-resolved: true + description: "healthcheck failed" ``` diff --git a/alerting/provider/pagerduty/pagerduty.go b/alerting/provider/pagerduty/pagerduty.go index 2f209d5f..5318a879 100644 --- a/alerting/provider/pagerduty/pagerduty.go +++ b/alerting/provider/pagerduty/pagerduty.go @@ -13,22 +13,54 @@ const ( restAPIURL = "https://events.pagerduty.com/v2/enqueue" ) +type Integrations struct { + IntegrationKey string `yaml:"integration-key"` + Group string `yaml:"group"` +} + // AlertProvider is the configuration necessary for sending an alert using PagerDuty type AlertProvider struct { IntegrationKey string `yaml:"integration-key"` // DefaultAlert is the default alert configuration to use for services with an alert of the appropriate type DefaultAlert *alert.Alert `yaml:"default-alert"` + + Integrations []Integrations `yaml:"integrations"` } // IsValid returns whether the provider's configuration is valid func (provider *AlertProvider) IsValid() bool { - return len(provider.IntegrationKey) == 32 + registeredGroups := make(map[string]bool) + if provider.Integrations != nil { + for _, integration := range provider.Integrations { + if isAlreadyRegistered := registeredGroups[integration.Group]; isAlreadyRegistered || integration.Group == "" || len(integration.IntegrationKey) != 32 { + return false + } + registeredGroups[integration.Group] = true + } + } + return len(provider.IntegrationKey) == 32 || provider.Integrations != nil +} + +// GetPagerDutyIntegrationKey returns the appropriate pagerduty integration key +func (provider *AlertProvider) GetPagerDutyIntegrationKey(group string) string { + if provider.Integrations != nil { + for _, integration := range provider.Integrations { + if group == integration.Group { + return integration.IntegrationKey + } + } + } + if provider.IntegrationKey != "" { + return provider.IntegrationKey + } + return "" } // ToCustomAlertProvider converts the provider into a custom.AlertProvider // // relevant: https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ + func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *alert.Alert, _ *core.Result, resolved bool) *custom.AlertProvider { var message, eventAction, resolveKey string if resolved { @@ -52,7 +84,7 @@ func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, aler "source": "%s", "severity": "critical" } -}`, provider.IntegrationKey, resolveKey, eventAction, message, service.Name), +}`, provider.GetPagerDutyIntegrationKey(service.Group), resolveKey, eventAction, message, service.Name), Headers: map[string]string{ "Content-Type": "application/json", }, diff --git a/alerting/provider/pagerduty/pagerduty_test.go b/alerting/provider/pagerduty/pagerduty_test.go index 31f046a6..c4b09c00 100644 --- a/alerting/provider/pagerduty/pagerduty_test.go +++ b/alerting/provider/pagerduty/pagerduty_test.go @@ -10,7 +10,7 @@ import ( "github.com/TwinProduction/gatus/v3/core" ) -func TestAlertProvider_IsValid(t *testing.T) { +func TestAlertDefaultProvider_IsValid(t *testing.T) { invalidProvider := AlertProvider{IntegrationKey: ""} if invalidProvider.IsValid() { t.Error("provider shouldn't have been valid") @@ -20,6 +20,44 @@ func TestAlertProvider_IsValid(t *testing.T) { t.Error("provider should've been valid") } } +func TestAlertPerGroupProvider_IsValid(t *testing.T) { + invalidGroup := Integrations{ + IntegrationKey: "00000000000000000000000000000000", + Group: "", + } + integrations := []Integrations{} + integrations = append(integrations, invalidGroup) + invalidProviderGroupNameError := AlertProvider{ + Integrations: integrations, + } + if invalidProviderGroupNameError.IsValid() { + t.Error("provider Group shouldn't have been valid") + } + invalidIntegrationKey := Integrations{ + IntegrationKey: "", + Group: "group", + } + integrations = []Integrations{} + integrations = append(integrations, invalidIntegrationKey) + invalidProviderIntegrationKey := AlertProvider{ + Integrations: integrations, + } + if invalidProviderIntegrationKey.IsValid() { + t.Error("provider integration key shouldn't have been valid") + } + validIntegration := Integrations{ + IntegrationKey: "00000000000000000000000000000000", + Group: "group", + } + integrations = []Integrations{} + integrations = append(integrations, validIntegration) + validProvider := AlertProvider{ + Integrations: integrations, + } + if !validProvider.IsValid() { + t.Error("provider should've been valid") + } +} func TestAlertProvider_ToCustomAlertProviderWithResolvedAlert(t *testing.T) { provider := AlertProvider{IntegrationKey: "00000000000000000000000000000000"} @@ -43,6 +81,37 @@ func TestAlertProvider_ToCustomAlertProviderWithResolvedAlert(t *testing.T) { } } +func TestAlertPerGroupProvider_ToCustomAlertProviderWithResolvedAlert(t *testing.T) { + validIntegration := Integrations{ + IntegrationKey: "00000000000000000000000000000000", + Group: "group", + } + integrations := []Integrations{} + integrations = append(integrations, validIntegration) + provider := AlertProvider{ + IntegrationKey: "", + Integrations: integrations, + } + customAlertProvider := provider.ToCustomAlertProvider(&core.Service{}, &alert.Alert{}, &core.Result{}, true) + if customAlertProvider == nil { + t.Fatal("customAlertProvider shouldn't have been nil") + } + if !strings.Contains(customAlertProvider.Body, "RESOLVED") { + t.Error("customAlertProvider.Body should've contained the substring RESOLVED") + } + if customAlertProvider.URL != "https://events.pagerduty.com/v2/enqueue" { + t.Errorf("expected URL to be %s, got %s", "https://events.pagerduty.com/v2/enqueue", customAlertProvider.URL) + } + if customAlertProvider.Method != http.MethodPost { + t.Errorf("expected method to be %s, got %s", http.MethodPost, customAlertProvider.Method) + } + body := make(map[string]interface{}) + err := json.Unmarshal([]byte(customAlertProvider.Body), &body) + if err != nil { + t.Error("expected body to be valid JSON, got error:", err.Error()) + } +} + func TestAlertProvider_ToCustomAlertProviderWithTriggeredAlert(t *testing.T) { provider := AlertProvider{IntegrationKey: "00000000000000000000000000000000"} customAlertProvider := provider.ToCustomAlertProvider(&core.Service{}, &alert.Alert{}, &core.Result{}, false) @@ -64,3 +133,34 @@ func TestAlertProvider_ToCustomAlertProviderWithTriggeredAlert(t *testing.T) { t.Error("expected body to be valid JSON, got error:", err.Error()) } } + +func TestAlertPerGroupProvider_ToCustomAlertProviderWithTriggeredAlert(t *testing.T) { + validIntegration := Integrations{ + IntegrationKey: "00000000000000000000000000000000", + Group: "group", + } + integrations := []Integrations{} + integrations = append(integrations, validIntegration) + provider := AlertProvider{ + IntegrationKey: "", + Integrations: integrations, + } + customAlertProvider := provider.ToCustomAlertProvider(&core.Service{}, &alert.Alert{}, &core.Result{}, false) + if customAlertProvider == nil { + t.Fatal("customAlertProvider shouldn't have been nil") + } + if !strings.Contains(customAlertProvider.Body, "TRIGGERED") { + t.Error("customAlertProvider.Body should've contained the substring TRIGGERED") + } + if customAlertProvider.URL != "https://events.pagerduty.com/v2/enqueue" { + t.Errorf("expected URL to be %s, got %s", "https://events.pagerduty.com/v2/enqueue", customAlertProvider.URL) + } + if customAlertProvider.Method != http.MethodPost { + t.Errorf("expected method to be %s, got %s", http.MethodPost, customAlertProvider.Method) + } + body := make(map[string]interface{}) + err := json.Unmarshal([]byte(customAlertProvider.Body), &body) + if err != nil { + t.Error("expected body to be valid JSON, got error:", err.Error()) + } +}