feat(alerting): Implement alert-level provider overrides (#929)

* feat(alerting): Implement alert-level provider overrides Fixes #96 * Fix tests * Add missing test cases for alerting providers * feat(alerting): Implement alert-level overrides on all providers * chore: Add config.yaml to .gitignore * fix typo in discord provider * test: Start fixing tests for alerting providers * test: Fix GitLab tests * Fix all tests * test: Improve coverage * test: Improve coverage * Rename override to provider-override * docs: Mention new provider-override config * test: Improve coverage * test: Improve coverage * chore: Rename Alert.OverrideAsBytes to Alert.ProviderOverrideAsBytes
2024-12-16 20:32:13 -05:00
parent be9ae6f55d
commit 79c9f24c15
54 changed files with 4623 additions and 2109 deletions
--- a/alerting/provider/pagerduty/pagerduty.go
+++ b/alerting/provider/pagerduty/pagerduty.go
@ -3,6 +3,7 @@ package pagerduty
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@ -11,15 +12,38 @@ import (
 	"github.com/TwiN/gatus/v5/client"
 	"github.com/TwiN/gatus/v5/config/endpoint"
 	"github.com/TwiN/logr"
+	"gopkg.in/yaml.v3"
 )

 const (
 	restAPIURL = "https://events.pagerduty.com/v2/enqueue"
 )

+var (
+	ErrIntegrationKeyNotSet   = errors.New("integration-key must have exactly 32 characters")
+	ErrDuplicateGroupOverride = errors.New("duplicate group override")
+)
+
+type Config struct {
+	IntegrationKey string `yaml:"integration-key"`
+}
+
+func (cfg *Config) Validate() error {
+	if len(cfg.IntegrationKey) != 32 {
+		return ErrIntegrationKeyNotSet
+	}
+	return nil
+}
+
+func (cfg *Config) Merge(override *Config) {
+	if len(override.IntegrationKey) > 0 {
+		cfg.IntegrationKey = override.IntegrationKey
+	}
+}
+
 // AlertProvider is the configuration necessary for sending an alert using PagerDuty
 type AlertProvider struct {
-	IntegrationKey string `yaml:"integration-key"`
+	DefaultConfig Config `yaml:",inline"`

 	// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
 	DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
@ -30,30 +54,34 @@ type AlertProvider struct {

 // Override is a case under which the default integration is overridden
 type Override struct {
-	Group          string `yaml:"group"`
-	IntegrationKey string `yaml:"integration-key"`
+	Group  string `yaml:"group"`
+	Config `yaml:",inline"`
 }

-// IsValid returns whether the provider's configuration is valid
-func (provider *AlertProvider) IsValid() bool {
+// Validate the provider's configuration
+func (provider *AlertProvider) Validate() error {
 	registeredGroups := make(map[string]bool)
 	if provider.Overrides != nil {
 		for _, override := range provider.Overrides {
-			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" || len(override.IntegrationKey) != 32 {
-				return false
+			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" {
+				return ErrDuplicateGroupOverride
 			}
 			registeredGroups[override.Group] = true
 		}
 	}
 	// Either the default integration key has the right length, or there are overrides who are properly configured.
-	return len(provider.IntegrationKey) == 32 || len(provider.Overrides) != 0
+	return provider.DefaultConfig.Validate()
 }

 // Send an alert using the provider
 //
 // Relevant: https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
 func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
-	buffer := bytes.NewBuffer(provider.buildRequestBody(ep, alert, result, resolved))
+	cfg, err := provider.GetConfig(ep.Group, alert)
+	if err != nil {
+		return err
+	}
+	buffer := bytes.NewBuffer(provider.buildRequestBody(cfg, ep, alert, result, resolved))
 	request, err := http.NewRequest(http.MethodPost, restAPIURL, buffer)
 	if err != nil {
 		return err
@ -100,7 +128,7 @@ type Payload struct {
 }

 // buildRequestBody builds the request body for the provider
-func (provider *AlertProvider) buildRequestBody(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
+func (provider *AlertProvider) buildRequestBody(cfg *Config, ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
 	var message, eventAction, resolveKey string
 	if resolved {
 		message = fmt.Sprintf("RESOLVED: %s - %s", ep.DisplayName(), alert.GetDescription())
@ -112,7 +140,7 @@ func (provider *AlertProvider) buildRequestBody(ep *endpoint.Endpoint, alert *al
 		resolveKey = ""
 	}
 	body, _ := json.Marshal(Body{
-		RoutingKey:  provider.getIntegrationKeyForGroup(ep.Group),
+		RoutingKey:  cfg.IntegrationKey,
 		DedupKey:    resolveKey,
 		EventAction: eventAction,
 		Payload: Payload{
@ -124,23 +152,42 @@ func (provider *AlertProvider) buildRequestBody(ep *endpoint.Endpoint, alert *al
 	return body
 }

-// getIntegrationKeyForGroup returns the appropriate pagerduty integration key for a given group
-func (provider *AlertProvider) getIntegrationKeyForGroup(group string) string {
-	if provider.Overrides != nil {
-		for _, override := range provider.Overrides {
-			if group == override.Group {
-				return override.IntegrationKey
-			}
-		}
-	}
-	return provider.IntegrationKey
-}
-
 // GetDefaultAlert returns the provider's default alert configuration
 func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
 	return provider.DefaultAlert
 }

+// GetConfig returns the configuration for the provider with the overrides applied
+func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
+	cfg := provider.DefaultConfig
+	// Handle group overrides
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if group == override.Group {
+				cfg.Merge(&override.Config)
+				break
+			}
+		}
+	}
+	// Handle alert overrides
+	if len(alert.ProviderOverride) != 0 {
+		overrideConfig := Config{}
+		if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
+			return nil, err
+		}
+		cfg.Merge(&overrideConfig)
+	}
+	// Validate the configuration
+	err := cfg.Validate()
+	return &cfg, err
+}
+
+// ValidateOverrides validates the alert's provider override and, if present, the group override
+func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
+	_, err := provider.GetConfig(group, alert)
+	return err
+}
+
 type pagerDutyResponsePayload struct {
 	Status   string `json:"status"`
 	Message  string `json:"message"`
--- a/alerting/provider/pagerduty/pagerduty_test.go
+++ b/alerting/provider/pagerduty/pagerduty_test.go
@ -11,50 +11,41 @@ import (
 	"github.com/TwiN/gatus/v5/test"
 )

-func TestAlertProvider_IsValid(t *testing.T) {
-	invalidProvider := AlertProvider{IntegrationKey: ""}
-	if invalidProvider.IsValid() {
+func TestAlertProvider_Validate(t *testing.T) {
+	invalidProvider := AlertProvider{DefaultConfig: Config{IntegrationKey: ""}}
+	if err := invalidProvider.Validate(); err == nil {
 		t.Error("provider shouldn't have been valid")
 	}
-	validProvider := AlertProvider{IntegrationKey: "00000000000000000000000000000000"}
-	if !validProvider.IsValid() {
+	validProvider := AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}}
+	if err := validProvider.Validate(); err != nil {
 		t.Error("provider should've been valid")
 	}
 }

-func TestAlertProvider_IsValidWithOverride(t *testing.T) {
+func TestAlertProvider_ValidateWithOverride(t *testing.T) {
 	providerWithInvalidOverrideGroup := AlertProvider{
+		DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
 		Overrides: []Override{
 			{
-				IntegrationKey: "00000000000000000000000000000000",
-				Group:          "",
+				Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+				Group:  "",
 			},
 		},
 	}
-	if providerWithInvalidOverrideGroup.IsValid() {
+	if err := providerWithInvalidOverrideGroup.Validate(); err == nil {
 		t.Error("provider Group shouldn't have been valid")
 	}
-	providerWithInvalidOverrideIntegrationKey := AlertProvider{
-		Overrides: []Override{
-			{
-				IntegrationKey: "",
-				Group:          "group",
-			},
-		},
-	}
-	if providerWithInvalidOverrideIntegrationKey.IsValid() {
-		t.Error("provider integration key shouldn't have been valid")
-	}
 	providerWithValidOverride := AlertProvider{
+		DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
 		Overrides: []Override{
 			{
-				IntegrationKey: "00000000000000000000000000000000",
-				Group:          "group",
+				Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+				Group:  "group",
 			},
 		},
 	}
-	if !providerWithValidOverride.IsValid() {
-		t.Error("provider should've been valid")
+	if err := providerWithValidOverride.Validate(); err != nil {
+		t.Error("provider should've been valid, got error:", err.Error())
 	}
 }

@ -72,7 +63,7 @@ func TestAlertProvider_Send(t *testing.T) {
 	}{
 		{
 			Name:     "triggered",
-			Provider: AlertProvider{},
+			Provider: AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
 			Resolved: false,
 			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
@ -82,7 +73,7 @@ func TestAlertProvider_Send(t *testing.T) {
 		},
 		{
 			Name:     "triggered-error",
-			Provider: AlertProvider{},
+			Provider: AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
 			Resolved: false,
 			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
@ -92,7 +83,7 @@ func TestAlertProvider_Send(t *testing.T) {
 		},
 		{
 			Name:     "resolved",
-			Provider: AlertProvider{},
+			Provider: AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
 			Resolved: true,
 			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
@ -102,7 +93,7 @@ func TestAlertProvider_Send(t *testing.T) {
 		},
 		{
 			Name:     "resolved-error",
-			Provider: AlertProvider{},
+			Provider: AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
 			Resolved: true,
 			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
@ -146,14 +137,14 @@ func TestAlertProvider_buildRequestBody(t *testing.T) {
 	}{
 		{
 			Name:         "triggered",
-			Provider:     AlertProvider{IntegrationKey: "00000000000000000000000000000000"},
+			Provider:     AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:        alert.Alert{Description: &description},
 			Resolved:     false,
 			ExpectedBody: "{\"routing_key\":\"00000000000000000000000000000000\",\"dedup_key\":\"\",\"event_action\":\"trigger\",\"payload\":{\"summary\":\"TRIGGERED: endpoint-name - test\",\"source\":\"Gatus\",\"severity\":\"critical\"}}",
 		},
 		{
 			Name:         "resolved",
-			Provider:     AlertProvider{IntegrationKey: "00000000000000000000000000000000"},
+			Provider:     AlertProvider{DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000000"}},
 			Alert:        alert.Alert{Description: &description, ResolveKey: "key"},
 			Resolved:     true,
 			ExpectedBody: "{\"routing_key\":\"00000000000000000000000000000000\",\"dedup_key\":\"key\",\"event_action\":\"resolve\",\"payload\":{\"summary\":\"RESOLVED: endpoint-name - test\",\"source\":\"Gatus\",\"severity\":\"critical\"}}",
@ -161,7 +152,7 @@ func TestAlertProvider_buildRequestBody(t *testing.T) {
 	}
 	for _, scenario := range scenarios {
 		t.Run(scenario.Name, func(t *testing.T) {
-			body := scenario.Provider.buildRequestBody(&endpoint.Endpoint{Name: "endpoint-name"}, &scenario.Alert, &endpoint.Result{}, scenario.Resolved)
+			body := scenario.Provider.buildRequestBody(&scenario.Provider.DefaultConfig, &endpoint.Endpoint{Name: "endpoint-name"}, &scenario.Alert, &endpoint.Result{}, scenario.Resolved)
 			if string(body) != scenario.ExpectedBody {
 				t.Errorf("expected:\n%s\ngot:\n%s", scenario.ExpectedBody, body)
 			}
@ -173,69 +164,6 @@ func TestAlertProvider_buildRequestBody(t *testing.T) {
 	}
 }

-func TestAlertProvider_getIntegrationKeyForGroup(t *testing.T) {
-	scenarios := []struct {
-		Name           string
-		Provider       AlertProvider
-		InputGroup     string
-		ExpectedOutput string
-	}{
-		{
-			Name: "provider-no-override-specify-no-group-should-default",
-			Provider: AlertProvider{
-				IntegrationKey: "00000000000000000000000000000001",
-				Overrides:      nil,
-			},
-			InputGroup:     "",
-			ExpectedOutput: "00000000000000000000000000000001",
-		},
-		{
-			Name: "provider-no-override-specify-group-should-default",
-			Provider: AlertProvider{
-				IntegrationKey: "00000000000000000000000000000001",
-				Overrides:      nil,
-			},
-			InputGroup:     "group",
-			ExpectedOutput: "00000000000000000000000000000001",
-		},
-		{
-			Name: "provider-with-override-specify-no-group-should-default",
-			Provider: AlertProvider{
-				IntegrationKey: "00000000000000000000000000000001",
-				Overrides: []Override{
-					{
-						Group:          "group",
-						IntegrationKey: "00000000000000000000000000000002",
-					},
-				},
-			},
-			InputGroup:     "",
-			ExpectedOutput: "00000000000000000000000000000001",
-		},
-		{
-			Name: "provider-with-override-specify-group-should-override",
-			Provider: AlertProvider{
-				IntegrationKey: "00000000000000000000000000000001",
-				Overrides: []Override{
-					{
-						Group:          "group",
-						IntegrationKey: "00000000000000000000000000000002",
-					},
-				},
-			},
-			InputGroup:     "group",
-			ExpectedOutput: "00000000000000000000000000000002",
-		},
-	}
-	for _, scenario := range scenarios {
-		t.Run(scenario.Name, func(t *testing.T) {
-			if output := scenario.Provider.getIntegrationKeyForGroup(scenario.InputGroup); output != scenario.ExpectedOutput {
-				t.Errorf("expected %s, got %s", scenario.ExpectedOutput, output)
-			}
-		})
-	}
-}
-
 func TestAlertProvider_GetDefaultAlert(t *testing.T) {
 	if (&AlertProvider{DefaultAlert: &alert.Alert{}}).GetDefaultAlert() == nil {
 		t.Error("expected default alert to be not nil")
@ -244,3 +172,94 @@ func TestAlertProvider_GetDefaultAlert(t *testing.T) {
 		t.Error("expected default alert to be nil")
 	}
 }
+
+func TestAlertProvider_GetConfig(t *testing.T) {
+	scenarios := []struct {
+		Name           string
+		Provider       AlertProvider
+		InputGroup     string
+		InputAlert     alert.Alert
+		ExpectedOutput Config
+	}{
+		{
+			Name: "provider-no-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides:     nil,
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-no-override-specify-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides:     nil,
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-with-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-with-override-specify-group-should-override",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000002"},
+		},
+		{
+			Name: "provider-with-group-override-and-alert-override--alert-override-should-take-precedence",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{ProviderOverride: map[string]any{"integration-key": "00000000000000000000000000000003"}},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000003"},
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			got, err := scenario.Provider.GetConfig(scenario.InputGroup, &scenario.InputAlert)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if got.IntegrationKey != scenario.ExpectedOutput.IntegrationKey {
+				t.Errorf("expected %s, got %s", scenario.ExpectedOutput.IntegrationKey, got.IntegrationKey)
+			}
+			// Test ValidateOverrides as well, since it really just calls GetConfig
+			if err = scenario.Provider.ValidateOverrides(scenario.InputGroup, &scenario.InputAlert); err != nil {
+				t.Errorf("unexpected error: %s", err)
+			}
+		})
+	}
+}