feat(alerting): Implement alert-level provider overrides (#929)
* feat(alerting): Implement alert-level provider overrides Fixes #96 * Fix tests * Add missing test cases for alerting providers * feat(alerting): Implement alert-level overrides on all providers * chore: Add config.yaml to .gitignore * fix typo in discord provider * test: Start fixing tests for alerting providers * test: Fix GitLab tests * Fix all tests * test: Improve coverage * test: Improve coverage * Rename override to provider-override * docs: Mention new provider-override config * test: Improve coverage * test: Improve coverage * chore: Rename Alert.OverrideAsBytes to Alert.ProviderOverrideAsBytes
This commit is contained in:
TwiN
GitHub
@ -13,55 +13,97 @@ import (
|
||||
"github.com/TwiN/gatus/v5/client"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint"
|
||||
"github.com/google/uuid"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultSeverity = "critical"
|
||||
DefaultMonitoringTool = "gatus"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidWebhookURL = fmt.Errorf("invalid webhook-url")
|
||||
ErrAuthorizationKeyNotSet = fmt.Errorf("authorization-key not set")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
WebhookURL string `yaml:"webhook-url"` // The webhook url provided by GitLab
|
||||
AuthorizationKey string `yaml:"authorization-key"` // The authorization key provided by GitLab
|
||||
Severity string `yaml:"severity,omitempty"` // Severity can be one of: critical, high, medium, low, info, unknown. Defaults to critical
|
||||
MonitoringTool string `yaml:"monitoring-tool,omitempty"` // MonitoringTool overrides the name sent to gitlab. Defaults to gatus
|
||||
EnvironmentName string `yaml:"environment-name,omitempty"` // EnvironmentName is the name of the associated GitLab environment. Required to display alerts on a dashboard.
|
||||
Service string `yaml:"service,omitempty"` // Service affected. Defaults to the endpoint's display name
|
||||
}
|
||||
|
||||
func (cfg *Config) Validate() error {
|
||||
if len(cfg.WebhookURL) == 0 {
|
||||
return ErrInvalidWebhookURL
|
||||
} else if _, err := url.Parse(cfg.WebhookURL); err != nil {
|
||||
return ErrInvalidWebhookURL
|
||||
}
|
||||
if len(cfg.AuthorizationKey) == 0 {
|
||||
return ErrAuthorizationKeyNotSet
|
||||
}
|
||||
if len(cfg.Severity) == 0 {
|
||||
cfg.Severity = DefaultSeverity
|
||||
}
|
||||
if len(cfg.MonitoringTool) == 0 {
|
||||
cfg.MonitoringTool = DefaultMonitoringTool
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cfg *Config) Merge(override *Config) {
|
||||
if len(override.WebhookURL) > 0 {
|
||||
cfg.WebhookURL = override.WebhookURL
|
||||
}
|
||||
if len(override.AuthorizationKey) > 0 {
|
||||
cfg.AuthorizationKey = override.AuthorizationKey
|
||||
}
|
||||
if len(override.Severity) > 0 {
|
||||
cfg.Severity = override.Severity
|
||||
}
|
||||
if len(override.MonitoringTool) > 0 {
|
||||
cfg.MonitoringTool = override.MonitoringTool
|
||||
}
|
||||
if len(override.EnvironmentName) > 0 {
|
||||
cfg.EnvironmentName = override.EnvironmentName
|
||||
}
|
||||
if len(override.Service) > 0 {
|
||||
cfg.Service = override.Service
|
||||
}
|
||||
}
|
||||
|
||||
// AlertProvider is the configuration necessary for sending an alert using GitLab
|
||||
type AlertProvider struct {
|
||||
WebhookURL string `yaml:"webhook-url"` // The webhook url provided by GitLab
|
||||
AuthorizationKey string `yaml:"authorization-key"` // The authorization key provided by GitLab
|
||||
DefaultConfig Config `yaml:",inline"`
|
||||
|
||||
// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
|
||||
DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
|
||||
|
||||
// Severity can be one of: critical, high, medium, low, info, unknown. Defaults to critical
|
||||
Severity string `yaml:"severity,omitempty"`
|
||||
|
||||
// MonitoringTool overrides the name sent to gitlab. Defaults to gatus
|
||||
MonitoringTool string `yaml:"monitoring-tool,omitempty"`
|
||||
|
||||
// EnvironmentName is the name of the associated GitLab environment. Required to display alerts on a dashboard.
|
||||
EnvironmentName string `yaml:"environment-name,omitempty"`
|
||||
|
||||
// Service affected. Defaults to endpoint display name
|
||||
Service string `yaml:"service,omitempty"`
|
||||
}
|
||||
|
||||
// IsValid returns whether the provider's configuration is valid
|
||||
func (provider *AlertProvider) IsValid() bool {
|
||||
if len(provider.AuthorizationKey) == 0 || len(provider.WebhookURL) == 0 {
|
||||
return false
|
||||
}
|
||||
// Validate format of the repository URL
|
||||
_, err := url.Parse(provider.WebhookURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
// Validate the provider's configuration
|
||||
func (provider *AlertProvider) Validate() error {
|
||||
return provider.DefaultConfig.Validate()
|
||||
}
|
||||
|
||||
// Send creates an issue in the designed RepositoryURL if the resolved parameter passed is false,
|
||||
// or closes the relevant issue(s) if the resolved parameter passed is true.
|
||||
func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
|
||||
cfg, err := provider.GetConfig(ep.Group, alert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(alert.ResolveKey) == 0 {
|
||||
alert.ResolveKey = uuid.NewString()
|
||||
}
|
||||
buffer := bytes.NewBuffer(provider.buildAlertBody(ep, alert, result, resolved))
|
||||
request, err := http.NewRequest(http.MethodPost, provider.WebhookURL, buffer)
|
||||
buffer := bytes.NewBuffer(provider.buildAlertBody(cfg, ep, alert, result, resolved))
|
||||
request, err := http.NewRequest(http.MethodPost, cfg.WebhookURL, buffer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", provider.AuthorizationKey))
|
||||
request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.AuthorizationKey))
|
||||
response, err := client.GetHTTPClient(nil).Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -87,30 +129,20 @@ type AlertBody struct {
|
||||
GitlabEnvironmentName string `json:"gitlab_environment_name,omitempty"` // The name of the associated GitLab environment. Required to display alerts on a dashboard.
|
||||
}
|
||||
|
||||
func (provider *AlertProvider) monitoringTool() string {
|
||||
if len(provider.MonitoringTool) > 0 {
|
||||
return provider.MonitoringTool
|
||||
}
|
||||
return "gatus"
|
||||
}
|
||||
|
||||
func (provider *AlertProvider) service(ep *endpoint.Endpoint) string {
|
||||
if len(provider.Service) > 0 {
|
||||
return provider.Service
|
||||
}
|
||||
return ep.DisplayName()
|
||||
}
|
||||
|
||||
// buildAlertBody builds the body of the alert
|
||||
func (provider *AlertProvider) buildAlertBody(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
|
||||
func (provider *AlertProvider) buildAlertBody(cfg *Config, ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
|
||||
service := cfg.Service
|
||||
if len(service) == 0 {
|
||||
service = ep.DisplayName()
|
||||
}
|
||||
body := AlertBody{
|
||||
Title: fmt.Sprintf("alert(%s): %s", provider.monitoringTool(), provider.service(ep)),
|
||||
Title: fmt.Sprintf("alert(%s): %s", cfg.MonitoringTool, service),
|
||||
StartTime: result.Timestamp.Format(time.RFC3339),
|
||||
Service: provider.service(ep),
|
||||
MonitoringTool: provider.monitoringTool(),
|
||||
Service: service,
|
||||
MonitoringTool: cfg.MonitoringTool,
|
||||
Hosts: ep.URL,
|
||||
GitlabEnvironmentName: provider.EnvironmentName,
|
||||
Severity: provider.Severity,
|
||||
GitlabEnvironmentName: cfg.EnvironmentName,
|
||||
Severity: cfg.Severity,
|
||||
Fingerprint: alert.ResolveKey,
|
||||
}
|
||||
if resolved {
|
||||
@ -148,3 +180,25 @@ func (provider *AlertProvider) buildAlertBody(ep *endpoint.Endpoint, alert *aler
|
||||
func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
|
||||
return provider.DefaultAlert
|
||||
}
|
||||
|
||||
// GetConfig returns the configuration for the provider with the overrides applied
|
||||
func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
|
||||
cfg := provider.DefaultConfig
|
||||
// Handle alert overrides
|
||||
if len(alert.ProviderOverride) != 0 {
|
||||
overrideConfig := Config{}
|
||||
if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.Merge(&overrideConfig)
|
||||
}
|
||||
// Validate the configuration (we're returning the cfg here even if there's an error mostly for testing purposes)
|
||||
err := cfg.Validate()
|
||||
return &cfg, err
|
||||
}
|
||||
|
||||
// ValidateOverrides validates the alert's provider override and, if present, the group override
|
||||
func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
|
||||
_, err := provider.GetConfig(group, alert)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -11,37 +11,41 @@ import (
|
||||
"github.com/TwiN/gatus/v5/test"
|
||||
)
|
||||
|
||||
func TestAlertDefaultProvider_IsValid(t *testing.T) {
|
||||
func TestAlertProvider_Validate(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
Name string
|
||||
Provider AlertProvider
|
||||
Expected bool
|
||||
Name string
|
||||
Provider AlertProvider
|
||||
ExpectedError bool
|
||||
}{
|
||||
{
|
||||
Name: "invalid",
|
||||
Provider: AlertProvider{WebhookURL: "", AuthorizationKey: ""},
|
||||
Expected: false,
|
||||
Name: "invalid",
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "", AuthorizationKey: ""}},
|
||||
ExpectedError: true,
|
||||
},
|
||||
{
|
||||
Name: "missing-webhook-url",
|
||||
Provider: AlertProvider{WebhookURL: "", AuthorizationKey: "12345"},
|
||||
Expected: false,
|
||||
Name: "missing-webhook-url",
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "", AuthorizationKey: "12345"}},
|
||||
ExpectedError: true,
|
||||
},
|
||||
{
|
||||
Name: "missing-authorization-key",
|
||||
Provider: AlertProvider{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: ""},
|
||||
Expected: false,
|
||||
Name: "missing-authorization-key",
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "https://gitlab.com/whatever/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: ""}},
|
||||
ExpectedError: true,
|
||||
},
|
||||
{
|
||||
Name: "invalid-url",
|
||||
Provider: AlertProvider{WebhookURL: " http://foo.com", AuthorizationKey: "12345"},
|
||||
Expected: false,
|
||||
Name: "invalid-url",
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: " http://foo.com", AuthorizationKey: "12345"}},
|
||||
ExpectedError: true,
|
||||
},
|
||||
}
|
||||
for _, scenario := range scenarios {
|
||||
t.Run(scenario.Name, func(t *testing.T) {
|
||||
if scenario.Provider.IsValid() != scenario.Expected {
|
||||
t.Errorf("expected %t, got %t", scenario.Expected, scenario.Provider.IsValid())
|
||||
err := scenario.Provider.Validate()
|
||||
if scenario.ExpectedError && err == nil {
|
||||
t.Error("expected error, got none")
|
||||
}
|
||||
if !scenario.ExpectedError && err != nil && !strings.Contains(err.Error(), "user does not exist") && !strings.Contains(err.Error(), "no such host") {
|
||||
t.Error("expected no error, got", err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -61,7 +65,7 @@ func TestAlertProvider_Send(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
Name: "triggered-error",
|
||||
Provider: AlertProvider{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"},
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"}},
|
||||
Alert: alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
|
||||
Resolved: false,
|
||||
ExpectedError: false,
|
||||
@ -71,7 +75,7 @@ func TestAlertProvider_Send(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "resolved-error",
|
||||
Provider: AlertProvider{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"},
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"}},
|
||||
Alert: alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
|
||||
Resolved: true,
|
||||
ExpectedError: false,
|
||||
@ -116,21 +120,26 @@ func TestAlertProvider_buildAlertBody(t *testing.T) {
|
||||
{
|
||||
Name: "triggered",
|
||||
Endpoint: endpoint.Endpoint{Name: "endpoint-name", URL: "https://example.org"},
|
||||
Provider: AlertProvider{},
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"}},
|
||||
Alert: alert.Alert{Description: &firstDescription, FailureThreshold: 3},
|
||||
ExpectedBody: "{\"title\":\"alert(gatus): endpoint-name\",\"description\":\"An alert for *endpoint-name* has been triggered due to having failed 3 time(s) in a row:\\n\\u003e description-1\\n\\n## Condition results\\n- :white_check_mark: - `[CONNECTED] == true`\\n- :x: - `[STATUS] == 200`\\n\",\"start_time\":\"0001-01-01T00:00:00Z\",\"service\":\"endpoint-name\",\"monitoring_tool\":\"gatus\",\"hosts\":\"https://example.org\"}",
|
||||
ExpectedBody: "{\"title\":\"alert(gatus): endpoint-name\",\"description\":\"An alert for *endpoint-name* has been triggered due to having failed 3 time(s) in a row:\\n\\u003e description-1\\n\\n## Condition results\\n- :white_check_mark: - `[CONNECTED] == true`\\n- :x: - `[STATUS] == 200`\\n\",\"start_time\":\"0001-01-01T00:00:00Z\",\"service\":\"endpoint-name\",\"monitoring_tool\":\"gatus\",\"hosts\":\"https://example.org\",\"severity\":\"critical\"}",
|
||||
},
|
||||
{
|
||||
Name: "no-description",
|
||||
Endpoint: endpoint.Endpoint{Name: "endpoint-name", URL: "https://example.org"},
|
||||
Provider: AlertProvider{},
|
||||
Provider: AlertProvider{DefaultConfig: Config{WebhookURL: "https://gitlab.com/hlidotbe/text/alerts/notify/gatus/xxxxxxxxxxxxxxxx.json", AuthorizationKey: "12345"}},
|
||||
Alert: alert.Alert{FailureThreshold: 10},
|
||||
ExpectedBody: "{\"title\":\"alert(gatus): endpoint-name\",\"description\":\"An alert for *endpoint-name* has been triggered due to having failed 10 time(s) in a row\\n\\n## Condition results\\n- :white_check_mark: - `[CONNECTED] == true`\\n- :x: - `[STATUS] == 200`\\n\",\"start_time\":\"0001-01-01T00:00:00Z\",\"service\":\"endpoint-name\",\"monitoring_tool\":\"gatus\",\"hosts\":\"https://example.org\"}",
|
||||
ExpectedBody: "{\"title\":\"alert(gatus): endpoint-name\",\"description\":\"An alert for *endpoint-name* has been triggered due to having failed 10 time(s) in a row\\n\\n## Condition results\\n- :white_check_mark: - `[CONNECTED] == true`\\n- :x: - `[STATUS] == 200`\\n\",\"start_time\":\"0001-01-01T00:00:00Z\",\"service\":\"endpoint-name\",\"monitoring_tool\":\"gatus\",\"hosts\":\"https://example.org\",\"severity\":\"critical\"}",
|
||||
},
|
||||
}
|
||||
for _, scenario := range scenarios {
|
||||
t.Run(scenario.Name, func(t *testing.T) {
|
||||
cfg, err := scenario.Provider.GetConfig("", &scenario.Alert)
|
||||
if err != nil {
|
||||
t.Error("expected no error, got", err.Error())
|
||||
}
|
||||
body := scenario.Provider.buildAlertBody(
|
||||
cfg,
|
||||
&scenario.Endpoint,
|
||||
&scenario.Alert,
|
||||
&endpoint.Result{
|
||||
@ -156,3 +165,59 @@ func TestAlertProvider_GetDefaultAlert(t *testing.T) {
|
||||
t.Error("expected default alert to be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertProvider_GetConfig(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
Name string
|
||||
Provider AlertProvider
|
||||
InputAlert alert.Alert
|
||||
ExpectedOutput Config
|
||||
}{
|
||||
{
|
||||
Name: "provider-no-override-should-default",
|
||||
Provider: AlertProvider{
|
||||
DefaultConfig: Config{WebhookURL: "https://github.com/TwiN/test", AuthorizationKey: "12345"},
|
||||
},
|
||||
InputAlert: alert.Alert{},
|
||||
ExpectedOutput: Config{WebhookURL: "https://github.com/TwiN/test", AuthorizationKey: "12345", Severity: DefaultSeverity, MonitoringTool: DefaultMonitoringTool},
|
||||
},
|
||||
{
|
||||
Name: "provider-with-alert-override",
|
||||
Provider: AlertProvider{
|
||||
DefaultConfig: Config{WebhookURL: "https://github.com/TwiN/test", AuthorizationKey: "12345"},
|
||||
},
|
||||
InputAlert: alert.Alert{ProviderOverride: map[string]any{"repository-url": "https://github.com/TwiN/alert-test", "authorization-key": "54321", "severity": "info", "monitoring-tool": "not-gatus", "environment-name": "prod", "service": "example"}},
|
||||
ExpectedOutput: Config{WebhookURL: "https://github.com/TwiN/test", AuthorizationKey: "54321", Severity: "info", MonitoringTool: "not-gatus", EnvironmentName: "prod", Service: "example"},
|
||||
},
|
||||
}
|
||||
for _, scenario := range scenarios {
|
||||
t.Run(scenario.Name, func(t *testing.T) {
|
||||
got, err := scenario.Provider.GetConfig("", &scenario.InputAlert)
|
||||
if err != nil && !strings.Contains(err.Error(), "user does not exist") && !strings.Contains(err.Error(), "no such host") {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if got.WebhookURL != scenario.ExpectedOutput.WebhookURL {
|
||||
t.Errorf("expected repository URL %s, got %s", scenario.ExpectedOutput.WebhookURL, got.WebhookURL)
|
||||
}
|
||||
if got.AuthorizationKey != scenario.ExpectedOutput.AuthorizationKey {
|
||||
t.Errorf("expected AuthorizationKey %s, got %s", scenario.ExpectedOutput.AuthorizationKey, got.AuthorizationKey)
|
||||
}
|
||||
if got.Severity != scenario.ExpectedOutput.Severity {
|
||||
t.Errorf("expected Severity %s, got %s", scenario.ExpectedOutput.Severity, got.Severity)
|
||||
}
|
||||
if got.MonitoringTool != scenario.ExpectedOutput.MonitoringTool {
|
||||
t.Errorf("expected MonitoringTool %s, got %s", scenario.ExpectedOutput.MonitoringTool, got.MonitoringTool)
|
||||
}
|
||||
if got.EnvironmentName != scenario.ExpectedOutput.EnvironmentName {
|
||||
t.Errorf("expected EnvironmentName %s, got %s", scenario.ExpectedOutput.EnvironmentName, got.EnvironmentName)
|
||||
}
|
||||
if got.Service != scenario.ExpectedOutput.Service {
|
||||
t.Errorf("expected Service %s, got %s", scenario.ExpectedOutput.Service, got.Service)
|
||||
}
|
||||
// Test ValidateOverrides as well, since it really just calls GetConfig
|
||||
if err = scenario.Provider.ValidateOverrides("", &scenario.InputAlert); err != nil {
|
||||
t.Errorf("unexpected error: %s", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user