From d4623f5c610cb1df53f55cdb3066e794e9f95db3 Mon Sep 17 00:00:00 2001 From: TwinProduction Date: Fri, 4 Sep 2020 21:57:31 -0400 Subject: [PATCH] Add [ALERT_TRIGGERED_OR_RESOLVED] placeholder for custom alert provider Fix placeholder bug in CustomAlertProvider --- README.md | 8 ++++++-- core/alerting.go | 40 +++++++++++++++++++++++++++------------- watchdog/watchdog.go | 16 ++++++++-------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 41344680..d99f0d02 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,10 @@ would then check if the service that started failing was recently deployed, and roll it back. The values `[ALERT_DESCRIPTION]` and `[SERVICE_NAME]` are automatically substituted for the alert description and the -service name respectively in the body (`alerting.custom.body`) and the url (`alerting.custom.url`). +service name respectively in the body (`alerting.custom.body`) as well as the url (`alerting.custom.url`). + +If you have `send-on-resolved` set to `true`, you may want to use `[ALERT_TRIGGERED_OR_RESOLVED]` to differentiate +the notifications. It will be replaced for either `TRIGGERED` or `RESOLVED`, based on the situation. For all intents and purpose, we'll configure the custom alert with a Slack webhook, but you can call anything you want. @@ -259,7 +262,7 @@ alerting: method: "POST" body: | { - "text": "[SERVICE_NAME] - [ALERT_DESCRIPTION]" + "text": "[ALERT_TRIGGERED_OR_RESOLVED]: [SERVICE_NAME] - [ALERT_DESCRIPTION]" } services: - name: twinnation @@ -269,6 +272,7 @@ services: - type: custom enabled: true threshold: 10 + send-on-resolved: true description: "healthcheck failed 10 times in a row" conditions: - "[STATUS] == 200" diff --git a/core/alerting.go b/core/alerting.go index 5f595f13..06daa331 100644 --- a/core/alerting.go +++ b/core/alerting.go @@ -38,31 +38,45 @@ func (provider *CustomAlertProvider) IsValid() bool { return len(provider.Url) > 0 } -func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string) *http.Request { +func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request { body := provider.Body - url := provider.Url - if strings.Contains(provider.Body, "[ALERT_DESCRIPTION]") { - body = strings.ReplaceAll(provider.Body, "[ALERT_DESCRIPTION]", alertDescription) + providerUrl := provider.Url + if strings.Contains(body, "[ALERT_DESCRIPTION]") { + body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription) } - if strings.Contains(provider.Body, "[SERVICE_NAME]") { - body = strings.ReplaceAll(provider.Body, "[SERVICE_NAME]", serviceName) + if strings.Contains(body, "[SERVICE_NAME]") { + body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName) } - if strings.Contains(provider.Url, "[ALERT_DESCRIPTION]") { - url = strings.ReplaceAll(provider.Url, "[ALERT_DESCRIPTION]", alertDescription) + if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") { + if resolved { + body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") + } else { + body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") + } } - if strings.Contains(provider.Url, "[SERVICE_NAME]") { - url = strings.ReplaceAll(provider.Url, "[SERVICE_NAME]", serviceName) + if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription) + } + if strings.Contains(providerUrl, "[SERVICE_NAME]") { + providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName) + } + if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") { + if resolved { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") + } else { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") + } } bodyBuffer := bytes.NewBuffer([]byte(body)) - request, _ := http.NewRequest(provider.Method, url, bodyBuffer) + request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer) for k, v := range provider.Headers { request.Header.Set(k, v) } return request } -func (provider *CustomAlertProvider) Send(serviceName, alertDescription string) error { - request := provider.buildRequest(serviceName, alertDescription) +func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) error { + request := provider.buildRequest(serviceName, alertDescription, resolved) response, err := client.GetHttpClient().Do(request) if err != nil { return err diff --git a/watchdog/watchdog.go b/watchdog/watchdog.go index c237dcfa..952e44b7 100644 --- a/watchdog/watchdog.go +++ b/watchdog/watchdog.go @@ -100,14 +100,14 @@ func handleAlerting(service *core.Service, result *core.Result) { } } else if alert.Type == core.TwilioAlert { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { - log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) - alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) + log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been resolved", alert.Description) + alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) } else { - log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio isn't configured properly'") + log.Printf("[watchdog][monitor] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly") } } else if alert.Type == core.CustomAlert { if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { - log.Printf("[watchdog][monitor] Sending custom alert because alert with description=%s has been triggered", alert.Description) + log.Printf("[watchdog][monitor] Sending custom alert because alert with description=%s has been resolved", alert.Description) alertProvider = &core.CustomAlertProvider{ Url: cfg.Alerting.Custom.Url, Method: cfg.Alerting.Custom.Method, @@ -115,11 +115,11 @@ func handleAlerting(service *core.Service, result *core.Result) { Headers: cfg.Alerting.Custom.Headers, } } else { - log.Printf("[watchdog][monitor] Not sending custom alert despite being triggered, because there is no custom url configured") + log.Printf("[watchdog][monitor] Not sending custom alert despite being resolved, because the custom provider isn't configured properly") } } if alertProvider != nil { - err := alertProvider.Send(service.Name, alert.Description) + err := alertProvider.Send(service.Name, alert.Description, true) if err != nil { log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error()) } @@ -145,7 +145,7 @@ func handleAlerting(service *core.Service, result *core.Result) { } else if alert.Type == core.TwilioAlert { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) - alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) + alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) } else { log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio config settings missing") } @@ -163,7 +163,7 @@ func handleAlerting(service *core.Service, result *core.Result) { } } if alertProvider != nil { - err := alertProvider.Send(service.Name, alert.Description) + err := alertProvider.Send(service.Name, alert.Description, false) if err != nil { log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error()) }