phares3757/gatus
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add [ALERT_TRIGGERED_OR_RESOLVED] placeholder for custom alert provider

Browse Source 
Fix placeholder bug in CustomAlertProvider
This commit is contained in:
TwinProduction 2020-09-04 21:57:31 -04:00
parent 139e186ac2
commit d4623f5c61
3 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -248,7 +248,10 @@ would then check if the service that started failing was recently deployed, and
roll it back. roll it back.
The values `[ALERT_DESCRIPTION]` and `[SERVICE_NAME]` are automatically substituted for the alert description and the The values `[ALERT_DESCRIPTION]` and `[SERVICE_NAME]` are automatically substituted for the alert description and the
service name respectively in the body (`alerting.custom.body`) and the url (`alerting.custom.url`). service name respectively in the body (`alerting.custom.body`) as well as the url (`alerting.custom.url`).
If you have `send-on-resolved` set to `true`, you may want to use `[ALERT_TRIGGERED_OR_RESOLVED]` to differentiate
the notifications. It will be replaced for either `TRIGGERED` or `RESOLVED`, based on the situation.
For all intents and purpose, we'll configure the custom alert with a Slack webhook, but you can call anything you want. For all intents and purpose, we'll configure the custom alert with a Slack webhook, but you can call anything you want.
@ -259,7 +262,7 @@ alerting:
method: "POST" method: "POST"
body: | body: |
{ {
"text": "[SERVICE_NAME] - [ALERT_DESCRIPTION]" "text": "[ALERT_TRIGGERED_OR_RESOLVED]: [SERVICE_NAME] - [ALERT_DESCRIPTION]"
} }
services: services:
- name: twinnation - name: twinnation
@ -269,6 +272,7 @@ services:
- type: custom - type: custom
enabled: true enabled: true
threshold: 10 threshold: 10
send-on-resolved: true
description: "healthcheck failed 10 times in a row" description: "healthcheck failed 10 times in a row"
conditions: conditions:
- "[STATUS] == 200" - "[STATUS] == 200"

40
core/alerting.go
View File

@ -38,31 +38,45 @@ func (provider *CustomAlertProvider) IsValid() bool {
return len(provider.Url) > 0 return len(provider.Url) > 0
} }
func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string) *http.Request { func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
body := provider.Body body := provider.Body
url := provider.Url providerUrl := provider.Url
if strings.Contains(provider.Body, "[ALERT_DESCRIPTION]") { if strings.Contains(body, "[ALERT_DESCRIPTION]") {
body = strings.ReplaceAll(provider.Body, "[ALERT_DESCRIPTION]", alertDescription) body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
} }
if strings.Contains(provider.Body, "[SERVICE_NAME]") { if strings.Contains(body, "[SERVICE_NAME]") {
body = strings.ReplaceAll(provider.Body, "[SERVICE_NAME]", serviceName) body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
} }
if strings.Contains(provider.Url, "[ALERT_DESCRIPTION]") { if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
url = strings.ReplaceAll(provider.Url, "[ALERT_DESCRIPTION]", alertDescription) if resolved {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
}
}
if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
}
if strings.Contains(providerUrl, "[SERVICE_NAME]") {
providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
}
if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
if resolved {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
} }
if strings.Contains(provider.Url, "[SERVICE_NAME]") {
url = strings.ReplaceAll(provider.Url, "[SERVICE_NAME]", serviceName)
} }
bodyBuffer := bytes.NewBuffer([]byte(body)) bodyBuffer := bytes.NewBuffer([]byte(body))
request, _ := http.NewRequest(provider.Method, url, bodyBuffer) request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer)
for k, v := range provider.Headers { for k, v := range provider.Headers {
request.Header.Set(k, v) request.Header.Set(k, v)
} }
return request return request
} }
func (provider *CustomAlertProvider) Send(serviceName, alertDescription string) error { func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) error {
request := provider.buildRequest(serviceName, alertDescription) request := provider.buildRequest(serviceName, alertDescription, resolved)
response, err := client.GetHttpClient().Do(request) response, err := client.GetHttpClient().Do(request)
if err != nil { if err != nil {
return err return err

16
watchdog/watchdog.go
View File

@ -100,14 +100,14 @@ func handleAlerting(service *core.Service, result *core.Result) {
} }
} else if alert.Type == core.TwilioAlert { } else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been resolved", alert.Description)
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else { } else {
log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio isn't configured properly'") log.Printf("[watchdog][monitor] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
} }
} else if alert.Type == core.CustomAlert { } else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[watchdog][monitor] Sending custom alert because alert with description=%s has been triggered", alert.Description) log.Printf("[watchdog][monitor] Sending custom alert because alert with description=%s has been resolved", alert.Description)
alertProvider = &core.CustomAlertProvider{ alertProvider = &core.CustomAlertProvider{
Url: cfg.Alerting.Custom.Url, Url: cfg.Alerting.Custom.Url,
Method: cfg.Alerting.Custom.Method, Method: cfg.Alerting.Custom.Method,
@ -115,11 +115,11 @@ func handleAlerting(service *core.Service, result *core.Result) {
Headers: cfg.Alerting.Custom.Headers, Headers: cfg.Alerting.Custom.Headers,
} }
} else { } else {
log.Printf("[watchdog][monitor] Not sending custom alert despite being triggered, because there is no custom url configured") log.Printf("[watchdog][monitor] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
} }
} }
if alertProvider != nil { if alertProvider != nil {
err := alertProvider.Send(service.Name, alert.Description) err := alertProvider.Send(service.Name, alert.Description, true)
if err != nil { if err != nil {
log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error()) log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error())
} }
@ -145,7 +145,7 @@ func handleAlerting(service *core.Service, result *core.Result) {
} else if alert.Type == core.TwilioAlert { } else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description)
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else { } else {
log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio config settings missing") log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
} }
@ -163,7 +163,7 @@ func handleAlerting(service *core.Service, result *core.Result) {
} }
} }
if alertProvider != nil { if alertProvider != nil {
err := alertProvider.Send(service.Name, alert.Description) err := alertProvider.Send(service.Name, alert.Description, false)
if err != nil { if err != nil {
log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error()) log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error())
} }