phares3757/gatus
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Work on common provider interface to make adding new providers easier

Browse Source
This commit is contained in:
TwinProduction
2020-09-26 14:23:43 -04:00
parent 7dcd462883
commit ac5ad9d173
7 changed files with 63 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
alerting/provider/custom/custom.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"github.com/TwinProduction/gatus/client"
"github.com/TwinProduction/gatus/core"
"io/ioutil"
"net/http"
"strings"
@ -20,6 +21,10 @@ func (provider *AlertProvider) IsValid() bool {
return len(provider.Url) > 0
}
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *AlertProvider {
return provider
}
func (provider *AlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
body := provider.Body
providerUrl := provider.Url

12
alerting/provider/pagerduty/pagerduty.go
View File

@ -15,7 +15,17 @@ func (provider *AlertProvider) IsValid() bool {
}
// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
func (provider *AlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *custom.AlertProvider {
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider {
var message, eventAction, resolveKey string
if resolved {
message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)
eventAction = "resolve"
resolveKey = alert.ResolveKey
} else {
message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)
eventAction = "trigger"
resolveKey = ""
}
return &custom.AlertProvider{
Url: "https://events.pagerduty.com/v2/enqueue",
Method: "POST",

6
alerting/provider/provider.go
View File

@ -1,5 +1,11 @@
package provider
import (
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/core"
)
type AlertProvider interface {
IsValid() bool
ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider
}

9
alerting/provider/twilio/twilio.go
View File

@ -4,6 +4,7 @@ import (
"encoding/base64"
"fmt"
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/core"
"net/url"
)
@ -18,7 +19,13 @@ func (provider *AlertProvider) IsValid() bool {
return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
}
func (provider *AlertProvider) ToCustomAlertProvider(message string) *custom.AlertProvider {
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider {
var message string
if resolved {
message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)
} else {
message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)
}
return &custom.AlertProvider{
Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
Method: "POST",

13
client/client_test.go Normal file
View File

@ -0,0 +1,13 @@
package client
import "testing"
func TestGetHttpClient(t *testing.T) {
if client != nil {
t.Error("client should've been nil since it hasn't been called a single time yet")
}
_ = GetHttpClient()
if client == nil {
t.Error("client shouldn't have been nil, since it has been called once")
}
}

4
config.yaml
View File

@ -1,15 +1,15 @@
metrics: true
services:
- name: twinnation
url: "https://twinnation.org/health"
interval: 30s
url: https://twinnation.org/health
conditions:
- "[STATUS] == 200"
- "[BODY].status == UP"
- "[RESPONSE_TIME] < 1000"
- name: cat-fact
interval: 1m
url: "https://cat-fact.herokuapp.com/facts/random"
interval: 1m
conditions:
- "[STATUS] == 200"
- "[BODY].deleted == false"

87
watchdog/alerting.go
View File

@ -2,8 +2,6 @@ package watchdog
import (
"encoding/json"
"fmt"
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/core"
"log"
@ -36,44 +34,16 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
}
continue
}
var alertProvider *custom.AlertProvider
// TODO: leverage provider.AlertingProvider interface and config.GetAlertingProviderByAlertType(cfg, alert.Type)
// TODO: to support all types of alerts without having to explicitly define them here and in handleAlertsToResolve()
if alert.Type == core.SlackAlert {
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false)
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Custom
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
}
}
if alertProvider != nil {
alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type)
if alertProvider != nil && alertProvider.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending %s alert because alert with description='%s' has been triggered", alert.Type, alert.Description)
customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, false)
// TODO: retry on error
var err error
// We need to extract the DedupKey from PagerDuty's response
if alert.Type == core.PagerDutyAlert {
var body []byte
body, err = alertProvider.Send(service.Name, alert.Description, true)
body, err = customAlertProvider.Send(service.Name, alert.Description, false)
if err == nil {
var response pagerDutyResponse
err = json.Unmarshal(body, &response)
@ -84,13 +54,17 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
}
}
} else {
_, err = alertProvider.Send(service.Name, alert.Description, false)
// All other alert types don't need to extract anything from the body, so we can just send the request right away
_, err = customAlertProvider.Send(service.Name, alert.Description, false)
}
if err != nil {
log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
} else {
alert.Triggered = true
}
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being triggered, because the provider wasn't configured properly", alert.Type)
}
}
}
@ -105,39 +79,12 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
if !alert.SendOnResolved {
continue
}
var alertProvider *custom.AlertProvider
if alert.Type == core.SlackAlert {
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true)
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Custom
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
}
}
if alertProvider != nil {
alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type)
if alertProvider != nil && alertProvider.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending %s alert because alert with description='%s' has been resolved", alert.Type, alert.Description)
customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, true)
// TODO: retry on error
_, err := alertProvider.Send(service.Name, alert.Description, true)
_, err := customAlertProvider.Send(service.Name, alert.Description, true)
if err != nil {
log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
} else {
@ -145,6 +92,8 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
alert.ResolveKey = ""
}
}
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being resolved, because the provider wasn't configured properly", alert.Type)
}
}
service.NumberOfFailuresInARow = 0