Rename Service to Endpoint (#192)
* Add clarifications in comments * #191: Rename Service to Endpoint
This commit is contained in:
TwiN
GitHub
@ -10,93 +10,93 @@ import (
|
||||
)
|
||||
|
||||
// HandleAlerting takes care of alerts to resolve and alerts to trigger based on result success or failure
|
||||
func HandleAlerting(service *core.Service, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
func HandleAlerting(endpoint *core.Endpoint, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
if alertingConfig == nil {
|
||||
return
|
||||
}
|
||||
if result.Success {
|
||||
handleAlertsToResolve(service, result, alertingConfig, debug)
|
||||
handleAlertsToResolve(endpoint, result, alertingConfig, debug)
|
||||
} else {
|
||||
handleAlertsToTrigger(service, result, alertingConfig, debug)
|
||||
handleAlertsToTrigger(endpoint, result, alertingConfig, debug)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToTrigger(service *core.Service, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
service.NumberOfSuccessesInARow = 0
|
||||
service.NumberOfFailuresInARow++
|
||||
for _, serviceAlert := range service.Alerts {
|
||||
// If the serviceAlert hasn't been triggered, move to the next one
|
||||
if !serviceAlert.IsEnabled() || serviceAlert.FailureThreshold > service.NumberOfFailuresInARow {
|
||||
func handleAlertsToTrigger(endpoint *core.Endpoint, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
endpoint.NumberOfSuccessesInARow = 0
|
||||
endpoint.NumberOfFailuresInARow++
|
||||
for _, endpointAlert := range endpoint.Alerts {
|
||||
// If the alert hasn't been triggered, move to the next one
|
||||
if !endpointAlert.IsEnabled() || endpointAlert.FailureThreshold > endpoint.NumberOfFailuresInARow {
|
||||
continue
|
||||
}
|
||||
if serviceAlert.Triggered {
|
||||
if endpointAlert.Triggered {
|
||||
if debug {
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Alert for service=%s with description='%s' has already been TRIGGERED, skipping", service.Name, serviceAlert.GetDescription())
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Alert for endpoint=%s with description='%s' has already been TRIGGERED, skipping", endpoint.Name, endpointAlert.GetDescription())
|
||||
}
|
||||
continue
|
||||
}
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(serviceAlert.Type)
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type)
|
||||
if alertProvider != nil && alertProvider.IsValid() {
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Sending %s serviceAlert because serviceAlert for service=%s with description='%s' has been TRIGGERED", serviceAlert.Type, service.Name, serviceAlert.GetDescription())
|
||||
customAlertProvider := alertProvider.ToCustomAlertProvider(service, serviceAlert, result, false)
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Sending %s alert because alert for endpoint=%s with description='%s' has been TRIGGERED", endpointAlert.Type, endpoint.Name, endpointAlert.GetDescription())
|
||||
customAlertProvider := alertProvider.ToCustomAlertProvider(endpoint, endpointAlert, result, false)
|
||||
// TODO: retry on error
|
||||
var err error
|
||||
// We need to extract the DedupKey from PagerDuty's response
|
||||
if serviceAlert.Type == alert.TypePagerDuty {
|
||||
if endpointAlert.Type == alert.TypePagerDuty {
|
||||
var body []byte
|
||||
if body, err = customAlertProvider.Send(service.Name, serviceAlert.GetDescription(), false); err == nil {
|
||||
if body, err = customAlertProvider.Send(endpoint.Name, endpointAlert.GetDescription(), false); err == nil {
|
||||
var response pagerDutyResponse
|
||||
if err = json.Unmarshal(body, &response); err != nil {
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Ran into error unmarshaling pagerduty response: %s", err.Error())
|
||||
} else {
|
||||
serviceAlert.ResolveKey = response.DedupKey
|
||||
endpointAlert.ResolveKey = response.DedupKey
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// All other serviceAlert types don't need to extract anything from the body, so we can just send the request right away
|
||||
_, err = customAlertProvider.Send(service.Name, serviceAlert.GetDescription(), false)
|
||||
// All other alert types don't need to extract anything from the body, so we can just send the request right away
|
||||
_, err = customAlertProvider.Send(endpoint.Name, endpointAlert.GetDescription(), false)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Failed to send an serviceAlert for service=%s: %s", service.Name, err.Error())
|
||||
log.Printf("[watchdog][handleAlertsToTrigger] Failed to send an alert for endpoint=%s: %s", endpoint.Name, err.Error())
|
||||
} else {
|
||||
serviceAlert.Triggered = true
|
||||
endpointAlert.Triggered = true
|
||||
}
|
||||
} else {
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Not sending serviceAlert of type=%s despite being TRIGGERED, because the provider wasn't configured properly", serviceAlert.Type)
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being TRIGGERED, because the provider wasn't configured properly", endpointAlert.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToResolve(service *core.Service, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
service.NumberOfSuccessesInARow++
|
||||
for _, serviceAlert := range service.Alerts {
|
||||
if !serviceAlert.IsEnabled() || !serviceAlert.Triggered || serviceAlert.SuccessThreshold > service.NumberOfSuccessesInARow {
|
||||
func handleAlertsToResolve(endpoint *core.Endpoint, result *core.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
endpoint.NumberOfSuccessesInARow++
|
||||
for _, endpointAlert := range endpoint.Alerts {
|
||||
if !endpointAlert.IsEnabled() || !endpointAlert.Triggered || endpointAlert.SuccessThreshold > endpoint.NumberOfSuccessesInARow {
|
||||
continue
|
||||
}
|
||||
// Even if the serviceAlert provider returns an error, we still set the serviceAlert's Triggered variable to false.
|
||||
// Even if the alert provider returns an error, we still set the alert's Triggered variable to false.
|
||||
// Further explanation can be found on Alert's Triggered field.
|
||||
serviceAlert.Triggered = false
|
||||
if !serviceAlert.IsSendingOnResolved() {
|
||||
endpointAlert.Triggered = false
|
||||
if !endpointAlert.IsSendingOnResolved() {
|
||||
continue
|
||||
}
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(serviceAlert.Type)
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type)
|
||||
if alertProvider != nil && alertProvider.IsValid() {
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Sending %s serviceAlert because serviceAlert for service=%s with description='%s' has been RESOLVED", serviceAlert.Type, service.Name, serviceAlert.GetDescription())
|
||||
customAlertProvider := alertProvider.ToCustomAlertProvider(service, serviceAlert, result, true)
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Sending %s alert because alert for endpoint=%s with description='%s' has been RESOLVED", endpointAlert.Type, endpoint.Name, endpointAlert.GetDescription())
|
||||
customAlertProvider := alertProvider.ToCustomAlertProvider(endpoint, endpointAlert, result, true)
|
||||
// TODO: retry on error
|
||||
_, err := customAlertProvider.Send(service.Name, serviceAlert.GetDescription(), true)
|
||||
_, err := customAlertProvider.Send(endpoint.Name, endpointAlert.GetDescription(), true)
|
||||
if err != nil {
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Failed to send an serviceAlert for service=%s: %s", service.Name, err.Error())
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Failed to send an alert for endpoint=%s: %s", endpoint.Name, err.Error())
|
||||
} else {
|
||||
if serviceAlert.Type == alert.TypePagerDuty {
|
||||
serviceAlert.ResolveKey = ""
|
||||
if endpointAlert.Type == alert.TypePagerDuty {
|
||||
endpointAlert.ResolveKey = ""
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Not sending serviceAlert of type=%s despite being RESOLVED, because the provider wasn't configured properly", serviceAlert.Type)
|
||||
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being RESOLVED, because the provider wasn't configured properly", endpointAlert.Type)
|
||||
}
|
||||
}
|
||||
service.NumberOfFailuresInARow = 0
|
||||
endpoint.NumberOfFailuresInARow = 0
|
||||
}
|
||||
|
||||
type pagerDutyResponse struct {
|
||||
|
||||
@ -26,8 +26,8 @@ func TestHandleAlerting(t *testing.T) {
|
||||
},
|
||||
}
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
URL: "http://example.com",
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "https://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
Type: alert.TypeCustom,
|
||||
@ -40,23 +40,23 @@ func TestHandleAlerting(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
verify(t, service, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 2, 0, true, "The alert should've triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 3, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 4, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, true, "The alert should still be triggered (because service.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 2, true, "The alert should still be triggered (because service.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 3, false, "The alert should've been resolved")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 4, false, "The alert should no longer be triggered")
|
||||
verify(t, endpoint, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 2, 0, true, "The alert should've triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 3, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 4, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, true, "The alert should still be triggered (because endpoint.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 2, true, "The alert should still be triggered (because endpoint.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 3, false, "The alert should've been resolved")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 4, false, "The alert should no longer be triggered")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWhenAlertingConfigIsNil(t *testing.T) {
|
||||
@ -70,7 +70,7 @@ func TestHandleAlertingWithBadAlertProvider(t *testing.T) {
|
||||
defer os.Clearenv()
|
||||
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -84,14 +84,14 @@ func TestHandleAlertingWithBadAlertProvider(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
verify(t, service, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, &alerting.Config{}, false)
|
||||
verify(t, service, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(service, &core.Result{Success: false}, &alerting.Config{}, false)
|
||||
verify(t, service, 2, 0, false, "The alert shouldn't have triggered, because the provider wasn't configured properly")
|
||||
verify(t, endpoint, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, &alerting.Config{}, false)
|
||||
verify(t, endpoint, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, &alerting.Config{}, false)
|
||||
verify(t, endpoint, 2, 0, false, "The alert shouldn't have triggered, because the provider wasn't configured properly")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButServiceStartFailingAgain(t *testing.T) {
|
||||
func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButendpointStartFailingAgain(t *testing.T) {
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER", "true")
|
||||
defer os.Clearenv()
|
||||
|
||||
@ -105,7 +105,7 @@ func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButServiceStartFailingA
|
||||
},
|
||||
}
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -121,8 +121,8 @@ func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButServiceStartFailingA
|
||||
}
|
||||
|
||||
// This test simulate an alert that was already triggered
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 2, 0, true, "The alert was already triggered at the beginning of this test")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 2, 0, true, "The alert was already triggered at the beginning of this test")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *testing.T) {
|
||||
@ -140,7 +140,7 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *t
|
||||
}
|
||||
enabled := true
|
||||
disabled := false
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -155,8 +155,8 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *t
|
||||
NumberOfFailuresInARow: 1,
|
||||
}
|
||||
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, false, "The alert should've been resolved")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, false, "The alert should've been resolved")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) {
|
||||
@ -172,7 +172,7 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) {
|
||||
},
|
||||
}
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -187,11 +187,11 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) {
|
||||
NumberOfFailuresInARow: 0,
|
||||
}
|
||||
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, true, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, true, "")
|
||||
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, false, "The alert should've been resolved")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, false, "The alert should've been resolved")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWithProviderThatReturnsAnError(t *testing.T) {
|
||||
@ -208,7 +208,7 @@ func TestHandleAlertingWithProviderThatReturnsAnError(t *testing.T) {
|
||||
},
|
||||
}
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -223,33 +223,33 @@ func TestHandleAlertingWithProviderThatReturnsAnError(t *testing.T) {
|
||||
}
|
||||
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, false, "")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 2, 0, false, "The alert should have failed to trigger, because the alert provider is returning an error")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 3, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 4, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 2, 0, false, "The alert should have failed to trigger, because the alert provider is returning an error")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 3, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 4, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 5, 0, true, "The alert should've been triggered because the alert provider is no longer returning an error")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, true, "The alert should've still been triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 5, 0, true, "The alert should've been triggered because the alert provider is no longer returning an error")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, true, "The alert should've still been triggered")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 2, false, "The alert should've been resolved DESPITE THE ALERT PROVIDER RETURNING AN ERROR. See Alert.Triggered for further explanation.")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 2, false, "The alert should've been resolved DESPITE THE ALERT PROVIDER RETURNING AN ERROR. See Alert.Triggered for further explanation.")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
|
||||
// Make sure that everything's working as expected after a rough patch
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, false, "")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 2, 0, true, "The alert should have triggered")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, true, "The alert should still be triggered")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 2, false, "The alert should have been resolved")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 2, 0, true, "The alert should have triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, true, "The alert should still be triggered")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 2, false, "The alert should have been resolved")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
|
||||
@ -266,7 +266,7 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
|
||||
},
|
||||
}
|
||||
enabled := true
|
||||
service := &core.Service{
|
||||
endpoint := &core.Endpoint{
|
||||
URL: "http://example.com",
|
||||
Alerts: []*alert.Alert{
|
||||
{
|
||||
@ -280,38 +280,38 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, true, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, true, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, false, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, true, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, true, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, false, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
|
||||
// Make sure that everything's working as expected after a rough patch
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 1, 0, true, "")
|
||||
HandleAlerting(service, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 2, 0, true, "")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 1, false, "")
|
||||
HandleAlerting(service, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, service, 0, 2, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 1, 0, true, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 2, 0, true, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 1, false, "")
|
||||
HandleAlerting(endpoint, &core.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
verify(t, endpoint, 0, 2, false, "")
|
||||
}
|
||||
|
||||
func verify(t *testing.T, service *core.Service, expectedNumberOfFailuresInARow, expectedNumberOfSuccessInARow int, expectedTriggered bool, expectedTriggeredReason string) {
|
||||
if service.NumberOfFailuresInARow != expectedNumberOfFailuresInARow {
|
||||
t.Fatalf("service.NumberOfFailuresInARow should've been %d, got %d", expectedNumberOfFailuresInARow, service.NumberOfFailuresInARow)
|
||||
func verify(t *testing.T, endpoint *core.Endpoint, expectedNumberOfFailuresInARow, expectedNumberOfSuccessInARow int, expectedTriggered bool, expectedTriggeredReason string) {
|
||||
if endpoint.NumberOfFailuresInARow != expectedNumberOfFailuresInARow {
|
||||
t.Fatalf("endpoint.NumberOfFailuresInARow should've been %d, got %d", expectedNumberOfFailuresInARow, endpoint.NumberOfFailuresInARow)
|
||||
}
|
||||
if service.NumberOfSuccessesInARow != expectedNumberOfSuccessInARow {
|
||||
t.Fatalf("service.NumberOfSuccessesInARow should've been %d, got %d", expectedNumberOfSuccessInARow, service.NumberOfSuccessesInARow)
|
||||
if endpoint.NumberOfSuccessesInARow != expectedNumberOfSuccessInARow {
|
||||
t.Fatalf("endpoint.NumberOfSuccessesInARow should've been %d, got %d", expectedNumberOfSuccessInARow, endpoint.NumberOfSuccessesInARow)
|
||||
}
|
||||
if service.Alerts[0].Triggered != expectedTriggered {
|
||||
if endpoint.Alerts[0].Triggered != expectedTriggered {
|
||||
if len(expectedTriggeredReason) != 0 {
|
||||
t.Fatal(expectedTriggeredReason)
|
||||
} else {
|
||||
|
||||
@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
// monitoringMutex is used to prevent multiple services from being evaluated at the same time.
|
||||
// monitoringMutex is used to prevent multiple endpoint from being evaluated at the same time.
|
||||
// Without this, conditions using response time may become inaccurate.
|
||||
monitoringMutex sync.Mutex
|
||||
|
||||
@ -23,77 +23,77 @@ var (
|
||||
cancelFunc context.CancelFunc
|
||||
)
|
||||
|
||||
// Monitor loops over each services and starts a goroutine to monitor each services separately
|
||||
// Monitor loops over each endpoint and starts a goroutine to monitor each endpoint separately
|
||||
func Monitor(cfg *config.Config) {
|
||||
ctx, cancelFunc = context.WithCancel(context.Background())
|
||||
for _, service := range cfg.Services {
|
||||
if service.IsEnabled() {
|
||||
for _, endpoint := range cfg.Endpoints {
|
||||
if endpoint.IsEnabled() {
|
||||
// To prevent multiple requests from running at the same time, we'll wait for a little before each iteration
|
||||
time.Sleep(1111 * time.Millisecond)
|
||||
go monitor(service, cfg.Alerting, cfg.Maintenance, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx)
|
||||
go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// monitor monitors a single service in a loop
|
||||
func monitor(service *core.Service, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) {
|
||||
// monitor monitors a single endpoint in a loop
|
||||
func monitor(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) {
|
||||
// Run it immediately on start
|
||||
execute(service, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug)
|
||||
execute(endpoint, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug)
|
||||
// Loop for the next executions
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("[watchdog][monitor] Canceling current execution of group=%s; service=%s", service.Group, service.Name)
|
||||
log.Printf("[watchdog][monitor] Canceling current execution of group=%s; endpoint=%s", endpoint.Group, endpoint.Name)
|
||||
return
|
||||
case <-time.After(service.Interval):
|
||||
execute(service, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug)
|
||||
case <-time.After(endpoint.Interval):
|
||||
execute(endpoint, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func execute(service *core.Service, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool) {
|
||||
func execute(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool) {
|
||||
if !disableMonitoringLock {
|
||||
// By placing the lock here, we prevent multiple services from being monitored at the exact same time, which
|
||||
// By placing the lock here, we prevent multiple endpoints from being monitored at the exact same time, which
|
||||
// could cause performance issues and return inaccurate results
|
||||
monitoringMutex.Lock()
|
||||
}
|
||||
if debug {
|
||||
log.Printf("[watchdog][execute] Monitoring group=%s; service=%s", service.Group, service.Name)
|
||||
log.Printf("[watchdog][execute] Monitoring group=%s; endpoint=%s", endpoint.Group, endpoint.Name)
|
||||
}
|
||||
result := service.EvaluateHealth()
|
||||
result := endpoint.EvaluateHealth()
|
||||
if enabledMetrics {
|
||||
metric.PublishMetricsForService(service, result)
|
||||
metric.PublishMetricsForEndpoint(endpoint, result)
|
||||
}
|
||||
UpdateServiceStatuses(service, result)
|
||||
UpdateEndpointStatuses(endpoint, result)
|
||||
log.Printf(
|
||||
"[watchdog][execute] Monitored group=%s; service=%s; success=%v; errors=%d; duration=%s",
|
||||
service.Group,
|
||||
service.Name,
|
||||
"[watchdog][execute] Monitored group=%s; endpoint=%s; success=%v; errors=%d; duration=%s",
|
||||
endpoint.Group,
|
||||
endpoint.Name,
|
||||
result.Success,
|
||||
len(result.Errors),
|
||||
result.Duration.Round(time.Millisecond),
|
||||
)
|
||||
if !maintenanceConfig.IsUnderMaintenance() {
|
||||
HandleAlerting(service, result, alertingConfig, debug)
|
||||
HandleAlerting(endpoint, result, alertingConfig, debug)
|
||||
} else if debug {
|
||||
log.Println("[watchdog][execute] Not handling alerting because currently in the maintenance window")
|
||||
}
|
||||
if debug {
|
||||
log.Printf("[watchdog][execute] Waiting for interval=%s before monitoring group=%s service=%s again", service.Interval, service.Group, service.Name)
|
||||
log.Printf("[watchdog][execute] Waiting for interval=%s before monitoring group=%s endpoint=%s again", endpoint.Interval, endpoint.Group, endpoint.Name)
|
||||
}
|
||||
if !disableMonitoringLock {
|
||||
monitoringMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateServiceStatuses updates the slice of service statuses
|
||||
func UpdateServiceStatuses(service *core.Service, result *core.Result) {
|
||||
if err := storage.Get().Insert(service, result); err != nil {
|
||||
log.Println("[watchdog][UpdateServiceStatuses] Failed to insert data in storage:", err.Error())
|
||||
// UpdateEndpointStatuses updates the slice of endpoint statuses
|
||||
func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
|
||||
if err := storage.Get().Insert(endpoint, result); err != nil {
|
||||
log.Println("[watchdog][UpdateEndpointStatuses] Failed to insert data in storage:", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops monitoring all services
|
||||
// Shutdown stops monitoring all endpoints
|
||||
func Shutdown() {
|
||||
cancelFunc()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user