fix(logging): Replace log-level parameter by GATUS_LOG_LEVEL env var (#895)
* fix(logging): Replace log-level parameter by GATUS_LOG_LEVEL env var * Improve log message if GATUS_LOG_LEVEL isn't set
This commit is contained in:
@ -2,27 +2,27 @@ package watchdog
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/TwiN/gatus/v5/alerting"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint"
|
||||
"github.com/TwiN/gatus/v5/storage/store"
|
||||
"github.com/TwiN/logr"
|
||||
)
|
||||
|
||||
// HandleAlerting takes care of alerts to resolve and alerts to trigger based on result success or failure
|
||||
func HandleAlerting(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
func HandleAlerting(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config) {
|
||||
if alertingConfig == nil {
|
||||
return
|
||||
}
|
||||
if result.Success {
|
||||
handleAlertsToResolve(ep, result, alertingConfig, debug)
|
||||
handleAlertsToResolve(ep, result, alertingConfig)
|
||||
} else {
|
||||
handleAlertsToTrigger(ep, result, alertingConfig, debug)
|
||||
handleAlertsToTrigger(ep, result, alertingConfig)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config) {
|
||||
ep.NumberOfSuccessesInARow = 0
|
||||
ep.NumberOfFailuresInARow++
|
||||
for _, endpointAlert := range ep.Alerts {
|
||||
@ -31,14 +31,12 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert
|
||||
continue
|
||||
}
|
||||
if endpointAlert.Triggered {
|
||||
if debug {
|
||||
log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' has already been TRIGGERED, skipping", ep.Name, endpointAlert.GetDescription())
|
||||
}
|
||||
logr.Debugf("[watchdog.handleAlertsToTrigger] Alert for endpoint with key=%s with description='%s' has already been TRIGGERED, skipping", ep.Key(), endpointAlert.GetDescription())
|
||||
continue
|
||||
}
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type)
|
||||
if alertProvider != nil {
|
||||
log.Printf("[watchdog.handleAlertsToTrigger] Sending %s alert because alert for endpoint=%s with description='%s' has been TRIGGERED", endpointAlert.Type, ep.Name, endpointAlert.GetDescription())
|
||||
logr.Infof("[watchdog.handleAlertsToTrigger] Sending %s alert because alert for endpoint with key=%s with description='%s' has been TRIGGERED", endpointAlert.Type, ep.Key(), endpointAlert.GetDescription())
|
||||
var err error
|
||||
if os.Getenv("MOCK_ALERT_PROVIDER") == "true" {
|
||||
if os.Getenv("MOCK_ALERT_PROVIDER_ERROR") == "true" {
|
||||
@ -48,27 +46,27 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert
|
||||
err = alertProvider.Send(ep, endpointAlert, result, false)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[watchdog.handleAlertsToTrigger] Failed to send an alert for endpoint=%s: %s", ep.Name, err.Error())
|
||||
logr.Errorf("[watchdog.handleAlertsToTrigger] Failed to send an alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
} else {
|
||||
endpointAlert.Triggered = true
|
||||
if err := store.Get().UpsertTriggeredEndpointAlert(ep, endpointAlert); err != nil {
|
||||
log.Printf("[watchdog.handleAlertsToTrigger] Failed to persist triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
logr.Errorf("[watchdog.handleAlertsToTrigger] Failed to persist triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Printf("[watchdog.handleAlertsToTrigger] Not sending alert of type=%s despite being TRIGGERED, because the provider wasn't configured properly", endpointAlert.Type)
|
||||
logr.Warnf("[watchdog.handleAlertsToTrigger] Not sending alert of type=%s endpoint with key=%s despite being TRIGGERED, because the provider wasn't configured properly", endpointAlert.Type, ep.Key())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToResolve(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config, debug bool) {
|
||||
func handleAlertsToResolve(ep *endpoint.Endpoint, result *endpoint.Result, alertingConfig *alerting.Config) {
|
||||
ep.NumberOfSuccessesInARow++
|
||||
for _, endpointAlert := range ep.Alerts {
|
||||
isStillBelowSuccessThreshold := endpointAlert.SuccessThreshold > ep.NumberOfSuccessesInARow
|
||||
if isStillBelowSuccessThreshold && endpointAlert.IsEnabled() && endpointAlert.Triggered {
|
||||
// Persist NumberOfSuccessesInARow
|
||||
if err := store.Get().UpsertTriggeredEndpointAlert(ep, endpointAlert); err != nil {
|
||||
log.Printf("[watchdog.handleAlertsToResolve] Failed to update triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
logr.Errorf("[watchdog.handleAlertsToResolve] Failed to update triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
}
|
||||
}
|
||||
if !endpointAlert.IsEnabled() || !endpointAlert.Triggered || isStillBelowSuccessThreshold {
|
||||
@ -78,20 +76,20 @@ func handleAlertsToResolve(ep *endpoint.Endpoint, result *endpoint.Result, alert
|
||||
// Further explanation can be found on Alert's Triggered field.
|
||||
endpointAlert.Triggered = false
|
||||
if err := store.Get().DeleteTriggeredEndpointAlert(ep, endpointAlert); err != nil {
|
||||
log.Printf("[watchdog.handleAlertsToResolve] Failed to delete persisted triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
logr.Errorf("[watchdog.handleAlertsToResolve] Failed to delete persisted triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
}
|
||||
if !endpointAlert.IsSendingOnResolved() {
|
||||
continue
|
||||
}
|
||||
alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type)
|
||||
if alertProvider != nil {
|
||||
log.Printf("[watchdog.handleAlertsToResolve] Sending %s alert because alert for endpoint with key=%s with description='%s' has been RESOLVED", endpointAlert.Type, ep.Key(), endpointAlert.GetDescription())
|
||||
logr.Infof("[watchdog.handleAlertsToResolve] Sending %s alert because alert for endpoint with key=%s with description='%s' has been RESOLVED", endpointAlert.Type, ep.Key(), endpointAlert.GetDescription())
|
||||
err := alertProvider.Send(ep, endpointAlert, result, true)
|
||||
if err != nil {
|
||||
log.Printf("[watchdog.handleAlertsToResolve] Failed to send an alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
logr.Errorf("[watchdog.handleAlertsToResolve] Failed to send an alert for endpoint with key=%s: %s", ep.Key(), err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Printf("[watchdog.handleAlertsToResolve] Not sending alert of type=%s despite being RESOLVED, because the provider wasn't configured properly", endpointAlert.Type)
|
||||
logr.Warnf("[watchdog.handleAlertsToResolve] Not sending alert of type=%s for endpoint with key=%s despite being RESOLVED, because the provider wasn't configured properly", endpointAlert.Type, ep.Key())
|
||||
}
|
||||
}
|
||||
ep.NumberOfFailuresInARow = 0
|
||||
|
@ -28,7 +28,6 @@ func TestHandleAlerting(t *testing.T) {
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
Custom: &custom.AlertProvider{
|
||||
URL: "https://twin.sh/health",
|
||||
@ -52,28 +51,28 @@ func TestHandleAlerting(t *testing.T) {
|
||||
}
|
||||
|
||||
verify(t, ep, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 2, 0, true, "The alert should've triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 3, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 4, 0, true, "The alert should still be triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, true, "The alert should still be triggered (because endpoint.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 2, true, "The alert should still be triggered (because endpoint.Alerts[0].SuccessThreshold is 3)")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 3, false, "The alert should've been resolved")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 4, false, "The alert should no longer be triggered")
|
||||
}
|
||||
|
||||
func TestHandleAlertingWhenAlertingConfigIsNil(t *testing.T) {
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER", "true")
|
||||
defer os.Clearenv()
|
||||
HandleAlerting(nil, nil, nil, true)
|
||||
HandleAlerting(nil, nil, nil)
|
||||
}
|
||||
|
||||
func TestHandleAlertingWithBadAlertProvider(t *testing.T) {
|
||||
@ -96,9 +95,9 @@ func TestHandleAlertingWithBadAlertProvider(t *testing.T) {
|
||||
}
|
||||
|
||||
verify(t, ep, 0, 0, false, "The alert shouldn't start triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, &alerting.Config{}, false)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, &alerting.Config{})
|
||||
verify(t, ep, 1, 0, false, "The alert shouldn't have triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, &alerting.Config{}, false)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, &alerting.Config{})
|
||||
verify(t, ep, 2, 0, false, "The alert shouldn't have triggered, because the provider wasn't configured properly")
|
||||
}
|
||||
|
||||
@ -107,7 +106,6 @@ func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButendpointStartFailing
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
Custom: &custom.AlertProvider{
|
||||
URL: "https://twin.sh/health",
|
||||
@ -132,7 +130,7 @@ func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButendpointStartFailing
|
||||
}
|
||||
|
||||
// This test simulate an alert that was already triggered
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 2, 0, true, "The alert was already triggered at the beginning of this test")
|
||||
}
|
||||
|
||||
@ -141,7 +139,6 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *t
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
Custom: &custom.AlertProvider{
|
||||
URL: "https://twin.sh/health",
|
||||
@ -166,7 +163,7 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *t
|
||||
NumberOfFailuresInARow: 1,
|
||||
}
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "The alert should've been resolved")
|
||||
}
|
||||
|
||||
@ -175,7 +172,6 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) {
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
PagerDuty: &pagerduty.AlertProvider{
|
||||
IntegrationKey: "00000000000000000000000000000000",
|
||||
@ -198,10 +194,10 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) {
|
||||
NumberOfFailuresInARow: 0,
|
||||
}
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, true, "")
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "The alert should've been resolved")
|
||||
}
|
||||
|
||||
@ -210,7 +206,6 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPushover(t *testing.T) {
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
Pushover: &pushover.AlertProvider{
|
||||
ApplicationToken: "000000000000000000000000000000",
|
||||
@ -234,10 +229,10 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPushover(t *testing.T) {
|
||||
NumberOfFailuresInARow: 0,
|
||||
}
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, true, "")
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "The alert should've been resolved")
|
||||
}
|
||||
|
||||
@ -403,32 +398,32 @@ func TestHandleAlertingWithProviderThatReturnsAnError(t *testing.T) {
|
||||
},
|
||||
}
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 1, 0, false, "")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 2, 0, false, "The alert should have failed to trigger, because the alert provider is returning an error")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 3, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 4, 0, false, "The alert should still not be triggered, because the alert provider is still returning an error")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 5, 0, true, "The alert should've been triggered because the alert provider is no longer returning an error")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig)
|
||||
verify(t, ep, 0, 1, true, "The alert should've still been triggered")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig)
|
||||
verify(t, ep, 0, 2, false, "The alert should've been resolved DESPITE THE ALERT PROVIDER RETURNING AN ERROR. See Alert.Triggered for further explanation.")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
|
||||
// Make sure that everything's working as expected after a rough patch
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 1, 0, false, "")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, scenario.AlertingConfig)
|
||||
verify(t, ep, 2, 0, true, "The alert should have triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig)
|
||||
verify(t, ep, 0, 1, true, "The alert should still be triggered")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig, true)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, scenario.AlertingConfig)
|
||||
verify(t, ep, 0, 2, false, "The alert should have been resolved")
|
||||
})
|
||||
}
|
||||
@ -440,7 +435,6 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
|
||||
defer os.Clearenv()
|
||||
|
||||
cfg := &config.Config{
|
||||
Debug: true,
|
||||
Alerting: &alerting.Config{
|
||||
Custom: &custom.AlertProvider{
|
||||
URL: "https://twin.sh/health",
|
||||
@ -463,27 +457,27 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, true, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, true, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "true")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "")
|
||||
_ = os.Setenv("MOCK_ALERT_PROVIDER_ERROR", "false")
|
||||
|
||||
// Make sure that everything's working as expected after a rough patch
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 1, 0, true, "")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
|
||||
verify(t, ep, 2, 0, true, "")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 1, false, "")
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug)
|
||||
HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
|
||||
verify(t, ep, 0, 2, false, "")
|
||||
}
|
||||
|
||||
|
@ -2,7 +2,6 @@ package watchdog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -45,7 +44,7 @@ func monitor(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("[watchdog.monitor] Canceling current execution of group=%s; endpoint=%s", ep.Group, ep.Name)
|
||||
logr.Warnf("[watchdog.monitor] Canceling current execution of group=%s; endpoint=%s; key=%s", ep.Group, ep.Name, ep.Key())
|
||||
return
|
||||
case <-time.After(ep.Interval):
|
||||
execute(ep, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics)
|
||||
@ -68,30 +67,30 @@ func execute(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
|
||||
logr.Infof("[watchdog.execute] No connectivity; skipping execution")
|
||||
return
|
||||
}
|
||||
logr.Debugf("[watchdog.execute] Monitoring group=%s; endpoint=%s", ep.Group, ep.Name)
|
||||
logr.Debugf("[watchdog.execute] Monitoring group=%s; endpoint=%s; key=%s", ep.Group, ep.Name, ep.Key())
|
||||
result := ep.EvaluateHealth()
|
||||
if enabledMetrics {
|
||||
metrics.PublishMetricsForEndpoint(ep, result)
|
||||
}
|
||||
UpdateEndpointStatuses(ep, result)
|
||||
if logr.GetThreshold() == logr.LevelDebug && !result.Success {
|
||||
logr.Debugf("[watchdog.execute] Monitored group=%s; endpoint=%s; success=%v; errors=%d; duration=%s; body=%s", ep.Group, ep.Name, result.Success, len(result.Errors), result.Duration.Round(time.Millisecond), result.Body)
|
||||
logr.Debugf("[watchdog.execute] Monitored group=%s; endpoint=%s; key=%s; success=%v; errors=%d; duration=%s; body=%s", ep.Group, ep.Name, ep.Key(), result.Success, len(result.Errors), result.Duration.Round(time.Millisecond), result.Body)
|
||||
} else {
|
||||
logr.Infof("[watchdog.execute] Monitored group=%s; endpoint=%s; success=%v; errors=%d; duration=%s", ep.Group, ep.Name, result.Success, len(result.Errors), result.Duration.Round(time.Millisecond))
|
||||
logr.Infof("[watchdog.execute] Monitored group=%s; endpoint=%s; key=%s; success=%v; errors=%d; duration=%s", ep.Group, ep.Name, ep.Key(), result.Success, len(result.Errors), result.Duration.Round(time.Millisecond))
|
||||
}
|
||||
if !maintenanceConfig.IsUnderMaintenance() {
|
||||
// TODO: Consider moving this after the monitoring lock is unlocked? I mean, how much noise can a single alerting provider cause...
|
||||
HandleAlerting(ep, result, alertingConfig, logr.GetThreshold() == logr.LevelDebug)
|
||||
HandleAlerting(ep, result, alertingConfig)
|
||||
} else {
|
||||
logr.Debugf("[watchdog.execute] Not handling alerting because currently in the maintenance window")
|
||||
logr.Debug("[watchdog.execute] Not handling alerting because currently in the maintenance window")
|
||||
}
|
||||
logr.Debugf("[watchdog.execute] Waiting for interval=%s before monitoring group=%s endpoint=%s again", ep.Interval, ep.Group, ep.Name)
|
||||
logr.Debugf("[watchdog.execute] Waiting for interval=%s before monitoring group=%s endpoint=%s (key=%s) again", ep.Interval, ep.Group, ep.Name, ep.Key())
|
||||
}
|
||||
|
||||
// UpdateEndpointStatuses updates the slice of endpoint statuses
|
||||
func UpdateEndpointStatuses(ep *endpoint.Endpoint, result *endpoint.Result) {
|
||||
if err := store.Get().Insert(ep, result); err != nil {
|
||||
logr.Errorf("[watchdog.UpdateEndpointStatuses] Failed to insert result in storage:", err.Error())
|
||||
logr.Errorf("[watchdog.UpdateEndpointStatuses] Failed to insert result in storage: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user