feat(alerting): Persist triggered alerts across application restart (#764)

* feat(alerting): Persist triggered alerts across application restart

Fixes #679

* test(alerting): Add numerous tests related to alerts
This commit is contained in:
TwiN
2024-05-15 21:29:45 -04:00
committed by GitHub
parent 9d151fcdb4
commit f2c5f5911c
13 changed files with 822 additions and 72 deletions

View File

@ -1,7 +1,10 @@
package alert
import (
"crypto/sha256"
"encoding/hex"
"errors"
"strconv"
"strings"
)
@ -26,6 +29,9 @@ type Alert struct {
// FailureThreshold is the number of failures in a row needed before triggering the alert
FailureThreshold int `yaml:"failure-threshold"`
// SuccessThreshold defines how many successful executions must happen in a row before an ongoing incident is marked as resolved
SuccessThreshold int `yaml:"success-threshold"`
// Description of the alert. Will be included in the alert sent.
//
// This is a pointer, because it is populated by YAML and we need to know whether it was explicitly set to a value
@ -38,9 +44,6 @@ type Alert struct {
// or not for provider.ParseWithDefaultAlert to work. Use Alert.IsSendingOnResolved() for a non-pointer
SendOnResolved *bool `yaml:"send-on-resolved"`
// SuccessThreshold defines how many successful executions must happen in a row before an ongoing incident is marked as resolved
SuccessThreshold int `yaml:"success-threshold"`
// ResolveKey is an optional field that is used by some providers (i.e. PagerDuty's dedup_key) to resolve
// ongoing/triggered incidents
ResolveKey string `yaml:"-"`
@ -94,3 +97,17 @@ func (alert *Alert) IsSendingOnResolved() bool {
}
return *alert.SendOnResolved
}
// Checksum returns a checksum of the alert
// Used to determine which persisted triggered alert should be deleted on application start
func (alert *Alert) Checksum() string {
hash := sha256.New()
hash.Write([]byte(string(alert.Type) + "_" +
strconv.FormatBool(alert.IsEnabled()) + "_" +
strconv.FormatBool(alert.IsSendingOnResolved()) + "_" +
strconv.Itoa(alert.SuccessThreshold) + "_" +
strconv.Itoa(alert.FailureThreshold) + "_" +
alert.GetDescription()),
)
return hex.EncodeToString(hash.Sum(nil))
}