Start working on #12: Fix inconsistencies in alerting providers
This commit is contained in:
TwinProduction
@ -1,158 +0,0 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TwinProduction/gatus/config"
|
||||
"github.com/TwinProduction/gatus/core"
|
||||
"log"
|
||||
)
|
||||
|
||||
// Handle takes care of alerts to resolve and alerts to trigger based on result success or failure
|
||||
func Handle(service *core.Service, result *core.Result) {
|
||||
cfg := config.Get()
|
||||
if cfg.Alerting == nil {
|
||||
return
|
||||
}
|
||||
if result.Success {
|
||||
handleAlertsToResolve(service, result, cfg)
|
||||
} else {
|
||||
handleAlertsToTrigger(service, result, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *config.Config) {
|
||||
service.NumberOfSuccessesInARow = 0
|
||||
service.NumberOfFailuresInARow++
|
||||
for _, alert := range service.Alerts {
|
||||
// If the alert hasn't been triggered, move to the next one
|
||||
if !alert.Enabled || alert.FailureThreshold != service.NumberOfFailuresInARow {
|
||||
continue
|
||||
}
|
||||
if alert.Triggered {
|
||||
if cfg.Debug {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var alertProvider *core.CustomAlertProvider
|
||||
if alert.Type == core.SlackAlert {
|
||||
if len(cfg.Alerting.Slack) > 0 {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
|
||||
alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false)
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
|
||||
}
|
||||
} else if alert.Type == core.PagerDutyAlert {
|
||||
if len(cfg.Alerting.PagerDuty) > 0 {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
|
||||
alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
|
||||
}
|
||||
} else if alert.Type == core.TwilioAlert {
|
||||
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
|
||||
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
|
||||
}
|
||||
} else if alert.Type == core.CustomAlert {
|
||||
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
|
||||
alertProvider = &core.CustomAlertProvider{
|
||||
Url: cfg.Alerting.Custom.Url,
|
||||
Method: cfg.Alerting.Custom.Method,
|
||||
Body: cfg.Alerting.Custom.Body,
|
||||
Headers: cfg.Alerting.Custom.Headers,
|
||||
}
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
|
||||
}
|
||||
}
|
||||
if alertProvider != nil {
|
||||
// TODO: retry on error
|
||||
var err error
|
||||
if alert.Type == core.PagerDutyAlert {
|
||||
var body []byte
|
||||
body, err = alertProvider.Send(service.Name, alert.Description, true)
|
||||
if err == nil {
|
||||
var response pagerDutyResponse
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error())
|
||||
} else {
|
||||
alert.ResolveKey = response.DedupKey
|
||||
}
|
||||
}
|
||||
} else {
|
||||
_, err = alertProvider.Send(service.Name, alert.Description, false)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[alerting][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
|
||||
} else {
|
||||
alert.Triggered = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *config.Config) {
|
||||
service.NumberOfSuccessesInARow++
|
||||
for _, alert := range service.Alerts {
|
||||
if !alert.Enabled || !alert.Triggered || alert.SuccessThreshold > service.NumberOfSuccessesInARow {
|
||||
continue
|
||||
}
|
||||
alert.Triggered = false
|
||||
if !alert.SendOnResolved {
|
||||
continue
|
||||
}
|
||||
var alertProvider *core.CustomAlertProvider
|
||||
if alert.Type == core.SlackAlert {
|
||||
if len(cfg.Alerting.Slack) > 0 {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
|
||||
alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true)
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
|
||||
}
|
||||
} else if alert.Type == core.PagerDutyAlert {
|
||||
if len(cfg.Alerting.PagerDuty) > 0 {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
|
||||
alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
|
||||
}
|
||||
} else if alert.Type == core.TwilioAlert {
|
||||
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
|
||||
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
|
||||
}
|
||||
} else if alert.Type == core.CustomAlert {
|
||||
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
|
||||
alertProvider = &core.CustomAlertProvider{
|
||||
Url: cfg.Alerting.Custom.Url,
|
||||
Method: cfg.Alerting.Custom.Method,
|
||||
Body: cfg.Alerting.Custom.Body,
|
||||
Headers: cfg.Alerting.Custom.Headers,
|
||||
}
|
||||
} else {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
|
||||
}
|
||||
}
|
||||
if alertProvider != nil {
|
||||
// TODO: retry on error
|
||||
_, err := alertProvider.Send(service.Name, alert.Description, true)
|
||||
if err != nil {
|
||||
log.Printf("[alerting][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
|
||||
} else {
|
||||
if alert.Type == core.PagerDutyAlert {
|
||||
alert.ResolveKey = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
service.NumberOfFailuresInARow = 0
|
||||
}
|
||||
8
alerting/config.go
Normal file
8
alerting/config.go
Normal file
@ -0,0 +1,8 @@
|
||||
package alerting
|
||||
|
||||
type Config struct {
|
||||
Slack *SlackAlertProvider `yaml:"slack"`
|
||||
PagerDuty *PagerDutyAlertProvider `yaml:"pagerduty"`
|
||||
Twilio *TwilioAlertProvider `yaml:"twilio"`
|
||||
Custom *CustomAlertProvider `yaml:"custom"`
|
||||
}
|
||||
80
alerting/custom.go
Normal file
80
alerting/custom.go
Normal file
@ -0,0 +1,80 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/TwinProduction/gatus/client"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type CustomAlertProvider struct {
|
||||
Url string `yaml:"url"`
|
||||
Method string `yaml:"method,omitempty"`
|
||||
Body string `yaml:"body,omitempty"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
}
|
||||
|
||||
func (provider *CustomAlertProvider) IsValid() bool {
|
||||
return len(provider.Url) > 0
|
||||
}
|
||||
|
||||
func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
|
||||
body := provider.Body
|
||||
providerUrl := provider.Url
|
||||
method := provider.Method
|
||||
if strings.Contains(body, "[ALERT_DESCRIPTION]") {
|
||||
body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
|
||||
}
|
||||
if strings.Contains(body, "[SERVICE_NAME]") {
|
||||
body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
|
||||
}
|
||||
if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
|
||||
if resolved {
|
||||
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
|
||||
} else {
|
||||
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
|
||||
}
|
||||
}
|
||||
if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
|
||||
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
|
||||
}
|
||||
if strings.Contains(providerUrl, "[SERVICE_NAME]") {
|
||||
providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
|
||||
}
|
||||
if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
|
||||
if resolved {
|
||||
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
|
||||
} else {
|
||||
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
|
||||
}
|
||||
}
|
||||
if len(method) == 0 {
|
||||
method = "GET"
|
||||
}
|
||||
bodyBuffer := bytes.NewBuffer([]byte(body))
|
||||
request, _ := http.NewRequest(method, providerUrl, bodyBuffer)
|
||||
for k, v := range provider.Headers {
|
||||
request.Header.Set(k, v)
|
||||
}
|
||||
return request
|
||||
}
|
||||
|
||||
// Send a request to the alert provider and return the body
|
||||
func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
|
||||
request := provider.buildRequest(serviceName, alertDescription, resolved)
|
||||
response, err := client.GetHttpClient().Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if response.StatusCode > 399 {
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
|
||||
} else {
|
||||
return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
return ioutil.ReadAll(response.Body)
|
||||
}
|
||||
59
alerting/custom_test.go
Normal file
59
alerting/custom_test.go
Normal file
@ -0,0 +1,59 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCustomAlertProvider_IsValid(t *testing.T) {
|
||||
invalidProvider := CustomAlertProvider{Url: ""}
|
||||
if invalidProvider.IsValid() {
|
||||
t.Error("provider shouldn't have been valid")
|
||||
}
|
||||
validProvider := CustomAlertProvider{Url: "http://example.com"}
|
||||
if !validProvider.IsValid() {
|
||||
t.Error("provider should've been valid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomAlertProvider_buildRequestWhenResolved(t *testing.T) {
|
||||
const (
|
||||
ExpectedUrl = "http://example.com/service-name"
|
||||
ExpectedBody = "service-name,alert-description,RESOLVED"
|
||||
)
|
||||
customAlertProvider := &CustomAlertProvider{
|
||||
Url: "http://example.com/[SERVICE_NAME]",
|
||||
Method: "GET",
|
||||
Body: "[SERVICE_NAME],[ALERT_DESCRIPTION],[ALERT_TRIGGERED_OR_RESOLVED]",
|
||||
Headers: nil,
|
||||
}
|
||||
request := customAlertProvider.buildRequest("service-name", "alert-description", true)
|
||||
if request.URL.String() != ExpectedUrl {
|
||||
t.Error("expected URL to be", ExpectedUrl, "was", request.URL.String())
|
||||
}
|
||||
body, _ := ioutil.ReadAll(request.Body)
|
||||
if string(body) != ExpectedBody {
|
||||
t.Error("expected body to be", ExpectedBody, "was", string(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomAlertProvider_buildRequestWhenTriggered(t *testing.T) {
|
||||
const (
|
||||
ExpectedUrl = "http://example.com/service-name"
|
||||
ExpectedBody = "service-name,alert-description,TRIGGERED"
|
||||
)
|
||||
customAlertProvider := &CustomAlertProvider{
|
||||
Url: "http://example.com/[SERVICE_NAME]",
|
||||
Method: "GET",
|
||||
Body: "[SERVICE_NAME],[ALERT_DESCRIPTION],[ALERT_TRIGGERED_OR_RESOLVED]",
|
||||
Headers: nil,
|
||||
}
|
||||
request := customAlertProvider.buildRequest("service-name", "alert-description", false)
|
||||
if request.URL.String() != ExpectedUrl {
|
||||
t.Error("expected URL to be", ExpectedUrl, "was", request.URL.String())
|
||||
}
|
||||
body, _ := ioutil.ReadAll(request.Body)
|
||||
if string(body) != ExpectedBody {
|
||||
t.Error("expected body to be", ExpectedBody, "was", string(body))
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,35 @@
|
||||
package alerting
|
||||
|
||||
type pagerDutyResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
DedupKey string `json:"dedup_key"`
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TwinProduction/gatus/core"
|
||||
)
|
||||
|
||||
type PagerDutyAlertProvider struct {
|
||||
IntegrationKey string `yaml:"integration-key"`
|
||||
}
|
||||
|
||||
func (provider *PagerDutyAlertProvider) IsValid() bool {
|
||||
return len(provider.IntegrationKey) == 32
|
||||
}
|
||||
|
||||
// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
|
||||
func (provider *PagerDutyAlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *CustomAlertProvider {
|
||||
return &CustomAlertProvider{
|
||||
Url: "https://events.pagerduty.com/v2/enqueue",
|
||||
Method: "POST",
|
||||
Body: fmt.Sprintf(`{
|
||||
"routing_key": "%s",
|
||||
"dedup_key": "%s",
|
||||
"event_action": "%s",
|
||||
"payload": {
|
||||
"summary": "%s",
|
||||
"source": "%s",
|
||||
"severity": "critical"
|
||||
}
|
||||
}`, provider.IntegrationKey, resolveKey, eventAction, message, service.Name),
|
||||
Headers: map[string]string{
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
14
alerting/pagerduty_test.go
Normal file
14
alerting/pagerduty_test.go
Normal file
@ -0,0 +1,14 @@
|
||||
package alerting
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestPagerDutyAlertProvider_IsValid(t *testing.T) {
|
||||
invalidProvider := PagerDutyAlertProvider{IntegrationKey: ""}
|
||||
if invalidProvider.IsValid() {
|
||||
t.Error("provider shouldn't have been valid")
|
||||
}
|
||||
validProvider := PagerDutyAlertProvider{IntegrationKey: "00000000000000000000000000000000"}
|
||||
if !validProvider.IsValid() {
|
||||
t.Error("provider should've been valid")
|
||||
}
|
||||
}
|
||||
59
alerting/slack.go
Normal file
59
alerting/slack.go
Normal file
@ -0,0 +1,59 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TwinProduction/gatus/core"
|
||||
)
|
||||
|
||||
type SlackAlertProvider struct {
|
||||
WebhookUrl string `yaml:"webhook-url"`
|
||||
}
|
||||
|
||||
func (provider *SlackAlertProvider) IsValid() bool {
|
||||
return len(provider.WebhookUrl) > 0
|
||||
}
|
||||
|
||||
func (provider *SlackAlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *CustomAlertProvider {
|
||||
var message string
|
||||
var color string
|
||||
if resolved {
|
||||
message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold)
|
||||
color = "#36A64F"
|
||||
} else {
|
||||
message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold)
|
||||
color = "#DD0000"
|
||||
}
|
||||
var results string
|
||||
for _, conditionResult := range result.ConditionResults {
|
||||
var prefix string
|
||||
if conditionResult.Success {
|
||||
prefix = ":heavy_check_mark:"
|
||||
} else {
|
||||
prefix = ":x:"
|
||||
}
|
||||
results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
|
||||
}
|
||||
return &CustomAlertProvider{
|
||||
Url: provider.WebhookUrl,
|
||||
Method: "POST",
|
||||
Body: fmt.Sprintf(`{
|
||||
"text": "",
|
||||
"attachments": [
|
||||
{
|
||||
"title": ":helmet_with_white_cross: Gatus",
|
||||
"text": "%s:\n> %s",
|
||||
"short": false,
|
||||
"color": "%s",
|
||||
"fields": [
|
||||
{
|
||||
"title": "Condition results",
|
||||
"value": "%s",
|
||||
"short": false
|
||||
}
|
||||
]
|
||||
},
|
||||
]
|
||||
}`, message, alert.Description, color, results),
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
}
|
||||
}
|
||||
14
alerting/slack_test.go
Normal file
14
alerting/slack_test.go
Normal file
@ -0,0 +1,14 @@
|
||||
package alerting
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSlackAlertProvider_IsValid(t *testing.T) {
|
||||
invalidProvider := SlackAlertProvider{WebhookUrl: ""}
|
||||
if invalidProvider.IsValid() {
|
||||
t.Error("provider shouldn't have been valid")
|
||||
}
|
||||
validProvider := SlackAlertProvider{WebhookUrl: "http://example.com"}
|
||||
if !validProvider.IsValid() {
|
||||
t.Error("provider should've been valid")
|
||||
}
|
||||
}
|
||||
34
alerting/twilio.go
Normal file
34
alerting/twilio.go
Normal file
@ -0,0 +1,34 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type TwilioAlertProvider struct {
|
||||
SID string `yaml:"sid"`
|
||||
Token string `yaml:"token"`
|
||||
From string `yaml:"from"`
|
||||
To string `yaml:"to"`
|
||||
}
|
||||
|
||||
func (provider *TwilioAlertProvider) IsValid() bool {
|
||||
return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
|
||||
}
|
||||
|
||||
func (provider *TwilioAlertProvider) ToCustomAlertProvider(message string) *CustomAlertProvider {
|
||||
return &CustomAlertProvider{
|
||||
Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
|
||||
Method: "POST",
|
||||
Body: url.Values{
|
||||
"To": {provider.To},
|
||||
"From": {provider.From},
|
||||
"Body": {message},
|
||||
}.Encode(),
|
||||
Headers: map[string]string{
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
|
||||
},
|
||||
}
|
||||
}
|
||||
19
alerting/twilio_test.go
Normal file
19
alerting/twilio_test.go
Normal file
@ -0,0 +1,19 @@
|
||||
package alerting
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTwilioAlertProvider_IsValid(t *testing.T) {
|
||||
invalidProvider := TwilioAlertProvider{}
|
||||
if invalidProvider.IsValid() {
|
||||
t.Error("provider shouldn't have been valid")
|
||||
}
|
||||
validProvider := TwilioAlertProvider{
|
||||
SID: "1",
|
||||
Token: "1",
|
||||
From: "1",
|
||||
To: "1",
|
||||
}
|
||||
if !validProvider.IsValid() {
|
||||
t.Error("provider should've been valid")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user