feat(alerting): Add Incident.io alerting provider (#972)

* feat(alerting): added incident.io provider alerting. * Tests: added incident.io provider unit tests. * Documentation: added incidentio documentation. * Refactor: Changed documentation + types to an alphabetical order. * Refactor: change wrong comment. Co-authored-by: Maksim Zhylinski <maksim@zhylinski.me> * Update README.md Co-authored-by: Maksim Zhylinski <maksim@zhylinski.me> * Update alerting/provider/incidentio/incident_io.go Co-authored-by: Julien Limoges <limoges@users.noreply.github.com> * Update alerting/provider/incidentio/incident_io.go Co-authored-by: Julien Limoges <limoges@users.noreply.github.com> * Update alerting/provider/incidentio/incident_io.go Co-authored-by: Julien Limoges <limoges@users.noreply.github.com> * Update alerting/provider/incidentio/incident_io.go Co-authored-by: Julien Limoges <limoges@users.noreply.github.com> * Refactor: changed alertSourceID to url. * Refactor: changed documentation. * Refactor: refactored tests, removed status from config. * Readme: updated docs. * Refactor: removed duplication key in favor of ResolveKey. * Refactor: change variable format. * Feat + Test: added support for passing metadata and source url, added resolved-with-metadata-source-url test case. * Refactor: chaned variable naming * Update alerting/config.go * Update README.md * Update README.md * Update README.md * Apply suggestions from code review * Refactor: sort var by abc --------- Co-authored-by: Maksim Zhylinski <maksim@zhylinski.me> Co-authored-by: Julien Limoges <limoges@users.noreply.github.com> Co-authored-by: TwiN <twin@linux.com>
2025-02-07 02:21:35 +02:00
parent ea2b7c4bdf
commit 541a70584d
7 changed files with 636 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -59,6 +59,7 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga
    - [Configuring GitLab alerts](#configuring-gitlab-alerts)
    - [Configuring Google Chat alerts](#configuring-google-chat-alerts)
    - [Configuring Gotify alerts](#configuring-gotify-alerts)
+    - [Configuring Incident.io alerts](#configuring-incidentio-alerts)
    - [Configuring JetBrains Space alerts](#configuring-jetbrains-space-alerts)
    - [Configuring Matrix alerts](#configuring-matrix-alerts)
    - [Configuring Mattermost alerts](#configuring-mattermost-alerts)
@ -579,6 +580,7 @@ endpoints:
 | `alerting.gitlab`          | Configuration for alerts of type `gitlab`. <br />See [Configuring GitLab alerts](#configuring-gitlab-alerts).                           | `{}`    |
 | `alerting.googlechat`      | Configuration for alerts of type `googlechat`. <br />See [Configuring Google Chat alerts](#configuring-google-chat-alerts).             | `{}`    |
 | `alerting.gotify`          | Configuration for alerts of type `gotify`. <br />See [Configuring Gotify alerts](#configuring-gotify-alerts).                           | `{}`    |
+| `alerting.incident-io`     | Configuration for alerts of type `incident-io`. <br />See [Configuring Incident.io alerts](#configuring-incidentio-alerts).              | `{}`    |
 | `alerting.jetbrainsspace`  | Configuration for alerts of type `jetbrainsspace`. <br />See [Configuring JetBrains Space alerts](#configuring-jetbrains-space-alerts). | `{}`    |
 | `alerting.matrix`          | Configuration for alerts of type `matrix`. <br />See [Configuring Matrix alerts](#configuring-matrix-alerts).                           | `{}`    |
 | `alerting.mattermost`      | Configuration for alerts of type `mattermost`. <br />See [Configuring Mattermost alerts](#configuring-mattermost-alerts).               | `{}`    |
@ -904,6 +906,42 @@ Here's an example of what the notifications look like:

 ![Gotify notifications](.github/assets/gotify-alerts.png)

+#### Configuring Incident.io alerts
+| Parameter                          | Description                                                                                | Default       |
+|:-----------------------------------|:-------------------------------------------------------------------------------------------|:--------------|
+| `alerting.incident-io`                   | Configuration for alerts of type `incident-io`                                                   | `{}`          |
+| `alerting.incident-io.url`       | url to trigger an alert event.                                                                         | Required `""` |
+| `alerting.incident-io.auth-token`     | Token that is used for authentication. |  Required `""`          |
+| `alerting.incident-io.overrides`         | List of overrides that may be prioritized over the default configuration                   | `[]`          |
+| `alerting.incident-io.default-alert`     | Default alert configuration. <br />See [Setting a default alert](#setting-a-default-alert) | N/A           |
+| `alerting.incident-io.overrides[].group` | Endpoint group for which the configuration will be overridden by this configuration        | `""`          |
+| `alerting.incident-io.overrides[].*`     | See `alerting.incident-io.*` parameters                                                          | `{}`          |
+
+```yaml
+alerting:
+  incident-io:
+    url: "*****************"
+    auth-token: "********************************************"
+
+endpoints:
+  - name: website
+    url: "https://twin.sh/health"
+    interval: 30s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 300"
+    alerts:
+      - type: incident-io
+        description: "healthcheck failed"
+        send-on-resolved: true
+```
+in order to get the required alert source config id and authentication token, you must configure an HTTP alert source.
+
+> **_NOTE:_**  the source config id is of the form `api.incident.io/v2/alert_events/http/$ID` and the token is expected to be passed as a bearer token like so: `Authorization: Bearer $TOKEN`
+
+
+> **_NOTE:_** ```

 #### Configuring JetBrains Space alerts
 | Parameter                                   | Description                                                                                | Default       |
--- a/alerting/alert/type.go
+++ b/alerting/alert/type.go
@ -32,6 +32,9 @@ const (
 	// TypeGotify is the Type for the gotify alerting provider
 	TypeGotify Type = "gotify"

+	// TypeIncidentIO is the Type for the incident-io alerting provider
+	TypeIncidentIO Type = "incident-io"
+
 	// TypeJetBrainsSpace is the Type for the jetbrains alerting provider
 	TypeJetBrainsSpace Type = "jetbrainsspace"

--- a/alerting/config.go
+++ b/alerting/config.go
@ -15,6 +15,7 @@ import (
 	"github.com/TwiN/gatus/v5/alerting/provider/gitlab"
 	"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
 	"github.com/TwiN/gatus/v5/alerting/provider/gotify"
+	"github.com/TwiN/gatus/v5/alerting/provider/incidentio"
 	"github.com/TwiN/gatus/v5/alerting/provider/jetbrainsspace"
 	"github.com/TwiN/gatus/v5/alerting/provider/matrix"
 	"github.com/TwiN/gatus/v5/alerting/provider/mattermost"
@ -61,6 +62,9 @@ type Config struct {
 	// Gotify is the configuration for the gotify alerting provider
 	Gotify *gotify.AlertProvider `yaml:"gotify,omitempty"`

+	// IncidentIO is the configuration for the incident-io alerting provider
+	IncidentIO *incidentio.AlertProvider `yaml:"incident-io,omitempty"`
+
 	// JetBrainsSpace is the configuration for the jetbrains space alerting provider
 	JetBrainsSpace *jetbrainsspace.AlertProvider `yaml:"jetbrainsspace,omitempty"`

--- a/alerting/provider/incidentio/incident_io.go
+++ b/alerting/provider/incidentio/incident_io.go
@ -0,0 +1,207 @@
+package incidentio
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/logr"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	restAPIUrl = "https://api.incident.io/v2/alert_events/http/"
+)
+
+var (
+	ErrURLNotSet              = errors.New("url not set")
+	ErrDuplicateGroupOverride = errors.New("duplicate group override")
+	ErrAuthTokenNotSet        = errors.New("auth-token not set")
+)
+
+type Config struct {
+	URL       string                 `yaml:"url,omitempty"`
+	AuthToken string                 `yaml:"auth-token,omitempty"`
+	SourceURL string                 `yaml:"source-url,omitempty"`
+	Metadata  map[string]interface{} `yaml:"metadata,omitempty"`
+}
+
+func (cfg *Config) Validate() error {
+	if len(cfg.URL) == 0 {
+		return ErrURLNotSet
+	}
+	if len(cfg.AuthToken) == 0 {
+		return ErrAuthTokenNotSet
+	}
+	return nil
+}
+
+func (cfg *Config) Merge(override *Config) {
+	if len(override.URL) > 0 {
+		cfg.URL = override.URL
+	}
+	if len(override.AuthToken) > 0 {
+		cfg.AuthToken = override.AuthToken
+	}
+	if len(override.SourceURL) > 0 {
+		cfg.SourceURL = override.SourceURL
+	}
+	if len(override.Metadata) > 0 {
+		cfg.Metadata = override.Metadata
+	}
+}
+
+// AlertProvider is the configuration necessary for sending an alert using incident.io
+type AlertProvider struct {
+	DefaultConfig Config `yaml:",inline"`
+
+	// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
+	DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
+
+	// Overrides is a list of Override that may be prioritized over the default configuration
+	Overrides []Override `yaml:"overrides,omitempty"`
+}
+
+type Override struct {
+	Group  string `yaml:"group"`
+	Config `yaml:",inline"`
+}
+
+func (provider *AlertProvider) Validate() error {
+	registeredGroups := make(map[string]bool)
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" {
+				return ErrDuplicateGroupOverride
+			}
+			registeredGroups[override.Group] = true
+		}
+	}
+	return provider.DefaultConfig.Validate()
+}
+
+func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
+	cfg, err := provider.GetConfig(ep.Group, alert)
+	if err != nil {
+		return err
+	}
+	buffer := bytes.NewBuffer(provider.buildRequestBody(cfg, ep, alert, result, resolved))
+	req, err := http.NewRequest(http.MethodPost, cfg.URL, buffer)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+cfg.AuthToken)
+	response, err := client.GetHTTPClient(nil).Do(req)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode > 399 {
+		body, _ := io.ReadAll(response.Body)
+		return fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
+	}
+	incidentioResponse := Response{}
+	err = json.NewDecoder(response.Body).Decode(&incidentioResponse)
+	if err != nil {
+		// Silently fail. We don't want to create tons of alerts just because we failed to parse the body.
+		logr.Errorf("[incident-io.Send] Ran into error decoding pagerduty response: %s", err.Error())
+	}
+	alert.ResolveKey = incidentioResponse.DeduplicationKey
+	return err
+}
+
+type Body struct {
+	AlertSourceConfigID string                 `json:"alert_source_config_id"`
+	Status              string                 `json:"status"`
+	Title               string                 `json:"title"`
+	DeduplicationKey    string                 `json:"deduplication_key,omitempty"`
+	Description         string                 `json:"description,omitempty"`
+	SourceURL           string                 `json:"source_url,omitempty"`
+	Metadata            map[string]interface{} `json:"metadata,omitempty"`
+}
+
+type Response struct {
+	DeduplicationKey string `json:"deduplication_key"`
+}
+
+func (provider *AlertProvider) buildRequestBody(cfg *Config, ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
+	var message, formattedConditionResults, status string
+	if resolved {
+		message = "An alert has been resolved after passing successfully " + strconv.Itoa(alert.SuccessThreshold) + " time(s) in a row"
+		status = "resolved"
+	} else {
+		message = "An alert has been triggered due to having failed " + strconv.Itoa(alert.FailureThreshold) + " time(s) in a row"
+		status = "firing"
+	}
+	for _, conditionResult := range result.ConditionResults {
+		var prefix string
+		if conditionResult.Success {
+			prefix = "🟢"
+		} else {
+			prefix = "🔴"
+		}
+		// No need for \n since incident.io trims it anyways.
+		formattedConditionResults += fmt.Sprintf(" %s %s ", prefix, conditionResult.Condition)
+	}
+	if len(alert.GetDescription()) > 0 {
+		message += " with the following description: " + alert.GetDescription()
+	}
+
+	message += fmt.Sprintf(" and the following conditions: %s ", formattedConditionResults)
+	var body []byte
+	alertSourceID := strings.Split(cfg.URL, restAPIUrl)[1]
+	body, _ = json.Marshal(Body{
+		AlertSourceConfigID: alertSourceID,
+		Title:               "Gatus: " + ep.DisplayName(),
+		Status:              status,
+		DeduplicationKey:    alert.ResolveKey,
+		Description:         message,
+		SourceURL:           cfg.SourceURL,
+		Metadata:            cfg.Metadata,
+	})
+	fmt.Printf("%v", string(body))
+	return body
+
+}
+func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
+	cfg := provider.DefaultConfig
+	// Handle group overrides
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if group == override.Group {
+				cfg.Merge(&override.Config)
+				break
+			}
+		}
+	}
+	// Handle alert overrides
+	if len(alert.ProviderOverride) != 0 {
+		overrideConfig := Config{}
+		if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
+			return nil, err
+		}
+		cfg.Merge(&overrideConfig)
+	}
+	// Validate the configuration
+	err := cfg.Validate()
+	return &cfg, err
+}
+
+// GetDefaultAlert returns the provider's default alert configuration
+func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
+	return provider.DefaultAlert
+}
+
+func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
+	_, err := provider.GetConfig(group, alert)
+	return err
+}
--- a/alerting/provider/incidentio/incident_io_test.go
+++ b/alerting/provider/incidentio/incident_io_test.go
@ -0,0 +1,380 @@
+package incidentio
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/gatus/v5/test"
+)
+
+func TestAlertProvider_Validate(t *testing.T) {
+	scenarios := []struct {
+		name     string
+		provider AlertProvider
+		expected bool
+	}{
+		{
+			name: "valid",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					URL:       "some-id",
+					AuthToken: "some-token",
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "invalid-missing-auth-token",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					URL: "some-id",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "invalid-missing-alert-source-config-id",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					AuthToken: "some-token",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "valid-override",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					AuthToken: "some-token",
+					URL:       "some-id",
+				},
+				Overrides: []Override{{Group: "core", Config: Config{URL: "another-id"}}},
+			},
+			expected: true,
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			err := scenario.provider.Validate()
+			if scenario.expected && err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+			if !scenario.expected && err == nil {
+				t.Error("expected error, got none")
+			}
+		})
+	}
+}
+
+func TestAlertProvider_Send(t *testing.T) {
+	defer client.InjectHTTPClient(nil)
+	firstDescription := "description-1"
+	secondDescription := "description-2"
+	restAPIUrl := "https://api.incident.io/v2/alert_events/http/"
+	scenarios := []struct {
+		Name             string
+		Provider         AlertProvider
+		Alert            alert.Alert
+		Resolved         bool
+		MockRoundTripper test.MockRoundTripper
+		ExpectedError    bool
+	}{
+		{
+			Name: "triggered",
+			Provider: AlertProvider{DefaultConfig: Config{
+				URL:       restAPIUrl + "some-id",
+				AuthToken: "some-token",
+			}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				var b bytes.Buffer
+
+				response := Response{DeduplicationKey: "some-key"}
+				json.NewEncoder(&b).Encode(response)
+				reader := io.NopCloser(&b)
+				return &http.Response{StatusCode: http.StatusAccepted, Body: reader}
+			}),
+			ExpectedError: false,
+		},
+		{
+			Name: "triggered-error",
+			Provider: AlertProvider{DefaultConfig: Config{
+				URL:       restAPIUrl + "some-id",
+				AuthToken: "some-token",
+			}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusInternalServerError, Body: http.NoBody}
+			}),
+			ExpectedError: true,
+		},
+		{
+			Name: "resolved",
+			Provider: AlertProvider{DefaultConfig: Config{
+				URL:       restAPIUrl + "some-id",
+				AuthToken: "some-token",
+			}},
+			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: true,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				var b bytes.Buffer
+				response := Response{DeduplicationKey: "some-key"}
+				json.NewEncoder(&b).Encode(response)
+				reader := io.NopCloser(&b)
+				return &http.Response{StatusCode: http.StatusAccepted, Body: reader}
+			}),
+			ExpectedError: false,
+		},
+		{
+			Name:     "resolved-error",
+			Provider: AlertProvider{DefaultConfig: Config{}},
+			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: true,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusInternalServerError, Body: http.NoBody}
+			}),
+			ExpectedError: true,
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			client.InjectHTTPClient(&http.Client{Transport: scenario.MockRoundTripper})
+			err := scenario.Provider.Send(
+				&endpoint.Endpoint{Name: "endpoint-name"},
+				&scenario.Alert,
+				&endpoint.Result{
+					ConditionResults: []*endpoint.ConditionResult{
+						{Condition: "[CONNECTED] == true", Success: scenario.Resolved},
+						{Condition: "[STATUS] == 200", Success: scenario.Resolved},
+					},
+				},
+				scenario.Resolved,
+			)
+			if scenario.ExpectedError && err == nil {
+				t.Error("expected error, got none")
+			}
+			if !scenario.ExpectedError && err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+		})
+	}
+}
+
+func TestAlertProvider_BuildRequestBody(t *testing.T) {
+	firstDescription := "description-1"
+	secondDescription := "description-2"
+	restAPIUrl := "https://api.incident.io/v2/alert_events/http/"
+	scenarios := []struct {
+		Name         string
+		Provider     AlertProvider
+		Alert        alert.Alert
+		Resolved     bool
+		ExpectedBody string
+	}{
+		{
+			Name:         "triggered",
+			Provider:     AlertProvider{DefaultConfig: Config{URL: restAPIUrl + "some-id", AuthToken: "some-token"}},
+			Alert:        alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved:     false,
+			ExpectedBody: `{"alert_source_config_id":"some-id","status":"firing","title":"Gatus: endpoint-name","description":"An alert has been triggered due to having failed 3 time(s) in a row with the following description: description-1 and the following conditions:  🔴 [CONNECTED] == true  🔴 [STATUS] == 200  "}`,
+		},
+		{
+			Name:         "resolved",
+			Provider:     AlertProvider{DefaultConfig: Config{URL: restAPIUrl + "some-id", AuthToken: "some-token"}},
+			Alert:        alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved:     true,
+			ExpectedBody: `{"alert_source_config_id":"some-id","status":"resolved","title":"Gatus: endpoint-name","description":"An alert has been resolved after passing successfully 5 time(s) in a row with the following description: description-2 and the following conditions:  🟢 [CONNECTED] == true  🟢 [STATUS] == 200  "}`,
+		},
+		{
+			Name:         "resolved-with-metadata-source-url",
+			Provider:     AlertProvider{DefaultConfig: Config{URL: restAPIUrl + "some-id", AuthToken: "some-token", Metadata: map[string]interface{}{"service": "some-service", "team": "very-core"}, SourceURL: "some-source-url"}},
+			Alert:        alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved:     true,
+			ExpectedBody: `{"alert_source_config_id":"some-id","status":"resolved","title":"Gatus: endpoint-name","description":"An alert has been resolved after passing successfully 5 time(s) in a row with the following description: description-2 and the following conditions:  🟢 [CONNECTED] == true  🟢 [STATUS] == 200  ","source_url":"some-source-url","metadata":{"service":"some-service","team":"very-core"}}`,
+		},
+		{
+			Name:         "group-override",
+			Provider:     AlertProvider{DefaultConfig: Config{URL: restAPIUrl + "some-id", AuthToken: "some-token"}, Overrides: []Override{{Group: "g", Config: Config{URL: restAPIUrl + "different-id", AuthToken: "some-token"}}}},
+			Alert:        alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved:     false,
+			ExpectedBody: `{"alert_source_config_id":"different-id","status":"firing","title":"Gatus: endpoint-name","description":"An alert has been triggered due to having failed 3 time(s) in a row with the following description: description-1 and the following conditions:  🔴 [CONNECTED] == true  🔴 [STATUS] == 200  "}`,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			cfg, err := scenario.Provider.GetConfig("g", &scenario.Alert)
+			if err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+			body := scenario.Provider.buildRequestBody(
+				cfg,
+				&endpoint.Endpoint{Name: "endpoint-name"},
+				&scenario.Alert,
+				&endpoint.Result{
+					ConditionResults: []*endpoint.ConditionResult{
+						{Condition: "[CONNECTED] == true", Success: scenario.Resolved},
+						{Condition: "[STATUS] == 200", Success: scenario.Resolved},
+					},
+				},
+				scenario.Resolved,
+			)
+			if string(body) != scenario.ExpectedBody {
+				t.Errorf("expected:\n%s\ngot:\n%s", scenario.ExpectedBody, body)
+			}
+			out := make(map[string]interface{})
+			if err := json.Unmarshal(body, &out); err != nil {
+				t.Error("expected body to be valid JSON, got error:", err.Error())
+			}
+		})
+	}
+}
+
+func TestAlertProvider_GetDefaultAlert(t *testing.T) {
+	if (&AlertProvider{DefaultAlert: &alert.Alert{}}).GetDefaultAlert() == nil {
+		t.Error("expected default alert to be not nil")
+	}
+	if (&AlertProvider{DefaultAlert: nil}).GetDefaultAlert() != nil {
+		t.Error("expected default alert to be nil")
+	}
+}
+
+func TestAlertProvider_GetConfig(t *testing.T) {
+	scenarios := []struct {
+		Name           string
+		Provider       AlertProvider
+		InputGroup     string
+		InputAlert     alert.Alert
+		ExpectedOutput Config
+	}{
+		{
+			Name: "provider-no-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{URL: "some-id", AuthToken: "some-token"},
+				Overrides:     nil,
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{URL: "some-id", AuthToken: "some-token"},
+		},
+		{
+			Name: "provider-no-override-specify-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{URL: "some-id", AuthToken: "some-token"},
+				Overrides:     nil,
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{URL: "some-id", AuthToken: "some-token"},
+		},
+		{
+			Name: "provider-with-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{URL: "some-id", AuthToken: "some-token"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{URL: "diff-id"},
+					},
+				},
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{URL: "some-id", AuthToken: "some-token"},
+		},
+		{
+			Name: "provider-with-override-specify-group-should-override",
+			Provider: AlertProvider{
+				DefaultConfig: Config{URL: "some-id", AuthToken: "some-token"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{URL: "diff-id", AuthToken: "some-token"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{URL: "diff-id", AuthToken: "some-token"},
+		},
+		{
+			Name: "provider-with-group-override-and-alert-override--alert-override-should-take-precedence",
+			Provider: AlertProvider{
+				DefaultConfig: Config{URL: "some-id", AuthToken: "some-token"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{URL: "diff-id", AuthToken: "some-token"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{ProviderOverride: map[string]any{"url": "another-id"}},
+			ExpectedOutput: Config{URL: "another-id", AuthToken: "some-token"},
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			got, err := scenario.Provider.GetConfig(scenario.InputGroup, &scenario.InputAlert)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if got.URL != scenario.ExpectedOutput.URL {
+				t.Errorf("expected alert source config to be %s, got %s", scenario.ExpectedOutput.URL, got.URL)
+			}
+			if got.AuthToken != scenario.ExpectedOutput.AuthToken {
+				t.Errorf("expected alert auth token to be %s, got %s", scenario.ExpectedOutput.AuthToken, got.AuthToken)
+			}
+
+			// Test ValidateOverrides as well, since it really just calls GetConfig
+			if err = scenario.Provider.ValidateOverrides(scenario.InputGroup, &scenario.InputAlert); err != nil {
+				t.Errorf("unexpected error: %s", err)
+			}
+		})
+	}
+}
+
+func TestAlertProvider_ValidateWithOverride(t *testing.T) {
+	providerWithInvalidOverrideGroup := AlertProvider{
+		Overrides: []Override{
+			{
+				Config: Config{URL: "some-id", AuthToken: "some-token"},
+				Group:  "",
+			},
+		},
+	}
+	if err := providerWithInvalidOverrideGroup.Validate(); err == nil {
+		t.Error("provider Group shouldn't have been valid")
+	}
+	providerWithInvalidOverrideTo := AlertProvider{
+		Overrides: []Override{
+			{
+				Config: Config{URL: "", AuthToken: "some-token"},
+				Group:  "group",
+			},
+		},
+	}
+	if err := providerWithInvalidOverrideTo.Validate(); err == nil {
+		t.Error("provider integration key shouldn't have been valid")
+	}
+	providerWithValidOverride := AlertProvider{
+		DefaultConfig: Config{URL: "nice-id", AuthToken: "some-token"},
+		Overrides: []Override{
+			{
+				Config: Config{URL: "very-good-id", AuthToken: "some-token"},
+				Group:  "group",
+			},
+		},
+	}
+	if err := providerWithValidOverride.Validate(); err != nil {
+		t.Error("provider should've been valid")
+	}
+}
--- a/alerting/provider/provider.go
+++ b/alerting/provider/provider.go
@ -10,6 +10,7 @@ import (
 	"github.com/TwiN/gatus/v5/alerting/provider/github"
 	"github.com/TwiN/gatus/v5/alerting/provider/gitlab"
 	"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
+	"github.com/TwiN/gatus/v5/alerting/provider/incidentio"
 	"github.com/TwiN/gatus/v5/alerting/provider/jetbrainsspace"
 	"github.com/TwiN/gatus/v5/alerting/provider/matrix"
 	"github.com/TwiN/gatus/v5/alerting/provider/mattermost"
@ -93,6 +94,7 @@ var (
 	_ AlertProvider = (*telegram.AlertProvider)(nil)
 	_ AlertProvider = (*twilio.AlertProvider)(nil)
 	_ AlertProvider = (*zulip.AlertProvider)(nil)
+	_ AlertProvider = (*incidentio.AlertProvider)(nil)

 	// Validate config interface implementation on compile
 	_ Config[awsses.Config]         = (*awsses.Config)(nil)
@ -103,6 +105,7 @@ var (
 	_ Config[github.Config]         = (*github.Config)(nil)
 	_ Config[gitlab.Config]         = (*gitlab.Config)(nil)
 	_ Config[googlechat.Config]     = (*googlechat.Config)(nil)
+	_ Config[incidentio.Config]     = (*incidentio.Config)(nil)
 	_ Config[jetbrainsspace.Config] = (*jetbrainsspace.Config)(nil)
 	_ Config[matrix.Config]         = (*matrix.Config)(nil)
 	_ Config[mattermost.Config]     = (*mattermost.Config)(nil)
--- a/config/config.go
+++ b/config/config.go
@ -421,6 +421,7 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*endpoi
 		alert.TypeTelegram,
 		alert.TypeTwilio,
 		alert.TypeZulip,
+		alert.TypeIncidentIO,
 	}
 	var validProviders, invalidProviders []alert.Type
 	for _, alertType := range alertTypes {