diff --git a/README.md b/README.md index 66a08910..cb139c4f 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga - [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries) - [Monitoring an endpoint using STARTTLS](#monitoring-an-endpoint-using-starttls) - [Monitoring an endpoint using TLS](#monitoring-an-endpoint-using-tls) + - [Monitoring domain expiration](#monitoring-domain-expiration) - [disable-monitoring-lock](#disable-monitoring-lock) - [Reloading configuration on the fly](#reloading-configuration-on-the-fly) - [Endpoint groups](#endpoint-groups) @@ -222,18 +223,20 @@ Here are some examples of conditions you can use: | `[BODY].name == pat(john*)` | String at JSONPath `$.name` matches pattern `john*` | `{"name":"john.doe"}` | `{"name":"bob"}` | | `[BODY].id == any(1, 2)` | Value at JSONPath `$.id` is equal to `1` or `2` | 1, 2 | 3, 4, 5 | | `[CERTIFICATE_EXPIRATION] > 48h` | Certificate expiration is more than 48h away | 49h, 50h, 123h | 1h, 24h, ... | +| `[DOMAIN_EXPIRATION] > 720h` | The domain must expire in more than 720h | 4000h | 1h, 24h, ... | #### Placeholders | Placeholder | Description | Example of resolved value | |:---------------------------|:------------------------------------------------------------------------------------------|:---------------------------------------------| -| `[STATUS]` | Resolves into the HTTP status of the request | 404 | -| `[RESPONSE_TIME]` | Resolves into the response time the request took, in ms | 10 | -| `[IP]` | Resolves into the IP of the target host | 192.168.0.232 | +| `[STATUS]` | Resolves into the HTTP status of the request | `404` | +| `[RESPONSE_TIME]` | Resolves into the response time the request took, in ms | `10` | +| `[IP]` | Resolves into the IP of the target host | `192.168.0.232` | | `[BODY]` | Resolves into the response body. Supports JSONPath. | `{"name":"john.doe"}` | | `[CONNECTED]` | Resolves into whether a connection could be established | `true` | | `[CERTIFICATE_EXPIRATION]` | Resolves into the duration before certificate expiration (valid units are "s", "m", "h".) | `24h`, `48h`, 0 (if not protocol with certs) | -| `[DNS_RCODE]` | Resolves into the DNS status of the response | NOERROR | +| `[DOMAIN_EXPIRATION]` | Resolves into the duration before the domain expires (valid units are "s", "m", "h".) | `24h`, `48h`, `1234h56m78s` | +| `[DNS_RCODE]` | Resolves into the DNS status of the response | `NOERROR` | #### Functions @@ -1337,6 +1340,25 @@ endpoints: ``` +### Monitoring domain expiration +You can monitor the expiration of a domain with all endpoint types except for DNS by using the `[DOMAIN_EXPIRATION]` +placeholder: +```yaml +endpoints: + - name: check-domain-and-certificate-expiration + url: "https://example.org" + interval: 1h + conditions: + - "[DOMAIN_EXPIRATION] > 720h" + - "[CERTIFICATE_EXPIRATION] > 240h" +``` + +**NOTE**: The usage of the `[DOMAIN_EXPIRATION]` placeholder requires Gatus to send a request to the official IANA WHOIS service [through a library](https://github.com/TwiN/whois) +and in some cases, a secondary request to a TLD-specific WHOIS server (e.g. `whois.nic.sh`). +You are also responsible for sending requests at a reasonable rate, as the WHOIS service may throttle your IP address if you send too many requests. +The duration taken by the WHOIS request(s) is excluded from the request's response time. + + ### disable-monitoring-lock Setting `disable-monitoring-lock` to `true` means that multiple endpoints could be monitored at the same time. @@ -1408,7 +1430,7 @@ endpoints: conditions: - "[STATUS] == 200" - - name: random endpoint that isn't part of a group + - name: random endpoint that is not part of a group url: "https://example.org/" interval: 5m conditions: @@ -1434,6 +1456,7 @@ web: port: ${PORT} ``` + ### Badges #### Uptime ![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg) @@ -1499,7 +1522,7 @@ Where: - `{key}` has the pattern `_` in which both variables have ` `, `/`, `_`, `,` and `.` replaced by `-`. ##### How to change the color thresholds of the response time badge -To change the response time badges threshold, a corresponding configuration can be added to an endpoint. +To change the response time badges' threshold, a corresponding configuration can be added to an endpoint. The values in the array correspond to the levels [Awesome, Great, Good, Passable, Bad] All five values must be given in milliseconds (ms). @@ -1517,6 +1540,7 @@ endpoints: thresholds: [550, 850, 1350, 1650, 1750] ``` + ### API Gatus provides a simple read-only API that can be queried in order to programmatically determine endpoint status and history. diff --git a/core/condition.go b/core/condition.go index 44b55f54..e8879ecf 100644 --- a/core/condition.go +++ b/core/condition.go @@ -46,6 +46,9 @@ const ( // Values that could replace the placeholder: 4461677039 (~52 days) CertificateExpirationPlaceholder = "[CERTIFICATE_EXPIRATION]" + // DomainExpirationPlaceholder is a placeholder for the duration before the domain expires, in milliseconds. + DomainExpirationPlaceholder = "[DOMAIN_EXPIRATION]" + // LengthFunctionPrefix is the prefix for the length function // // Usage: len([BODY].articles) == 10, len([BODY].name) > 5 @@ -142,9 +145,21 @@ func (c Condition) hasBodyPlaceholder() bool { return strings.Contains(string(c), BodyPlaceholder) } +// hasDomainExpirationPlaceholder checks whether the condition has a DomainExpirationPlaceholder +// Used for determining whether a whois operation is necessary +func (c Condition) hasDomainExpirationPlaceholder() bool { + return strings.Contains(string(c), DomainExpirationPlaceholder) +} + +// hasIPPlaceholder checks whether the condition has an IPPlaceholder +// Used for determining whether an IP lookup is necessary +func (c Condition) hasIPPlaceholder() bool { + return strings.Contains(string(c), IPPlaceholder) +} + // isEqual compares two strings. // -// Supports the pattern and the any functions. +// Supports the "pat" and the "any" functions. // i.e. if one of the parameters starts with PatternFunctionPrefix and ends with FunctionSuffix, it will be treated like // a pattern. func isEqual(first, second string) bool { @@ -219,6 +234,8 @@ func sanitizeAndResolve(elements []string, result *Result) ([]string, []string) element = strconv.FormatBool(result.Connected) case CertificateExpirationPlaceholder: element = strconv.FormatInt(result.CertificateExpiration.Milliseconds(), 10) + case DomainExpirationPlaceholder: + element = strconv.FormatInt(result.DomainExpiration.Milliseconds(), 10) default: // if contains the BodyPlaceholder, then evaluate json path if strings.Contains(element, BodyPlaceholder) { diff --git a/core/endpoint.go b/core/endpoint.go index edfdf436..1d7c90e2 100644 --- a/core/endpoint.go +++ b/core/endpoint.go @@ -16,6 +16,7 @@ import ( "github.com/TwiN/gatus/v4/client" "github.com/TwiN/gatus/v4/core/ui" "github.com/TwiN/gatus/v4/util" + "github.com/TwiN/whois" ) type EndpointType string @@ -138,7 +139,7 @@ func (endpoint Endpoint) Type() EndpointType { } } -// ValidateAndSetDefaults validates the endpoint's configuration and sets the default value of fields that have one +// ValidateAndSetDefaults validates the endpoint's configuration and sets the default value of args that have one func (endpoint *Endpoint) ValidateAndSetDefaults() error { // Set default values if endpoint.ClientConfig == nil { @@ -220,7 +221,26 @@ func (endpoint Endpoint) Key() string { // EvaluateHealth sends a request to the endpoint's URL and evaluates the conditions of the endpoint. func (endpoint *Endpoint) EvaluateHealth() *Result { result := &Result{Success: true, Errors: []string{}} - endpoint.getIP(result) + // Parse or extract hostname from URL + if endpoint.DNS != nil { + result.Hostname = strings.TrimSuffix(endpoint.URL, ":53") + } else { + urlObject, err := url.Parse(endpoint.URL) + if err != nil { + result.AddError(err.Error()) + } else { + result.Hostname = urlObject.Hostname() + } + } + // Retrieve IP if necessary + if endpoint.needsToRetrieveIP() { + endpoint.getIP(result) + } + // Retrieve domain expiration if necessary + if endpoint.needsToRetrieveDomainExpiration() && len(result.Hostname) > 0 { + endpoint.getDomainExpiration(result) + } + // if len(result.Errors) == 0 { endpoint.call(result) } else { @@ -251,16 +271,6 @@ func (endpoint *Endpoint) EvaluateHealth() *Result { } func (endpoint *Endpoint) getIP(result *Result) { - if endpoint.DNS != nil { - result.Hostname = strings.TrimSuffix(endpoint.URL, ":53") - } else { - urlObject, err := url.Parse(endpoint.URL) - if err != nil { - result.AddError(err.Error()) - return - } - result.Hostname = urlObject.Hostname() - } ips, err := net.LookupIP(result.Hostname) if err != nil { result.AddError(err.Error()) @@ -269,6 +279,15 @@ func (endpoint *Endpoint) getIP(result *Result) { result.IP = ips[0].String() } +func (endpoint *Endpoint) getDomainExpiration(result *Result) { + whoisClient := whois.NewClient() + if whoisResponse, err := whoisClient.QueryAndParse(result.Hostname); err != nil { + result.AddError("error querying and parsing hostname using whois client: " + err.Error()) + } else { + result.DomainExpiration = time.Until(whoisResponse.ExpirationDate) + } +} + func (endpoint *Endpoint) call(result *Result) { var request *http.Request var response *http.Response @@ -317,7 +336,7 @@ func (endpoint *Endpoint) call(result *Result) { if endpoint.needsToReadBody() { result.body, err = io.ReadAll(response.Body) if err != nil { - result.AddError(err.Error()) + result.AddError("error reading response body:" + err.Error()) } } } @@ -344,7 +363,7 @@ func (endpoint *Endpoint) buildHTTPRequest() *http.Request { return request } -// needsToReadBody checks if there's any conditions that requires the response body to be read +// needsToReadBody checks if there's any condition that requires the response body to be read func (endpoint *Endpoint) needsToReadBody() bool { for _, condition := range endpoint.Conditions { if condition.hasBodyPlaceholder() { @@ -353,3 +372,23 @@ func (endpoint *Endpoint) needsToReadBody() bool { } return false } + +// needsToRetrieveDomainExpiration checks if there's any condition that requires a whois query to be performed +func (endpoint *Endpoint) needsToRetrieveDomainExpiration() bool { + for _, condition := range endpoint.Conditions { + if condition.hasDomainExpirationPlaceholder() { + return true + } + } + return false +} + +// needsToRetrieveIP checks if there's any condition that requires an IP lookup +func (endpoint *Endpoint) needsToRetrieveIP() bool { + for _, condition := range endpoint.Conditions { + if condition.hasIPPlaceholder() { + return true + } + } + return false +} diff --git a/core/endpoint_test.go b/core/endpoint_test.go index c32d18b8..b2be7f2a 100644 --- a/core/endpoint_test.go +++ b/core/endpoint_test.go @@ -1,7 +1,11 @@ package core import ( + "bytes" + "crypto/tls" + "crypto/x509" "io" + "net/http" "strings" "testing" "time" @@ -9,8 +13,233 @@ import ( "github.com/TwiN/gatus/v4/alerting/alert" "github.com/TwiN/gatus/v4/client" "github.com/TwiN/gatus/v4/core/ui" + "github.com/TwiN/gatus/v4/test" ) +func TestEndpoint(t *testing.T) { + defer client.InjectHTTPClient(nil) + scenarios := []struct { + Name string + Endpoint Endpoint + ExpectedResult *Result + MockRoundTripper test.MockRoundTripper + }{ + { + Name: "success", + Endpoint: Endpoint{ + Name: "website-health", + URL: "https://twin.sh/health", + Conditions: []Condition{"[STATUS] == 200", "[BODY].status == UP", "[CERTIFICATE_EXPIRATION] > 24h"}, + }, + ExpectedResult: &Result{ + Success: true, + Connected: true, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + {Condition: "[STATUS] == 200", Success: true}, + {Condition: "[BODY].status == UP", Success: true}, + {Condition: "[CERTIFICATE_EXPIRATION] > 24h", Success: true}, + }, + DomainExpiration: 0, // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + }, + MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`{"status": "UP"}`)), + TLS: &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{NotAfter: time.Now().Add(9999 * time.Hour)}}}, + } + }), + }, + { + Name: "failed-body-condition", + Endpoint: Endpoint{ + Name: "website-health", + URL: "https://twin.sh/health", + Conditions: []Condition{"[STATUS] == 200", "[BODY].status == UP"}, + }, + ExpectedResult: &Result{ + Success: false, + Connected: true, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + {Condition: "[STATUS] == 200", Success: true}, + {Condition: "[BODY].status (DOWN) == UP", Success: false}, + }, + DomainExpiration: 0, // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + }, + MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response { + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString(`{"status": "DOWN"}`))} + }), + }, + { + Name: "failed-status-condition", + Endpoint: Endpoint{ + Name: "website-health", + URL: "https://twin.sh/health", + Conditions: []Condition{"[STATUS] == 200"}, + }, + ExpectedResult: &Result{ + Success: false, + Connected: true, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + {Condition: "[STATUS] (502) == 200", Success: false}, + }, + DomainExpiration: 0, // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + }, + MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response { + return &http.Response{StatusCode: http.StatusBadGateway, Body: http.NoBody} + }), + }, + { + Name: "condition-with-failed-certificate-expiration", + Endpoint: Endpoint{ + Name: "website-health", + URL: "https://twin.sh/health", + Conditions: []Condition{"[CERTIFICATE_EXPIRATION] > 100h"}, + UIConfig: &ui.Config{DontResolveFailedConditions: true}, + }, + ExpectedResult: &Result{ + Success: false, + Connected: true, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + // Because UIConfig.DontResolveFailedConditions is true, the values in the condition should not be resolved + {Condition: "[CERTIFICATE_EXPIRATION] > 100h", Success: false}, + }, + DomainExpiration: 0, // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + }, + MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: http.NoBody, + TLS: &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{NotAfter: time.Now().Add(5 * time.Hour)}}}, + } + }), + }, + { + Name: "domain-expiration", + Endpoint: Endpoint{ + Name: "website-health", + URL: "https://twin.sh/health", + Conditions: []Condition{"[DOMAIN_EXPIRATION] > 100h"}, + }, + ExpectedResult: &Result{ + Success: true, + Connected: true, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + {Condition: "[DOMAIN_EXPIRATION] > 100h", Success: true}, + }, + DomainExpiration: 999999 * time.Hour, // Note that this test only checks if it's non-zero. + }, + MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response { + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody} + }), + }, + { + Name: "endpoint-that-will-time-out-and-hidden-hostname", + Endpoint: Endpoint{ + Name: "endpoint-that-will-time-out", + URL: "https://twin.sh/health", + Conditions: []Condition{"[CONNECTED] == true"}, + UIConfig: &ui.Config{HideHostname: true}, + ClientConfig: &client.Config{Timeout: time.Millisecond}, + }, + ExpectedResult: &Result{ + Success: false, + Connected: false, + Hostname: "", // Because Endpoint.UIConfig.HideHostname is true, this should be empty. + ConditionResults: []*ConditionResult{ + {Condition: "[CONNECTED] (false) == true", Success: false}, + }, + // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + DomainExpiration: 0, + // Because Endpoint.UIConfig.HideHostname is true, the hostname should be replaced by . + Errors: []string{`Get "https:///health": context deadline exceeded (Client.Timeout exceeded while awaiting headers)`}, + }, + MockRoundTripper: nil, + }, + { + Name: "endpoint-that-will-time-out-and-hidden-url", + Endpoint: Endpoint{ + Name: "endpoint-that-will-time-out", + URL: "https://twin.sh/health", + Conditions: []Condition{"[CONNECTED] == true"}, + UIConfig: &ui.Config{HideURL: true}, + ClientConfig: &client.Config{Timeout: time.Millisecond}, + }, + ExpectedResult: &Result{ + Success: false, + Connected: false, + Hostname: "twin.sh", + ConditionResults: []*ConditionResult{ + {Condition: "[CONNECTED] (false) == true", Success: false}, + }, + // Because there's no [DOMAIN_EXPIRATION] condition, this is not resolved, so it should be 0. + DomainExpiration: 0, + // Because Endpoint.UIConfig.HideURL is true, the URL should be replaced by . + Errors: []string{`Get "": context deadline exceeded (Client.Timeout exceeded while awaiting headers)`}, + }, + MockRoundTripper: nil, + }, + } + for _, scenario := range scenarios { + t.Run(scenario.Name, func(t *testing.T) { + if scenario.MockRoundTripper != nil { + mockClient := &http.Client{Transport: scenario.MockRoundTripper} + if scenario.Endpoint.ClientConfig != nil && scenario.Endpoint.ClientConfig.Timeout > 0 { + mockClient.Timeout = scenario.Endpoint.ClientConfig.Timeout + } + client.InjectHTTPClient(mockClient) + } else { + client.InjectHTTPClient(nil) + } + scenario.Endpoint.ValidateAndSetDefaults() + result := scenario.Endpoint.EvaluateHealth() + if result.Success != scenario.ExpectedResult.Success { + t.Errorf("Expected success to be %v, got %v", scenario.ExpectedResult.Success, result.Success) + } + if result.Connected != scenario.ExpectedResult.Connected { + t.Errorf("Expected connected to be %v, got %v", scenario.ExpectedResult.Connected, result.Connected) + } + if result.Hostname != scenario.ExpectedResult.Hostname { + t.Errorf("Expected hostname to be %v, got %v", scenario.ExpectedResult.Hostname, result.Hostname) + } + if len(result.ConditionResults) != len(scenario.ExpectedResult.ConditionResults) { + t.Errorf("Expected %v condition results, got %v", len(scenario.ExpectedResult.ConditionResults), len(result.ConditionResults)) + } else { + for i, conditionResult := range result.ConditionResults { + if conditionResult.Condition != scenario.ExpectedResult.ConditionResults[i].Condition { + t.Errorf("Expected condition to be %v, got %v", scenario.ExpectedResult.ConditionResults[i].Condition, conditionResult.Condition) + } + if conditionResult.Success != scenario.ExpectedResult.ConditionResults[i].Success { + t.Errorf("Expected success of condition '%s' to be %v, got %v", conditionResult.Condition, scenario.ExpectedResult.ConditionResults[i].Success, conditionResult.Success) + } + } + } + if len(result.Errors) != len(scenario.ExpectedResult.Errors) { + t.Errorf("Expected %v errors, got %v", len(scenario.ExpectedResult.Errors), len(result.Errors)) + } else { + for i, err := range result.Errors { + if err != scenario.ExpectedResult.Errors[i] { + t.Errorf("Expected error to be %v, got %v", scenario.ExpectedResult.Errors[i], err) + } + } + } + if result.DomainExpiration != scenario.ExpectedResult.DomainExpiration { + // Note that DomainExpiration is only resolved if there's a condition with the DomainExpirationPlaceholder in it. + // In other words, if there's no condition with [DOMAIN_EXPIRATION] in it, the DomainExpiration field will be 0. + // Because this is a live call, mocking it would be too much of a pain, so we're just going to check if + // the actual value is non-zero when the expected result is non-zero. + if scenario.ExpectedResult.DomainExpiration.Hours() > 0 && !(result.DomainExpiration.Hours() > 0) { + t.Errorf("Expected domain expiration to be non-zero, got %v", result.DomainExpiration) + } + } + }) + } +} + func TestEndpoint_IsEnabled(t *testing.T) { if !(Endpoint{Enabled: nil}).IsEnabled() { t.Error("endpoint.IsEnabled() should've returned true, because Enabled was set to nil") @@ -349,26 +578,6 @@ func TestIntegrationEvaluateHealth(t *testing.T) { } } -func TestIntegrationEvaluateHealthWithFailure(t *testing.T) { - condition := Condition("[STATUS] == 500") - endpoint := Endpoint{ - Name: "website-health", - URL: "https://twin.sh/health", - Conditions: []Condition{condition}, - } - endpoint.ValidateAndSetDefaults() - result := endpoint.EvaluateHealth() - if result.ConditionResults[0].Success { - t.Errorf("Condition '%s' should have been a failure", condition) - } - if !result.Connected { - t.Error("Because the connection has been established, result.Connected should've been true") - } - if result.Success { - t.Error("Because one of the conditions failed, result.Success should have been false") - } -} - func TestIntegrationEvaluateHealthWithInvalidCondition(t *testing.T) { condition := Condition("[STATUS] invalid 200") endpoint := Endpoint{ @@ -389,32 +598,6 @@ func TestIntegrationEvaluateHealthWithInvalidCondition(t *testing.T) { } } -func TestIntegrationEvaluateHealthWithError(t *testing.T) { - condition := Condition("[STATUS] == 200") - endpoint := Endpoint{ - Name: "invalid-host", - URL: "http://invalid/health", - Conditions: []Condition{condition}, - UIConfig: &ui.Config{ - HideHostname: true, - }, - } - endpoint.ValidateAndSetDefaults() - result := endpoint.EvaluateHealth() - if result.Success { - t.Error("Because one of the conditions was invalid, result.Success should have been false") - } - if len(result.Errors) == 0 { - t.Error("There should've been an error") - } - if !strings.Contains(result.Errors[0], "") { - t.Error("result.Errors[0] should've had the hostname redacted because ui.hide-hostname is set to true") - } - if result.Hostname != "" { - t.Error("result.Hostname should've been empty because ui.hide-hostname is set to true") - } -} - func TestIntegrationEvaluateHealthWithErrorAndHideURL(t *testing.T) { endpoint := Endpoint{ Name: "invalid-url", @@ -455,7 +638,7 @@ func TestIntegrationEvaluateHealthForDNS(t *testing.T) { endpoint.ValidateAndSetDefaults() result := endpoint.EvaluateHealth() if !result.ConditionResults[0].Success { - t.Errorf("Conditions '%s' and %s should have been a success", conditionSuccess, conditionBody) + t.Errorf("Conditions '%s' and '%s' should have been a success", conditionSuccess, conditionBody) } if !result.Connected { t.Error("Because the connection has been established, result.Connected should've been true") @@ -466,16 +649,15 @@ func TestIntegrationEvaluateHealthForDNS(t *testing.T) { } func TestIntegrationEvaluateHealthForICMP(t *testing.T) { - conditionSuccess := Condition("[CONNECTED] == true") endpoint := Endpoint{ Name: "icmp-test", URL: "icmp://127.0.0.1", - Conditions: []Condition{conditionSuccess}, + Conditions: []Condition{"[CONNECTED] == true"}, } endpoint.ValidateAndSetDefaults() result := endpoint.EvaluateHealth() if !result.ConditionResults[0].Success { - t.Errorf("Conditions '%s' should have been a success", conditionSuccess) + t.Errorf("Conditions '%s' should have been a success", endpoint.Conditions[0]) } if !result.Connected { t.Error("Because the connection has been established, result.Connected should've been true") @@ -486,11 +668,10 @@ func TestIntegrationEvaluateHealthForICMP(t *testing.T) { } func TestEndpoint_getIP(t *testing.T) { - conditionSuccess := Condition("[CONNECTED] == true") endpoint := Endpoint{ Name: "invalid-url-test", URL: "", - Conditions: []Condition{conditionSuccess}, + Conditions: []Condition{"[CONNECTED] == true"}, } result := &Result{} endpoint.getIP(result) @@ -499,7 +680,7 @@ func TestEndpoint_getIP(t *testing.T) { } } -func TestEndpoint_NeedsToReadBody(t *testing.T) { +func TestEndpoint_needsToReadBody(t *testing.T) { statusCondition := Condition("[STATUS] == 200") bodyCondition := Condition("[BODY].status == UP") bodyConditionWithLength := Condition("len([BODY].tags) > 0") @@ -522,3 +703,21 @@ func TestEndpoint_NeedsToReadBody(t *testing.T) { t.Error("expected true, got false") } } + +func TestEndpoint_needsToRetrieveDomainExpiration(t *testing.T) { + if (&Endpoint{Conditions: []Condition{"[STATUS] == 200"}}).needsToRetrieveDomainExpiration() { + t.Error("expected false, got true") + } + if !(&Endpoint{Conditions: []Condition{"[STATUS] == 200", "[DOMAIN_EXPIRATION] < 720h"}}).needsToRetrieveDomainExpiration() { + t.Error("expected true, got false") + } +} + +func TestEndpoint_needsToRetrieveIP(t *testing.T) { + if (&Endpoint{Conditions: []Condition{"[STATUS] == 200"}}).needsToRetrieveIP() { + t.Error("expected false, got true") + } + if !(&Endpoint{Conditions: []Condition{"[STATUS] == 200", "[IP] == 127.0.0.1"}}).needsToRetrieveIP() { + t.Error("expected true, got false") + } +} diff --git a/core/result.go b/core/result.go index 12142fca..680653d7 100644 --- a/core/result.go +++ b/core/result.go @@ -41,6 +41,9 @@ type Result struct { // CertificateExpiration is the duration before the certificate expires CertificateExpiration time.Duration `json:"-"` + // DomainExpiration is the duration before the domain expires + DomainExpiration time.Duration `json:"-"` + // body is the response body // // Note that this variable is only used during the evaluation of an Endpoint's health. diff --git a/go.mod b/go.mod index 93cfbf8c..6af0b9af 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/TwiN/g8 v1.3.0 github.com/TwiN/gocache/v2 v2.1.0 github.com/TwiN/health v1.4.0 + github.com/TwiN/whois v1.0.0 github.com/coreos/go-oidc/v3 v3.1.0 github.com/go-ping/ping v0.0.0-20210911151512-381826476871 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 2c7efcc6..f10dd9ff 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/TwiN/gocache/v2 v2.1.0 h1:AJnSX7Sgz22fsO7rdXYQzMQ4zWpMjBKqk70ADeqtLDU github.com/TwiN/gocache/v2 v2.1.0/go.mod h1:AKHAFZSwLLmoLm1a2siDOWmZ2RjIKqentRGfOFWkllY= github.com/TwiN/health v1.4.0 h1:Ts7lb4ihYDpVEbFSGAhSEZTSwuDOADnwJLFngFl4xzw= github.com/TwiN/health v1.4.0/go.mod h1:CSUh+ryfD2POS2vKtc/yO4IxgR58lKvQ0/8qnoPqPqs= +github.com/TwiN/whois v1.0.0 h1:I+aQzXLPmhWovkFUzlPV2DdfLZUWDLrkMDlM6QwCv+Q= +github.com/TwiN/whois v1.0.0/go.mod h1:9WbCzYlR+r5eq9vbgJVh7A4H2uR2ct4wKEB0/QITJ/c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/storage/store/sql/specific_postgres.go b/storage/store/sql/specific_postgres.go index ff651add..49ebfeca 100644 --- a/storage/store/sql/specific_postgres.go +++ b/storage/store/sql/specific_postgres.go @@ -34,6 +34,7 @@ func (s *Store) createPostgresSchema() error { status BIGINT, dns_rcode TEXT, certificate_expiration BIGINT, + domain_expiration BIGINT, hostname TEXT, ip TEXT, duration BIGINT, @@ -65,5 +66,7 @@ func (s *Store) createPostgresSchema() error { UNIQUE(endpoint_id, hour_unix_timestamp) ) `) + // Silent table modifications + _, _ = s.db.Exec(`ALTER TABLE endpoint_results ADD IF NOT EXISTS domain_expiration BIGINT`) return err } diff --git a/storage/store/sql/specific_sqlite.go b/storage/store/sql/specific_sqlite.go index 5abcdf1c..45b06562 100644 --- a/storage/store/sql/specific_sqlite.go +++ b/storage/store/sql/specific_sqlite.go @@ -34,6 +34,7 @@ func (s *Store) createSQLiteSchema() error { status INTEGER, dns_rcode TEXT, certificate_expiration INTEGER, + domain_expiration INTEGER, hostname TEXT, ip TEXT, duration INTEGER, @@ -65,5 +66,7 @@ func (s *Store) createSQLiteSchema() error { UNIQUE(endpoint_id, hour_unix_timestamp) ) `) + // Silent table modifications + _, _ = s.db.Exec(`ALTER TABLE endpoint_results ADD domain_expiration INTEGER`) return err } diff --git a/storage/store/sql/sql.go b/storage/store/sql/sql.go index 8478f674..87a91cc8 100644 --- a/storage/store/sql/sql.go +++ b/storage/store/sql/sql.go @@ -439,8 +439,8 @@ func (s *Store) insertEndpointResult(tx *sql.Tx, endpointID int64, result *core. var endpointResultID int64 err := tx.QueryRow( ` - INSERT INTO endpoint_results (endpoint_id, success, errors, connected, status, dns_rcode, certificate_expiration, hostname, ip, duration, timestamp) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + INSERT INTO endpoint_results (endpoint_id, success, errors, connected, status, dns_rcode, certificate_expiration, domain_expiration, hostname, ip, duration, timestamp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING endpoint_result_id `, endpointID, @@ -450,6 +450,7 @@ func (s *Store) insertEndpointResult(tx *sql.Tx, endpointID int64, result *core. result.HTTPStatus, result.DNSRCode, result.CertificateExpiration, + result.DomainExpiration, result.Hostname, result.IP, result.Duration, @@ -590,7 +591,7 @@ func (s *Store) getEndpointEventsByEndpointID(tx *sql.Tx, endpointID int64, page func (s *Store) getEndpointResultsByEndpointID(tx *sql.Tx, endpointID int64, page, pageSize int) (results []*core.Result, err error) { rows, err := tx.Query( ` - SELECT endpoint_result_id, success, errors, connected, status, dns_rcode, certificate_expiration, hostname, ip, duration, timestamp + SELECT endpoint_result_id, success, errors, connected, status, dns_rcode, certificate_expiration, domain_expiration, hostname, ip, duration, timestamp FROM endpoint_results WHERE endpoint_id = $1 ORDER BY endpoint_result_id DESC -- Normally, we'd sort by timestamp, but sorting by endpoint_result_id is faster @@ -608,7 +609,7 @@ func (s *Store) getEndpointResultsByEndpointID(tx *sql.Tx, endpointID int64, pag result := &core.Result{} var id int64 var joinedErrors string - _ = rows.Scan(&id, &result.Success, &joinedErrors, &result.Connected, &result.HTTPStatus, &result.DNSRCode, &result.CertificateExpiration, &result.Hostname, &result.IP, &result.Duration, &result.Timestamp) + _ = rows.Scan(&id, &result.Success, &joinedErrors, &result.Connected, &result.HTTPStatus, &result.DNSRCode, &result.CertificateExpiration, &result.DomainExpiration, &result.Hostname, &result.IP, &result.Duration, &result.Timestamp) if len(joinedErrors) != 0 { result.Errors = strings.Split(joinedErrors, arraySeparator) } diff --git a/vendor/github.com/TwiN/whois/.gitignore b/vendor/github.com/TwiN/whois/.gitignore new file mode 100644 index 00000000..a676215f --- /dev/null +++ b/vendor/github.com/TwiN/whois/.gitignore @@ -0,0 +1,2 @@ +.idea +bin diff --git a/vendor/github.com/TwiN/whois/LICENSE b/vendor/github.com/TwiN/whois/LICENSE new file mode 100644 index 00000000..ea31ca11 --- /dev/null +++ b/vendor/github.com/TwiN/whois/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 TwiN + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/TwiN/whois/Makefile b/vendor/github.com/TwiN/whois/Makefile new file mode 100644 index 00000000..9572706d --- /dev/null +++ b/vendor/github.com/TwiN/whois/Makefile @@ -0,0 +1,4 @@ +.PHONY: build-binaries + +build-binaries: + ./scripts/build.sh diff --git a/vendor/github.com/TwiN/whois/README.md b/vendor/github.com/TwiN/whois/README.md new file mode 100644 index 00000000..fe3c61f1 --- /dev/null +++ b/vendor/github.com/TwiN/whois/README.md @@ -0,0 +1,65 @@ +# whois +![test](https://github.com/TwiN/whois/workflows/test/badge.svg?branch=master) + +Lightweight library for retrieving WHOIS information on a domain. + +It automatically retrieves the appropriate WHOIS server based on the domain's TLD by first querying IANA. + + +## Usage +### As an executable +To install it: +```console +go install github.com/TwiN/whois/cmd/whois@latest +``` +To run it: +```console +whois example.com +``` + +### As a library +```console +go get github.com/TwiN/whois +``` + +#### Query +If all you want is the text a WHOIS server would return you, you can use the `Query` method of the `whois.Client` type: +```go +package main + +import "github.com/TwiN/whois" + +func main() { + client := whois.NewClient() + output, err := client.Query("example.com") + if err != nil { + panic(err) + } + println(output) +} +``` + +#### QueryAndParse +If you want specific pieces of information, you can use the `QueryAndParse` method of the `whois.Client` type: +```go +package main + +import "github.com/TwiN/whois" + +func main() { + client := whois.NewClient() + response, err := client.QueryAndParse("example.com") + if err != nil { + panic(err) + } + println(response.ExpirationDate.String()) +} +``` +Note that because there is no standardized format for WHOIS responses, this parsing may not be successful for every single TLD. + +Currently, the only fields parsed are: +- `ExpirationDate`: The time.Time at which the domain will expire +- `DomainStatuses`: The statuses that the domain currently has (e.g. `clientTransferProhibited`) +- `NameServers`: The nameservers currently tied to the domain + +If you'd like one or more other fields to be parsed, please don't be shy and create an issue or a pull request. diff --git a/vendor/github.com/TwiN/whois/whois.go b/vendor/github.com/TwiN/whois/whois.go new file mode 100644 index 00000000..2f7b8ee9 --- /dev/null +++ b/vendor/github.com/TwiN/whois/whois.go @@ -0,0 +1,94 @@ +package whois + +import ( + "io" + "net" + "strings" + "time" +) + +const ( + ianaWHOISServerAddress = "whois.iana.org:43" +) + +type Client struct { + whoisServerAddress string +} + +func NewClient() *Client { + return &Client{ + whoisServerAddress: ianaWHOISServerAddress, + } +} + +func (c Client) Query(domain string) (string, error) { + parts := strings.Split(domain, ".") + output, err := c.query(c.whoisServerAddress, parts[len(parts)-1]) + if err != nil { + return "", err + } + if strings.Contains(output, "whois:") { + startIndex := strings.Index(output, "whois:") + 6 + endIndex := strings.Index(output[startIndex:], "\n") + startIndex + whois := strings.TrimSpace(output[startIndex:endIndex]) + if referOutput, err := c.query(whois+":43", domain); err == nil { + return referOutput, nil + } + return "", err + } + return output, nil +} + +func (c Client) query(whoisServerAddress, domain string) (string, error) { + connection, err := net.DialTimeout("tcp", whoisServerAddress, 10*time.Second) + if err != nil { + return "", err + } + defer connection.Close() + connection.SetDeadline(time.Now().Add(5 * time.Second)) + _, err = connection.Write([]byte(domain + "\r\n")) + if err != nil { + return "", err + } + output, err := io.ReadAll(connection) + if err != nil { + return "", err + } + return string(output), nil +} + +type Response struct { + ExpirationDate time.Time + DomainStatuses []string + NameServers []string +} + +// QueryAndParse tries to parse the response from the WHOIS server +// There is no standardized format for WHOIS responses, so this is an attempt at best. +// +// Being the selfish person that I am, I also only parse the fields that I need. +// If you need more fields, please open an issue or pull request. +func (c Client) QueryAndParse(domain string) (*Response, error) { + text, err := c.Query(domain) + if err != nil { + return nil, err + } + response := Response{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + valueStartIndex := strings.Index(line, ":") + if valueStartIndex == -1 { + continue + } + key := strings.ToLower(strings.TrimSpace(line[:valueStartIndex])) + value := strings.TrimSpace(line[valueStartIndex+1:]) + if response.ExpirationDate.Unix() != 0 && strings.Contains(key, "expir") && strings.Contains(key, "date") { + response.ExpirationDate, _ = time.Parse(time.RFC3339, strings.ToUpper(value)) + } else if strings.Contains(key, "domain status") { + response.DomainStatuses = append(response.DomainStatuses, value) + } else if strings.Contains(key, "name server") { + response.NameServers = append(response.NameServers, value) + } + } + return &response, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 78744aea..ad582040 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,6 +7,9 @@ github.com/TwiN/gocache/v2 # github.com/TwiN/health v1.4.0 ## explicit; go 1.18 github.com/TwiN/health +# github.com/TwiN/whois v1.0.0 +## explicit; go 1.19 +github.com/TwiN/whois # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile