From cf9c00a2adec1de910f4f607a4c3c7873d289fc7 Mon Sep 17 00:00:00 2001 From: wei Date: Tue, 17 May 2022 09:10:45 +0800 Subject: [PATCH] feat(metrics): Add more metrics (#278) * add gatus_results_success and gatus_results_duration_seconds * add metrics namespace * add result http metrics * add more metrics * update * extract endpoint type method * initializedMetrics * remove too many metrics * update naming * chore(metrics): Refactor code and merge results_dns_return_code_total, results_http_status_code_total into results_code_total * docs(metrics): Update results_certificate_expiration_seconds description * add TestEndpoint_Type * remove name in table test Co-authored-by: TwiN --- core/endpoint.go | 47 +++++++++++++++++++++++-------- core/endpoint_test.go | 56 +++++++++++++++++++++++++++++++++++++ metric/metric.go | 64 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 146 insertions(+), 21 deletions(-) diff --git a/core/endpoint.go b/core/endpoint.go index acaed8ec..4006e7ea 100644 --- a/core/endpoint.go +++ b/core/endpoint.go @@ -18,6 +18,8 @@ import ( "github.com/TwiN/gatus/v3/util" ) +type EndpointType string + const ( // HostHeader is the name of the header used to specify the host HostHeader = "Host" @@ -30,6 +32,14 @@ const ( // GatusUserAgent is the default user agent that Gatus uses to send requests. GatusUserAgent = "Gatus/1.0" + + // EndpointType enum for the endpoint type. + EndpointTypeDNS EndpointType = "DNS" + EndpointTypeTCP EndpointType = "TCP" + EndpointTypeICMP EndpointType = "ICMP" + EndpointTypeSTARTTLS EndpointType = "STARTTLS" + EndpointTypeTLS EndpointType = "TLS" + EndpointTypeHTTP EndpointType = "HTTP" ) var ( @@ -105,6 +115,24 @@ func (endpoint Endpoint) IsEnabled() bool { return *endpoint.Enabled } +// Type returns the endpoint type +func (endpoint Endpoint) Type() EndpointType { + switch { + case endpoint.DNS != nil: + return EndpointTypeDNS + case strings.HasPrefix(endpoint.URL, "tcp://"): + return EndpointTypeTCP + case strings.HasPrefix(endpoint.URL, "icmp://"): + return EndpointTypeICMP + case strings.HasPrefix(endpoint.URL, "starttls://"): + return EndpointTypeSTARTTLS + case strings.HasPrefix(endpoint.URL, "tls://"): + return EndpointTypeTLS + default: + return EndpointTypeHTTP + } +} + // ValidateAndSetDefaults validates the endpoint's configuration and sets the default value of fields that have one func (endpoint *Endpoint) ValidateAndSetDefaults() error { // Set default values @@ -229,21 +257,16 @@ func (endpoint *Endpoint) call(result *Result) { var response *http.Response var err error var certificate *x509.Certificate - isTypeDNS := endpoint.DNS != nil - isTypeTCP := strings.HasPrefix(endpoint.URL, "tcp://") - isTypeICMP := strings.HasPrefix(endpoint.URL, "icmp://") - isTypeSTARTTLS := strings.HasPrefix(endpoint.URL, "starttls://") - isTypeTLS := strings.HasPrefix(endpoint.URL, "tls://") - isTypeHTTP := !isTypeDNS && !isTypeTCP && !isTypeICMP && !isTypeSTARTTLS && !isTypeTLS - if isTypeHTTP { + endpointType := endpoint.Type() + if endpointType == EndpointTypeHTTP { request = endpoint.buildHTTPRequest() } startTime := time.Now() - if isTypeDNS { + if endpointType == EndpointTypeDNS { endpoint.DNS.query(endpoint.URL, result) result.Duration = time.Since(startTime) - } else if isTypeSTARTTLS || isTypeTLS { - if isTypeSTARTTLS { + } else if endpointType == EndpointTypeSTARTTLS || endpointType == EndpointTypeTLS { + if endpointType == EndpointTypeSTARTTLS { result.Connected, certificate, err = client.CanPerformStartTLS(strings.TrimPrefix(endpoint.URL, "starttls://"), endpoint.ClientConfig) } else { result.Connected, certificate, err = client.CanPerformTLS(strings.TrimPrefix(endpoint.URL, "tls://"), endpoint.ClientConfig) @@ -254,10 +277,10 @@ func (endpoint *Endpoint) call(result *Result) { } result.Duration = time.Since(startTime) result.CertificateExpiration = time.Until(certificate.NotAfter) - } else if isTypeTCP { + } else if endpointType == EndpointTypeTCP { result.Connected = client.CanCreateTCPConnection(strings.TrimPrefix(endpoint.URL, "tcp://"), endpoint.ClientConfig) result.Duration = time.Since(startTime) - } else if isTypeICMP { + } else if endpointType == EndpointTypeICMP { result.Connected, result.Duration = client.Ping(strings.TrimPrefix(endpoint.URL, "icmp://"), endpoint.ClientConfig) } else { response, err = client.GetHTTPClient(endpoint.ClientConfig).Do(request) diff --git a/core/endpoint_test.go b/core/endpoint_test.go index ddf5c552..03b00b1f 100644 --- a/core/endpoint_test.go +++ b/core/endpoint_test.go @@ -23,6 +23,62 @@ func TestEndpoint_IsEnabled(t *testing.T) { } } +func TestEndpoint_Type(t *testing.T) { + type fields struct { + URL string + DNS *DNS + } + tests := []struct { + fields fields + want EndpointType + }{{ + fields: fields{ + URL: "8.8.8.8", + DNS: &DNS{ + QueryType: "A", + QueryName: "example.com", + }, + }, + want: EndpointTypeDNS, + }, { + fields: fields{ + URL: "tcp://127.0.0.1:6379", + }, + want: EndpointTypeTCP, + }, { + fields: fields{ + URL: "icmp://example.com", + }, + want: EndpointTypeICMP, + }, { + fields: fields{ + URL: "starttls://smtp.gmail.com:587", + }, + want: EndpointTypeSTARTTLS, + }, { + fields: fields{ + URL: "tls://example.com:443", + }, + want: EndpointTypeTLS, + }, { + fields: fields{ + URL: "https://twin.sh/health", + }, + want: EndpointTypeHTTP, + }} + for _, tt := range tests { + t.Run(string(tt.want), func(t *testing.T) { + endpoint := Endpoint{ + URL: tt.fields.URL, + DNS: tt.fields.DNS, + } + if got := endpoint.Type(); got != tt.want { + t.Errorf("Endpoint.Type() = %v, want %v", got, tt.want) + } + }) + } +} + func TestEndpoint_ValidateAndSetDefaults(t *testing.T) { condition := Condition("[STATUS] == 200") endpoint := Endpoint{ diff --git a/metric/metric.go b/metric/metric.go index a3e8dfcd..89edbe1a 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -8,20 +8,66 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) +const namespace = "gatus" // The prefix of the metrics + var ( - // This will be initialized once PublishMetricsForEndpoint. - // The reason why we're doing this is that if metrics are disabled, we don't want to initialize it unnecessarily. - resultCount *prometheus.CounterVec = nil + initializedMetrics bool // Whether the metrics have been initialized + + resultTotal *prometheus.CounterVec + resultDurationSeconds *prometheus.GaugeVec + resultConnectedTotal *prometheus.CounterVec + resultCodeTotal *prometheus.CounterVec + resultCertificateExpirationSeconds *prometheus.GaugeVec ) +func initializePrometheusMetrics() { + resultTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "results_total", + Help: "Number of results per endpoint", + }, []string{"key", "group", "name", "type", "success"}) + resultDurationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "results_duration_seconds", + Help: "Duration of the request in seconds", + }, []string{"key", "group", "name", "type"}) + resultConnectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "results_connected_total", + Help: "Total number of results in which a connection was successfully established", + }, []string{"key", "group", "name", "type"}) + resultCodeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "results_code_total", + Help: "Total number of results by code", + }, []string{"key", "group", "name", "type", "code"}) + resultCertificateExpirationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "results_certificate_expiration_seconds", + Help: "Number of seconds until the certificate expires", + }, []string{"key", "group", "name", "type"}) +} + // PublishMetricsForEndpoint publishes metrics for the given endpoint and its result. // These metrics will be exposed at /metrics if the metrics are enabled func PublishMetricsForEndpoint(endpoint *core.Endpoint, result *core.Result) { - if resultCount == nil { - resultCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "gatus_results_total", - Help: "Number of results per endpoint", - }, []string{"key", "group", "name", "success"}) + if !initializedMetrics { + initializePrometheusMetrics() + initializedMetrics = true + } + endpointType := endpoint.Type() + resultTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), strconv.FormatBool(result.Success)).Inc() + resultDurationSeconds.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Set(result.Duration.Seconds()) + if result.Connected { + resultConnectedTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Inc() + } + if result.DNSRCode != "" { + resultCodeTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), result.DNSRCode).Inc() + } + if result.HTTPStatus != 0 { + resultCodeTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)).Inc() + } + if result.CertificateExpiration != 0 { + resultCertificateExpirationSeconds.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Set(result.CertificateExpiration.Seconds()) } - resultCount.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, strconv.FormatBool(result.Success)).Inc() }