feat: Support multiple configuration files (#396)

* Revert "Revert "feat: Support multiple configuration files" (#395)" This reverts commit 87740e74a6. * feat: Properly implement support for config directory
2023-01-08 17:53:37 -05:00
parent 87740e74a6
commit 3059e3e028
35 changed files with 3130 additions and 1163 deletions
--- a/config/config.go
+++ b/config/config.go
@ -3,10 +3,13 @@ package config
 import (
 	"errors"
 	"fmt"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 	"time"

+	"github.com/TwiN/deepmerge"
 	"github.com/TwiN/gatus/v5/alerting"
 	"github.com/TwiN/gatus/v5/alerting/alert"
 	"github.com/TwiN/gatus/v5/alerting/provider"
@ -18,12 +21,12 @@ import (
 	"github.com/TwiN/gatus/v5/security"
 	"github.com/TwiN/gatus/v5/storage"
 	"github.com/TwiN/gatus/v5/util"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )

 const (
 	// DefaultConfigurationFilePath is the default path that will be used to search for the configuration file
-	// if a custom path isn't configured through the GATUS_CONFIG_FILE environment variable
+	// if a custom path isn't configured through the GATUS_CONFIG_PATH environment variable
 	DefaultConfigurationFilePath = "config/config.yaml"

 	// DefaultFallbackConfigurationFilePath is the default fallback path that will be used to search for the
@ -32,14 +35,17 @@ const (
 )

 var (
-	// ErrNoEndpointInConfig is an error returned when a configuration file has no endpoints configured
-	ErrNoEndpointInConfig = errors.New("configuration file should contain at least 1 endpoint")
+	// ErrNoEndpointInConfig is an error returned when a configuration file or directory has no endpoints configured
+	ErrNoEndpointInConfig = errors.New("configuration should contain at least 1 endpoint")

-	// ErrConfigFileNotFound is an error returned when the configuration file could not be found
+	// ErrConfigFileNotFound is an error returned when a configuration file could not be found
 	ErrConfigFileNotFound = errors.New("configuration file not found")

 	// ErrInvalidSecurityConfig is an error returned when the security configuration is invalid
 	ErrInvalidSecurityConfig = errors.New("invalid security configuration")
+
+	// errEarlyReturn is returned to break out of a loop from a callback early
+	errEarlyReturn = errors.New("early escape")
 )

 // Config is the main configuration structure
@ -84,11 +90,12 @@ type Config struct {
 	// WARNING: This is in ALPHA and may change or be completely removed in the future
 	Remote *remote.Config `yaml:"remote,omitempty"`

-	filePath        string    // path to the file from which config was loaded from
+	configPath      string    // path to the file or directory from which config was loaded
 	lastFileModTime time.Time // last modification time
 }

 func (config *Config) GetEndpointByKey(key string) *core.Endpoint {
+	// TODO: Should probably add a mutex here to prevent concurrent access
 	for i := 0; i < len(config.Endpoints); i++ {
 		ep := config.Endpoints[i]
 		if util.ConvertGroupAndEndpointNameToKey(ep.Group, ep.Name) == key {
@ -98,63 +105,111 @@ func (config *Config) GetEndpointByKey(key string) *core.Endpoint {
 	return nil
 }

-// HasLoadedConfigurationFileBeenModified returns whether the file that the
+// HasLoadedConfigurationBeenModified returns whether one of the file that the
 // configuration has been loaded from has been modified since it was last read
-func (config Config) HasLoadedConfigurationFileBeenModified() bool {
-	if fileInfo, err := os.Stat(config.filePath); err == nil {
-		if !fileInfo.ModTime().IsZero() {
-			return config.lastFileModTime.Unix() != fileInfo.ModTime().Unix()
-		}
+func (config Config) HasLoadedConfigurationBeenModified() bool {
+	lastMod := config.lastFileModTime.Unix()
+	fileInfo, err := os.Stat(config.configPath)
+	if err != nil {
+		return false
 	}
-	return false
+	if fileInfo.IsDir() {
+		err = walkConfigDir(config.configPath, func(path string, d fs.DirEntry, err error) error {
+			if info, err := d.Info(); err == nil && lastMod < info.ModTime().Unix() {
+				return errEarlyReturn
+			}
+			return nil
+		})
+		return err == errEarlyReturn
+	}
+	return !fileInfo.ModTime().IsZero() && config.lastFileModTime.Unix() < fileInfo.ModTime().Unix()
 }

 // UpdateLastFileModTime refreshes Config.lastFileModTime
 func (config *Config) UpdateLastFileModTime() {
-	if fileInfo, err := os.Stat(config.filePath); err == nil {
-		if !fileInfo.ModTime().IsZero() {
-			config.lastFileModTime = fileInfo.ModTime()
+	config.lastFileModTime = time.Now()
+}
+
+// LoadConfiguration loads the full configuration composed from the main configuration file
+// and all composed configuration files
+func LoadConfiguration(configPath string) (*Config, error) {
+	var configBytes []byte
+	var fileInfo os.FileInfo
+	var usedConfigPath string
+	// Figure out what config path we'll use (either configPath or the default config path)
+	for _, configurationPath := range []string{configPath, DefaultConfigurationFilePath, DefaultFallbackConfigurationFilePath} {
+		if len(configurationPath) == 0 {
+			continue
+		}
+		var err error
+		fileInfo, err = os.Stat(configurationPath)
+		if err != nil {
+			continue
+		}
+		usedConfigPath = configPath
+		break
+	}
+	if len(usedConfigPath) == 0 {
+		return nil, ErrConfigFileNotFound
+	}
+	var config *Config
+	if fileInfo.IsDir() {
+		err := walkConfigDir(configPath, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				log.Printf("[config][LoadConfiguration] Error walking path=%s: %s", path, err)
+				return err
+			}
+			log.Printf("[config][LoadConfiguration] Reading configuration from %s", path)
+			data, err := os.ReadFile(path)
+			if err != nil {
+				log.Printf("[config][LoadConfiguration] Error reading configuration from %s: %s", path, err)
+				return fmt.Errorf("error reading configuration from file %s: %w", path, err)
+			}
+			configBytes, err = deepmerge.YAML(configBytes, data)
+			return err
+		})
+		if err != nil {
+			return nil, fmt.Errorf("error reading configuration from directory %s: %w", usedConfigPath, err)
 		}
 	} else {
-		log.Println("[config][UpdateLastFileModTime] Ran into error updating lastFileModTime:", err.Error())
-	}
-}
-
-// Load loads a custom configuration file
-// Note that the misconfiguration of some fields may lead to panics. This is on purpose.
-func Load(configFile string) (*Config, error) {
-	log.Printf("[config][Load] Reading configuration from configFile=%s", configFile)
-	cfg, err := readConfigurationFile(configFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, ErrConfigFileNotFound
+		log.Printf("[config][LoadConfiguration] Reading configuration from configFile=%s", configPath)
+		if data, err := os.ReadFile(usedConfigPath); err != nil {
+			return nil, err
+		} else {
+			configBytes = data
 		}
+	}
+	if len(configBytes) == 0 {
+		return nil, ErrConfigFileNotFound
+	}
+	config, err := parseAndValidateConfigBytes(configBytes)
+	if err != nil {
 		return nil, err
 	}
-	cfg.filePath = configFile
-	cfg.UpdateLastFileModTime()
-	return cfg, nil
+	config.configPath = usedConfigPath
+	config.UpdateLastFileModTime()
+	return config, err
 }

-// LoadDefaultConfiguration loads the default configuration file
-func LoadDefaultConfiguration() (*Config, error) {
-	cfg, err := Load(DefaultConfigurationFilePath)
-	if err != nil {
-		if err == ErrConfigFileNotFound {
-			return Load(DefaultFallbackConfigurationFilePath)
+// walkConfigDir is a wrapper for filepath.WalkDir that strips directories and non-config files
+func walkConfigDir(path string, fn fs.WalkDirFunc) error {
+	if len(path) == 0 {
+		// If the user didn't provide a directory, we'll just use the default config file, so we can return nil now.
+		return nil
+	}
+	return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil
 		}
-		return nil, err
-	}
-	return cfg, nil
-}
-
-func readConfigurationFile(fileName string) (config *Config, err error) {
-	var bytes []byte
-	if bytes, err = os.ReadFile(fileName); err == nil {
-		// file exists, so we'll parse it and return it
-		return parseAndValidateConfigBytes(bytes)
-	}
-	return
+		if d == nil || d.IsDir() {
+			return nil
+		}
+		ext := filepath.Ext(path)
+		if ext != ".yml" && ext != ".yaml" {
+			return nil
+		}
+		return fn(path, d, err)
+	})
 }

 // parseAndValidateConfigBytes parses a Gatus configuration file into a Config struct and validates its parameters
--- a/config/config_test.go
+++ b/config/config_test.go
@ -1,7 +1,10 @@
 package config

 import (
+	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"

@ -11,6 +14,7 @@ import (
 	"github.com/TwiN/gatus/v5/alerting/provider/custom"
 	"github.com/TwiN/gatus/v5/alerting/provider/discord"
 	"github.com/TwiN/gatus/v5/alerting/provider/email"
+	"github.com/TwiN/gatus/v5/alerting/provider/github"
 	"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
 	"github.com/TwiN/gatus/v5/alerting/provider/matrix"
 	"github.com/TwiN/gatus/v5/alerting/provider/mattermost"
@ -26,20 +30,270 @@ import (
 	"github.com/TwiN/gatus/v5/config/web"
 	"github.com/TwiN/gatus/v5/core"
 	"github.com/TwiN/gatus/v5/storage"
+	"gopkg.in/yaml.v3"
 )

-func TestLoadFileThatDoesNotExist(t *testing.T) {
-	_, err := Load("file-that-does-not-exist.yaml")
-	if err == nil {
-		t.Error("Should've returned an error, because the file specified doesn't exist")
+func TestLoadConfiguration(t *testing.T) {
+	dir := t.TempDir()
+	scenarios := []struct {
+		name           string
+		configPath     string            // value to pass as the configPath parameter in LoadConfiguration
+		pathAndFiles   map[string]string // files to create in dir
+		expectedConfig *Config
+		expectedError  error
+	}{
+		{
+			name:       "empty-config-file",
+			configPath: filepath.Join(dir, "config.yaml"),
+			pathAndFiles: map[string]string{
+				"config.yaml": "",
+			},
+			expectedError: ErrConfigFileNotFound,
+		},
+		{
+			name:          "config-file-that-does-not-exist",
+			configPath:    filepath.Join(dir, "config.yaml"),
+			expectedError: ErrConfigFileNotFound,
+		},
+		{
+			name:       "config-file-with-endpoint-that-has-no-url",
+			configPath: filepath.Join(dir, "config.yaml"),
+			pathAndFiles: map[string]string{
+				"config.yaml": `
+endpoints:
+  - name: website`,
+			},
+			expectedError: core.ErrEndpointWithNoURL,
+		},
+		{
+			name:       "config-file-with-endpoint-that-has-no-conditions",
+			configPath: filepath.Join(dir, "config.yaml"),
+			pathAndFiles: map[string]string{
+				"config.yaml": `
+endpoints:
+  - name: website
+    url: https://twin.sh/health`,
+			},
+			expectedError: core.ErrEndpointWithNoCondition,
+		},
+		{
+			name:       "config-file",
+			configPath: filepath.Join(dir, "config.yaml"),
+			pathAndFiles: map[string]string{
+				"config.yaml": `
+endpoints:
+  - name: website
+    url: https://twin.sh/health
+    conditions:
+      - "[STATUS] == 200"`,
+			},
+			expectedConfig: &Config{
+				Endpoints: []*core.Endpoint{
+					{
+						Name:       "website",
+						URL:        "https://twin.sh/health",
+						Conditions: []core.Condition{"[STATUS] == 200"},
+					},
+				},
+			},
+		},
+		{
+			name:          "empty-dir",
+			configPath:    dir,
+			pathAndFiles:  map[string]string{},
+			expectedError: ErrConfigFileNotFound,
+		},
+		{
+			name:       "dir-with-empty-config-file",
+			configPath: dir,
+			pathAndFiles: map[string]string{
+				"config.yaml": "",
+			},
+			expectedError: ErrNoEndpointInConfig,
+		},
+		{
+			name:       "dir-with-two-config-files",
+			configPath: dir,
+			pathAndFiles: map[string]string{
+				"config.yaml": `endpoints: 
+  - name: one
+    url: https://example.com
+    conditions:
+      - "[CONNECTED] == true"
+      - "[STATUS] == 200"
+
+  - name: two
+    url: https://example.org
+    conditions:
+      - "len([BODY]) > 0"`,
+				"config.yml": `endpoints: 
+  - name: three
+    url: https://twin.sh/health
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"`,
+			},
+			expectedConfig: &Config{
+				Endpoints: []*core.Endpoint{
+					{
+						Name:       "one",
+						URL:        "https://example.com",
+						Conditions: []core.Condition{"[CONNECTED] == true", "[STATUS] == 200"},
+					},
+					{
+						Name:       "two",
+						URL:        "https://example.org",
+						Conditions: []core.Condition{"len([BODY]) > 0"},
+					},
+					{
+						Name:       "three",
+						URL:        "https://twin.sh/health",
+						Conditions: []core.Condition{"[STATUS] == 200", "[BODY].status == UP"},
+					},
+				},
+			},
+		},
+		{
+			name:       "dir-with-2-config-files-deep-merge-with-map-slice-and-primitive",
+			configPath: dir,
+			pathAndFiles: map[string]string{
+				"a.yaml": `
+metrics: true
+
+alerting:
+  slack:
+    webhook-url: https://hooks.slack.com/services/xxx/yyy/zzz
+
+endpoints:
+  - name: example
+    url: https://example.org
+    interval: 5s
+    conditions:
+      - "[STATUS] == 200"`,
+				"b.yaml": `
+debug: true
+
+alerting:
+  discord:
+    webhook-url: https://discord.com/api/webhooks/xxx/yyy
+
+endpoints:
+  - name: frontend
+    url: https://example.com
+    conditions:
+      - "[STATUS] == 200"`,
+			},
+			expectedConfig: &Config{
+				Debug:   true,
+				Metrics: true,
+				Alerting: &alerting.Config{
+					Discord: &discord.AlertProvider{WebhookURL: "https://discord.com/api/webhooks/xxx/yyy"},
+					Slack:   &slack.AlertProvider{WebhookURL: "https://hooks.slack.com/services/xxx/yyy/zzz"},
+				},
+				Endpoints: []*core.Endpoint{
+					{
+						Name:       "example",
+						URL:        "https://example.org",
+						Interval:   5 * time.Second,
+						Conditions: []core.Condition{"[STATUS] == 200"},
+					},
+					{
+						Name:       "frontend",
+						URL:        "https://example.com",
+						Conditions: []core.Condition{"[STATUS] == 200"},
+					},
+				},
+			},
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			for path, content := range scenario.pathAndFiles {
+				if err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0644); err != nil {
+					t.Fatalf("[%s] failed to write file: %v", scenario.name, err)
+				}
+			}
+			defer func(pathAndFiles map[string]string) {
+				for path := range pathAndFiles {
+					_ = os.Remove(filepath.Join(dir, path))
+				}
+			}(scenario.pathAndFiles)
+			config, err := LoadConfiguration(scenario.configPath)
+			if !errors.Is(err, scenario.expectedError) {
+				t.Errorf("[%s] expected error %v, got %v", scenario.name, scenario.expectedError, err)
+				return
+			} else if err != nil && errors.Is(err, scenario.expectedError) {
+				return
+			}
+			// parse the expected output so that expectations are closer to reality (under the right circumstances, even I can be poetic)
+			expectedConfigAsYAML, _ := yaml.Marshal(scenario.expectedConfig)
+			expectedConfigAfterBeingParsedAndValidated, err := parseAndValidateConfigBytes(expectedConfigAsYAML)
+			if err != nil {
+				t.Fatalf("[%s] failed to parse expected config: %v", scenario.name, err)
+			}
+			// Marshal em' before comparing em' so that we don't have to deal with formatting and ordering
+			actualConfigAsYAML, err := yaml.Marshal(config)
+			if err != nil {
+				t.Fatalf("[%s] failed to marshal actual config: %v", scenario.name, err)
+			}
+			expectedConfigAfterBeingParsedAndValidatedAsYAML, _ := yaml.Marshal(expectedConfigAfterBeingParsedAndValidated)
+			if string(actualConfigAsYAML) != string(expectedConfigAfterBeingParsedAndValidatedAsYAML) {
+				t.Errorf("[%s] expected config %s, got %s", scenario.name, string(expectedConfigAfterBeingParsedAndValidatedAsYAML), string(actualConfigAsYAML))
+			}
+		})
 	}
 }

-func TestLoadDefaultConfigurationFile(t *testing.T) {
-	_, err := LoadDefaultConfiguration()
-	if err == nil {
-		t.Error("Should've returned an error, because there's no configuration files at the default path nor the default fallback path")
-	}
+func TestConfig_HasLoadedConfigurationBeenModified(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	configFilePath := filepath.Join(dir, "config.yaml")
+	_ = os.WriteFile(configFilePath, []byte(`endpoints:
+  - name: website
+    url: https://twin.sh/health
+    conditions:
+      - "[STATUS] == 200"
+`), 0644)
+
+	t.Run("config-file-as-config-path", func(t *testing.T) {
+		config, err := LoadConfiguration(configFilePath)
+		if err != nil {
+			t.Fatalf("failed to load configuration: %v", err)
+		}
+		if config.HasLoadedConfigurationBeenModified() {
+			t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return false because nothing has happened since it was created")
+		}
+		time.Sleep(time.Second) // Because the file mod time only has second precision, we have to wait for a second
+		// Update the config file
+		if err = os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(`endpoints:
+  - name: website
+    url: https://twin.sh/health
+    conditions:
+      - "[STATUS] == 200"`), 0644); err != nil {
+			t.Fatalf("failed to overwrite config file: %v", err)
+		}
+		if !config.HasLoadedConfigurationBeenModified() {
+			t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return true because a new file has been added in the directory")
+		}
+	})
+	t.Run("config-directory-as-config-path", func(t *testing.T) {
+		config, err := LoadConfiguration(dir)
+		if err != nil {
+			t.Fatalf("failed to load configuration: %v", err)
+		}
+		if config.HasLoadedConfigurationBeenModified() {
+			t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return false because nothing has happened since it was created")
+		}
+		time.Sleep(time.Second) // Because the file mod time only has second precision, we have to wait for a second
+		// Update the config file
+		if err = os.WriteFile(filepath.Join(dir, "metrics.yaml"), []byte(`metrics: true`), 0644); err != nil {
+			t.Fatalf("failed to overwrite config file: %v", err)
+		}
+		if !config.HasLoadedConfigurationBeenModified() {
+			t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return true because a new file has been added in the directory")
+		}
+	})
 }

 func TestParseAndValidateConfigBytes(t *testing.T) {
@ -1219,6 +1473,7 @@ func TestGetAlertingProviderByAlertType(t *testing.T) {
 		Custom:      &custom.AlertProvider{},
 		Discord:     &discord.AlertProvider{},
 		Email:       &email.AlertProvider{},
+		GitHub:      &github.AlertProvider{},
 		GoogleChat:  &googlechat.AlertProvider{},
 		Matrix:      &matrix.AlertProvider{},
 		Mattermost:  &mattermost.AlertProvider{},
@ -1238,6 +1493,7 @@ func TestGetAlertingProviderByAlertType(t *testing.T) {
 		{alertType: alert.TypeCustom, expected: alertingConfig.Custom},
 		{alertType: alert.TypeDiscord, expected: alertingConfig.Discord},
 		{alertType: alert.TypeEmail, expected: alertingConfig.Email},
+		{alertType: alert.TypeGitHub, expected: alertingConfig.GitHub},
 		{alertType: alert.TypeGoogleChat, expected: alertingConfig.GoogleChat},
 		{alertType: alert.TypeMatrix, expected: alertingConfig.Matrix},
 		{alertType: alert.TypeMattermost, expected: alertingConfig.Mattermost},