feat: Support multiple configuration files (#396)

* Revert "Revert "feat: Support multiple configuration files" (#395)"

This reverts commit 87740e74a6.

* feat: Properly implement support for config directory
This commit is contained in:
TwiN
2023-01-08 17:53:37 -05:00
committed by GitHub
parent 87740e74a6
commit 3059e3e028
35 changed files with 3130 additions and 1163 deletions

View File

@ -3,10 +3,13 @@ package config
import (
"errors"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"time"
"github.com/TwiN/deepmerge"
"github.com/TwiN/gatus/v5/alerting"
"github.com/TwiN/gatus/v5/alerting/alert"
"github.com/TwiN/gatus/v5/alerting/provider"
@ -18,12 +21,12 @@ import (
"github.com/TwiN/gatus/v5/security"
"github.com/TwiN/gatus/v5/storage"
"github.com/TwiN/gatus/v5/util"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
)
const (
// DefaultConfigurationFilePath is the default path that will be used to search for the configuration file
// if a custom path isn't configured through the GATUS_CONFIG_FILE environment variable
// if a custom path isn't configured through the GATUS_CONFIG_PATH environment variable
DefaultConfigurationFilePath = "config/config.yaml"
// DefaultFallbackConfigurationFilePath is the default fallback path that will be used to search for the
@ -32,14 +35,17 @@ const (
)
var (
// ErrNoEndpointInConfig is an error returned when a configuration file has no endpoints configured
ErrNoEndpointInConfig = errors.New("configuration file should contain at least 1 endpoint")
// ErrNoEndpointInConfig is an error returned when a configuration file or directory has no endpoints configured
ErrNoEndpointInConfig = errors.New("configuration should contain at least 1 endpoint")
// ErrConfigFileNotFound is an error returned when the configuration file could not be found
// ErrConfigFileNotFound is an error returned when a configuration file could not be found
ErrConfigFileNotFound = errors.New("configuration file not found")
// ErrInvalidSecurityConfig is an error returned when the security configuration is invalid
ErrInvalidSecurityConfig = errors.New("invalid security configuration")
// errEarlyReturn is returned to break out of a loop from a callback early
errEarlyReturn = errors.New("early escape")
)
// Config is the main configuration structure
@ -84,11 +90,12 @@ type Config struct {
// WARNING: This is in ALPHA and may change or be completely removed in the future
Remote *remote.Config `yaml:"remote,omitempty"`
filePath string // path to the file from which config was loaded from
configPath string // path to the file or directory from which config was loaded
lastFileModTime time.Time // last modification time
}
func (config *Config) GetEndpointByKey(key string) *core.Endpoint {
// TODO: Should probably add a mutex here to prevent concurrent access
for i := 0; i < len(config.Endpoints); i++ {
ep := config.Endpoints[i]
if util.ConvertGroupAndEndpointNameToKey(ep.Group, ep.Name) == key {
@ -98,63 +105,111 @@ func (config *Config) GetEndpointByKey(key string) *core.Endpoint {
return nil
}
// HasLoadedConfigurationFileBeenModified returns whether the file that the
// HasLoadedConfigurationBeenModified returns whether one of the file that the
// configuration has been loaded from has been modified since it was last read
func (config Config) HasLoadedConfigurationFileBeenModified() bool {
if fileInfo, err := os.Stat(config.filePath); err == nil {
if !fileInfo.ModTime().IsZero() {
return config.lastFileModTime.Unix() != fileInfo.ModTime().Unix()
}
func (config Config) HasLoadedConfigurationBeenModified() bool {
lastMod := config.lastFileModTime.Unix()
fileInfo, err := os.Stat(config.configPath)
if err != nil {
return false
}
return false
if fileInfo.IsDir() {
err = walkConfigDir(config.configPath, func(path string, d fs.DirEntry, err error) error {
if info, err := d.Info(); err == nil && lastMod < info.ModTime().Unix() {
return errEarlyReturn
}
return nil
})
return err == errEarlyReturn
}
return !fileInfo.ModTime().IsZero() && config.lastFileModTime.Unix() < fileInfo.ModTime().Unix()
}
// UpdateLastFileModTime refreshes Config.lastFileModTime
func (config *Config) UpdateLastFileModTime() {
if fileInfo, err := os.Stat(config.filePath); err == nil {
if !fileInfo.ModTime().IsZero() {
config.lastFileModTime = fileInfo.ModTime()
config.lastFileModTime = time.Now()
}
// LoadConfiguration loads the full configuration composed from the main configuration file
// and all composed configuration files
func LoadConfiguration(configPath string) (*Config, error) {
var configBytes []byte
var fileInfo os.FileInfo
var usedConfigPath string
// Figure out what config path we'll use (either configPath or the default config path)
for _, configurationPath := range []string{configPath, DefaultConfigurationFilePath, DefaultFallbackConfigurationFilePath} {
if len(configurationPath) == 0 {
continue
}
var err error
fileInfo, err = os.Stat(configurationPath)
if err != nil {
continue
}
usedConfigPath = configPath
break
}
if len(usedConfigPath) == 0 {
return nil, ErrConfigFileNotFound
}
var config *Config
if fileInfo.IsDir() {
err := walkConfigDir(configPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.Printf("[config][LoadConfiguration] Error walking path=%s: %s", path, err)
return err
}
log.Printf("[config][LoadConfiguration] Reading configuration from %s", path)
data, err := os.ReadFile(path)
if err != nil {
log.Printf("[config][LoadConfiguration] Error reading configuration from %s: %s", path, err)
return fmt.Errorf("error reading configuration from file %s: %w", path, err)
}
configBytes, err = deepmerge.YAML(configBytes, data)
return err
})
if err != nil {
return nil, fmt.Errorf("error reading configuration from directory %s: %w", usedConfigPath, err)
}
} else {
log.Println("[config][UpdateLastFileModTime] Ran into error updating lastFileModTime:", err.Error())
}
}
// Load loads a custom configuration file
// Note that the misconfiguration of some fields may lead to panics. This is on purpose.
func Load(configFile string) (*Config, error) {
log.Printf("[config][Load] Reading configuration from configFile=%s", configFile)
cfg, err := readConfigurationFile(configFile)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrConfigFileNotFound
log.Printf("[config][LoadConfiguration] Reading configuration from configFile=%s", configPath)
if data, err := os.ReadFile(usedConfigPath); err != nil {
return nil, err
} else {
configBytes = data
}
}
if len(configBytes) == 0 {
return nil, ErrConfigFileNotFound
}
config, err := parseAndValidateConfigBytes(configBytes)
if err != nil {
return nil, err
}
cfg.filePath = configFile
cfg.UpdateLastFileModTime()
return cfg, nil
config.configPath = usedConfigPath
config.UpdateLastFileModTime()
return config, err
}
// LoadDefaultConfiguration loads the default configuration file
func LoadDefaultConfiguration() (*Config, error) {
cfg, err := Load(DefaultConfigurationFilePath)
if err != nil {
if err == ErrConfigFileNotFound {
return Load(DefaultFallbackConfigurationFilePath)
// walkConfigDir is a wrapper for filepath.WalkDir that strips directories and non-config files
func walkConfigDir(path string, fn fs.WalkDirFunc) error {
if len(path) == 0 {
// If the user didn't provide a directory, we'll just use the default config file, so we can return nil now.
return nil
}
return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
return nil, err
}
return cfg, nil
}
func readConfigurationFile(fileName string) (config *Config, err error) {
var bytes []byte
if bytes, err = os.ReadFile(fileName); err == nil {
// file exists, so we'll parse it and return it
return parseAndValidateConfigBytes(bytes)
}
return
if d == nil || d.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext != ".yml" && ext != ".yaml" {
return nil
}
return fn(path, d, err)
})
}
// parseAndValidateConfigBytes parses a Gatus configuration file into a Config struct and validates its parameters

View File

@ -1,7 +1,10 @@
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"time"
@ -11,6 +14,7 @@ import (
"github.com/TwiN/gatus/v5/alerting/provider/custom"
"github.com/TwiN/gatus/v5/alerting/provider/discord"
"github.com/TwiN/gatus/v5/alerting/provider/email"
"github.com/TwiN/gatus/v5/alerting/provider/github"
"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
"github.com/TwiN/gatus/v5/alerting/provider/matrix"
"github.com/TwiN/gatus/v5/alerting/provider/mattermost"
@ -26,20 +30,270 @@ import (
"github.com/TwiN/gatus/v5/config/web"
"github.com/TwiN/gatus/v5/core"
"github.com/TwiN/gatus/v5/storage"
"gopkg.in/yaml.v3"
)
func TestLoadFileThatDoesNotExist(t *testing.T) {
_, err := Load("file-that-does-not-exist.yaml")
if err == nil {
t.Error("Should've returned an error, because the file specified doesn't exist")
func TestLoadConfiguration(t *testing.T) {
dir := t.TempDir()
scenarios := []struct {
name string
configPath string // value to pass as the configPath parameter in LoadConfiguration
pathAndFiles map[string]string // files to create in dir
expectedConfig *Config
expectedError error
}{
{
name: "empty-config-file",
configPath: filepath.Join(dir, "config.yaml"),
pathAndFiles: map[string]string{
"config.yaml": "",
},
expectedError: ErrConfigFileNotFound,
},
{
name: "config-file-that-does-not-exist",
configPath: filepath.Join(dir, "config.yaml"),
expectedError: ErrConfigFileNotFound,
},
{
name: "config-file-with-endpoint-that-has-no-url",
configPath: filepath.Join(dir, "config.yaml"),
pathAndFiles: map[string]string{
"config.yaml": `
endpoints:
- name: website`,
},
expectedError: core.ErrEndpointWithNoURL,
},
{
name: "config-file-with-endpoint-that-has-no-conditions",
configPath: filepath.Join(dir, "config.yaml"),
pathAndFiles: map[string]string{
"config.yaml": `
endpoints:
- name: website
url: https://twin.sh/health`,
},
expectedError: core.ErrEndpointWithNoCondition,
},
{
name: "config-file",
configPath: filepath.Join(dir, "config.yaml"),
pathAndFiles: map[string]string{
"config.yaml": `
endpoints:
- name: website
url: https://twin.sh/health
conditions:
- "[STATUS] == 200"`,
},
expectedConfig: &Config{
Endpoints: []*core.Endpoint{
{
Name: "website",
URL: "https://twin.sh/health",
Conditions: []core.Condition{"[STATUS] == 200"},
},
},
},
},
{
name: "empty-dir",
configPath: dir,
pathAndFiles: map[string]string{},
expectedError: ErrConfigFileNotFound,
},
{
name: "dir-with-empty-config-file",
configPath: dir,
pathAndFiles: map[string]string{
"config.yaml": "",
},
expectedError: ErrNoEndpointInConfig,
},
{
name: "dir-with-two-config-files",
configPath: dir,
pathAndFiles: map[string]string{
"config.yaml": `endpoints:
- name: one
url: https://example.com
conditions:
- "[CONNECTED] == true"
- "[STATUS] == 200"
- name: two
url: https://example.org
conditions:
- "len([BODY]) > 0"`,
"config.yml": `endpoints:
- name: three
url: https://twin.sh/health
conditions:
- "[STATUS] == 200"
- "[BODY].status == UP"`,
},
expectedConfig: &Config{
Endpoints: []*core.Endpoint{
{
Name: "one",
URL: "https://example.com",
Conditions: []core.Condition{"[CONNECTED] == true", "[STATUS] == 200"},
},
{
Name: "two",
URL: "https://example.org",
Conditions: []core.Condition{"len([BODY]) > 0"},
},
{
Name: "three",
URL: "https://twin.sh/health",
Conditions: []core.Condition{"[STATUS] == 200", "[BODY].status == UP"},
},
},
},
},
{
name: "dir-with-2-config-files-deep-merge-with-map-slice-and-primitive",
configPath: dir,
pathAndFiles: map[string]string{
"a.yaml": `
metrics: true
alerting:
slack:
webhook-url: https://hooks.slack.com/services/xxx/yyy/zzz
endpoints:
- name: example
url: https://example.org
interval: 5s
conditions:
- "[STATUS] == 200"`,
"b.yaml": `
debug: true
alerting:
discord:
webhook-url: https://discord.com/api/webhooks/xxx/yyy
endpoints:
- name: frontend
url: https://example.com
conditions:
- "[STATUS] == 200"`,
},
expectedConfig: &Config{
Debug: true,
Metrics: true,
Alerting: &alerting.Config{
Discord: &discord.AlertProvider{WebhookURL: "https://discord.com/api/webhooks/xxx/yyy"},
Slack: &slack.AlertProvider{WebhookURL: "https://hooks.slack.com/services/xxx/yyy/zzz"},
},
Endpoints: []*core.Endpoint{
{
Name: "example",
URL: "https://example.org",
Interval: 5 * time.Second,
Conditions: []core.Condition{"[STATUS] == 200"},
},
{
Name: "frontend",
URL: "https://example.com",
Conditions: []core.Condition{"[STATUS] == 200"},
},
},
},
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
for path, content := range scenario.pathAndFiles {
if err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0644); err != nil {
t.Fatalf("[%s] failed to write file: %v", scenario.name, err)
}
}
defer func(pathAndFiles map[string]string) {
for path := range pathAndFiles {
_ = os.Remove(filepath.Join(dir, path))
}
}(scenario.pathAndFiles)
config, err := LoadConfiguration(scenario.configPath)
if !errors.Is(err, scenario.expectedError) {
t.Errorf("[%s] expected error %v, got %v", scenario.name, scenario.expectedError, err)
return
} else if err != nil && errors.Is(err, scenario.expectedError) {
return
}
// parse the expected output so that expectations are closer to reality (under the right circumstances, even I can be poetic)
expectedConfigAsYAML, _ := yaml.Marshal(scenario.expectedConfig)
expectedConfigAfterBeingParsedAndValidated, err := parseAndValidateConfigBytes(expectedConfigAsYAML)
if err != nil {
t.Fatalf("[%s] failed to parse expected config: %v", scenario.name, err)
}
// Marshal em' before comparing em' so that we don't have to deal with formatting and ordering
actualConfigAsYAML, err := yaml.Marshal(config)
if err != nil {
t.Fatalf("[%s] failed to marshal actual config: %v", scenario.name, err)
}
expectedConfigAfterBeingParsedAndValidatedAsYAML, _ := yaml.Marshal(expectedConfigAfterBeingParsedAndValidated)
if string(actualConfigAsYAML) != string(expectedConfigAfterBeingParsedAndValidatedAsYAML) {
t.Errorf("[%s] expected config %s, got %s", scenario.name, string(expectedConfigAfterBeingParsedAndValidatedAsYAML), string(actualConfigAsYAML))
}
})
}
}
func TestLoadDefaultConfigurationFile(t *testing.T) {
_, err := LoadDefaultConfiguration()
if err == nil {
t.Error("Should've returned an error, because there's no configuration files at the default path nor the default fallback path")
}
func TestConfig_HasLoadedConfigurationBeenModified(t *testing.T) {
t.Parallel()
dir := t.TempDir()
configFilePath := filepath.Join(dir, "config.yaml")
_ = os.WriteFile(configFilePath, []byte(`endpoints:
- name: website
url: https://twin.sh/health
conditions:
- "[STATUS] == 200"
`), 0644)
t.Run("config-file-as-config-path", func(t *testing.T) {
config, err := LoadConfiguration(configFilePath)
if err != nil {
t.Fatalf("failed to load configuration: %v", err)
}
if config.HasLoadedConfigurationBeenModified() {
t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return false because nothing has happened since it was created")
}
time.Sleep(time.Second) // Because the file mod time only has second precision, we have to wait for a second
// Update the config file
if err = os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(`endpoints:
- name: website
url: https://twin.sh/health
conditions:
- "[STATUS] == 200"`), 0644); err != nil {
t.Fatalf("failed to overwrite config file: %v", err)
}
if !config.HasLoadedConfigurationBeenModified() {
t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return true because a new file has been added in the directory")
}
})
t.Run("config-directory-as-config-path", func(t *testing.T) {
config, err := LoadConfiguration(dir)
if err != nil {
t.Fatalf("failed to load configuration: %v", err)
}
if config.HasLoadedConfigurationBeenModified() {
t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return false because nothing has happened since it was created")
}
time.Sleep(time.Second) // Because the file mod time only has second precision, we have to wait for a second
// Update the config file
if err = os.WriteFile(filepath.Join(dir, "metrics.yaml"), []byte(`metrics: true`), 0644); err != nil {
t.Fatalf("failed to overwrite config file: %v", err)
}
if !config.HasLoadedConfigurationBeenModified() {
t.Errorf("expected config.HasLoadedConfigurationBeenModified() to return true because a new file has been added in the directory")
}
})
}
func TestParseAndValidateConfigBytes(t *testing.T) {
@ -1219,6 +1473,7 @@ func TestGetAlertingProviderByAlertType(t *testing.T) {
Custom: &custom.AlertProvider{},
Discord: &discord.AlertProvider{},
Email: &email.AlertProvider{},
GitHub: &github.AlertProvider{},
GoogleChat: &googlechat.AlertProvider{},
Matrix: &matrix.AlertProvider{},
Mattermost: &mattermost.AlertProvider{},
@ -1238,6 +1493,7 @@ func TestGetAlertingProviderByAlertType(t *testing.T) {
{alertType: alert.TypeCustom, expected: alertingConfig.Custom},
{alertType: alert.TypeDiscord, expected: alertingConfig.Discord},
{alertType: alert.TypeEmail, expected: alertingConfig.Email},
{alertType: alert.TypeGitHub, expected: alertingConfig.GitHub},
{alertType: alert.TypeGoogleChat, expected: alertingConfig.GoogleChat},
{alertType: alert.TypeMatrix, expected: alertingConfig.Matrix},
{alertType: alert.TypeMattermost, expected: alertingConfig.Mattermost},