From 34313bec7ee7488e1ef9fd811be406d2d9719e26 Mon Sep 17 00:00:00 2001
From: I-HSIN Cheng <57989092+vax-r@users.noreply.github.com>
Date: Sat, 5 Aug 2023 06:30:15 +0800
Subject: [PATCH] fix(watchdog): Close dangling file descriptors on shutdown
 and config reload (#544)

* fix(watchdog): Add functions to avoid dangling file descriptors

* Change function name and add comment under core/endpoint.go
- change the function name of CloseHTTPConnection() to Close()
- add some comments above Close() function

* Update core/endpoint.go

* Update core/endpoint.go

---------

Co-authored-by: Richard Cheng <richard_cheng@trendmicro.com>
Co-authored-by: TwiN <twin@linux.com>
---
 core/endpoint.go     | 9 +++++++++
 main.go              | 9 +++++----
 watchdog/watchdog.go | 6 +++++-
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/core/endpoint.go b/core/endpoint.go
index 1214a6e8..b206dbda 100644
--- a/core/endpoint.go
+++ b/core/endpoint.go
@@ -364,6 +364,15 @@ func (endpoint *Endpoint) call(result *Result) {
 	}
 }
 
+// Close HTTP connections between watchdog and endpoints to avoid dangling socket file descriptors
+// on configuration reload.
+// More context on https://github.com/TwiN/gatus/issues/536
+func (endpoint *Endpoint) Close() {
+	if endpoint.Type() == EndpointTypeHTTP {
+		client.GetHTTPClient(endpoint.ClientConfig).CloseIdleConnections()
+	}
+}
+
 func (endpoint *Endpoint) buildHTTPRequest() *http.Request {
 	var bodyBuffer *bytes.Buffer
 	if endpoint.GraphQL {
diff --git a/main.go b/main.go
index 5b67a531..cd0c5972 100644
--- a/main.go
+++ b/main.go
@@ -27,7 +27,7 @@ func main() {
 	go func() {
 		<-signalChannel
 		log.Println("Received termination signal, attempting to gracefully shut down")
-		stop()
+		stop(cfg)
 		save()
 		done <- true
 	}()
@@ -41,8 +41,8 @@ func start(cfg *config.Config) {
 	go listenToConfigurationFileChanges(cfg)
 }
 
-func stop() {
-	watchdog.Shutdown()
+func stop(cfg *config.Config) {
+	watchdog.Shutdown(cfg)
 	controller.Shutdown()
 }
 
@@ -89,7 +89,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
 		time.Sleep(30 * time.Second)
 		if cfg.HasLoadedConfigurationBeenModified() {
 			log.Println("[main][listenToConfigurationFileChanges] Configuration file has been modified")
-			stop()
+			stop(cfg)
 			time.Sleep(time.Second) // Wait a bit to make sure everything is done.
 			save()
 			updatedConfig, err := loadConfiguration()
@@ -104,6 +104,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
 					panic(err)
 				}
 			}
+			store.Get().Close()
 			initializeStorage(updatedConfig)
 			start(updatedConfig)
 			return
diff --git a/watchdog/watchdog.go b/watchdog/watchdog.go
index a3257246..17d45205 100644
--- a/watchdog/watchdog.go
+++ b/watchdog/watchdog.go
@@ -96,6 +96,10 @@ func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
 }
 
 // Shutdown stops monitoring all endpoints
-func Shutdown() {
+func Shutdown(cfg *config.Config) {
+	// Disable all the old HTTP connections
+	for _, endpoint := range cfg.Endpoints {
+		endpoint.Close()
+	}
 	cancelFunc()
 }