Restore Graceful Restarting & Socket Activation (#7274)

* Prevent deadlock in indexer initialisation during graceful restart

* Move from gracehttp to our own service to add graceful ssh

* Add timeout for start of indexers and make hammer time configurable

* Fix issue with re-initialization in indexer during tests

* move the code to detect use of closed to graceful

* Handle logs gracefully - add a pid suffix just before restart

* Move to using a cond and a holder for indexers

* use time.Since

* Add some comments and attribution

* update modules.txt

* Use zero to disable timeout

* Move RestartProcess to its own file

* Add cleanup routine

cmd/web.go

@@ -75,17 +75,13 @@ func runLetsEncrypt(listenAddr, domain, directory, email string, m http.Handler)
 	}
 	go func() {
 		log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
-		var err = http.ListenAndServe(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler))) // all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
+		// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
+		var err = runHTTP(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))
 		if err != nil {
 			log.Fatal("Failed to start the Let's Encrypt handler on port %s: %v", setting.PortToRedirect, err)
 		}
 	}()
-	server := &http.Server{
-		Addr:      listenAddr,
-		Handler:   m,
-		TLSConfig: certManager.TLSConfig(),
-	}
-	return server.ListenAndServeTLS("", "")
+	return runHTTPSWithTLSConfig(listenAddr, certManager.TLSConfig(), context2.ClearHandler(m))
 }
 
 func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
@@ -101,12 +97,21 @@ func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func runWeb(ctx *cli.Context) error {
+	if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
+		log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
+	} else {
+		log.Info("Starting Gitea on PID: %d", os.Getpid())
+	}
+
+	// Set pid file setting
 	if ctx.IsSet("pid") {
 		setting.CustomPID = ctx.String("pid")
 	}
 
+	// Perform global initialization
 	routers.GlobalInit()
 
+	// Set up Macaron
 	m := routes.NewMacaron()
 	routes.RegisterRoutes(m)
 
@@ -164,6 +169,7 @@ func runWeb(ctx *cli.Context) error {
 	var err error
 	switch setting.Protocol {
 	case setting.HTTP:
+		NoHTTPRedirector()
 		err = runHTTP(listenAddr, context2.ClearHandler(m))
 	case setting.HTTPS:
 		if setting.EnableLetsEncrypt {
@@ -172,9 +178,15 @@ func runWeb(ctx *cli.Context) error {
 		}
 		if setting.RedirectOtherPort {
 			go runHTTPRedirector()
+		} else {
+			NoHTTPRedirector()
 		}
 		err = runHTTPS(listenAddr, setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
 	case setting.FCGI:
+		NoHTTPRedirector()
+		// FCGI listeners are provided as stdin - this is orthogonal to the LISTEN_FDS approach
+		// in graceful and systemD
+		NoMainListener()
 		var listener net.Listener
 		listener, err = net.Listen("tcp", listenAddr)
 		if err != nil {
@@ -187,6 +199,10 @@ func runWeb(ctx *cli.Context) error {
 		}()
 		err = fcgi.Serve(listener, context2.ClearHandler(m))
 	case setting.UnixSocket:
+		// This could potentially be inherited using LISTEN_FDS but currently
+		// these cannot be inherited
+		NoHTTPRedirector()
+		NoMainListener()
 		if err := os.Remove(listenAddr); err != nil && !os.IsNotExist(err) {
 			log.Fatal("Failed to remove unix socket directory %s: %v", listenAddr, err)
 		}
@@ -207,8 +223,9 @@ func runWeb(ctx *cli.Context) error {
 	}
 
 	if err != nil {
-		log.Fatal("Failed to start server: %v", err)
+		log.Critical("Failed to start server: %v", err)
 	}
-
+	log.Info("HTTP Listener: %s Closed", listenAddr)
+	log.Close()
 	return nil
 }

cmd/web_graceful.go

@@ -10,36 +10,28 @@ import (
 	"crypto/tls"
 	"net/http"
 
-	"code.gitea.io/gitea/modules/log"
-
-	"github.com/facebookgo/grace/gracehttp"
+	"code.gitea.io/gitea/modules/graceful"
 )
 
 func runHTTP(listenAddr string, m http.Handler) error {
-	return gracehttp.Serve(&http.Server{
-		Addr:    listenAddr,
-		Handler: m,
-	})
+	return graceful.HTTPListenAndServe("tcp", listenAddr, m)
 }
 
 func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
-	config := &tls.Config{
-		MinVersion: tls.VersionTLS10,
-	}
-	if config.NextProtos == nil {
-		config.NextProtos = []string{"http/1.1"}
-	}
-
-	config.Certificates = make([]tls.Certificate, 1)
-	var err error
-	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
-		log.Fatal("Failed to load https cert file %s: %v", listenAddr, err)
-	}
-
-	return gracehttp.Serve(&http.Server{
-		Addr:      listenAddr,
-		Handler:   m,
-		TLSConfig: config,
-	})
+	return graceful.HTTPListenAndServeTLS("tcp", listenAddr, certFile, keyFile, m)
+}
+
+func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
+	return graceful.HTTPListenAndServeTLSConfig("tcp", listenAddr, tlsConfig, m)
+}
+
+// NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
+func NoHTTPRedirector() {
+	graceful.InformCleanup()
+}
+
+// NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
+// for our main HTTP/HTTPS service
+func NoMainListener() {
+	graceful.InformCleanup()
 }

cmd/web_windows.go

@@ -7,6 +7,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"net/http"
 )
 
@@ -17,3 +18,20 @@ func runHTTP(listenAddr string, m http.Handler) error {
 func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
 	return http.ListenAndServeTLS(listenAddr, certFile, keyFile, m)
 }
+
+func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
+	server := &http.Server{
+		Addr:      listenAddr,
+		Handler:   m,
+		TLSConfig: tlsConfig,
+	}
+	return server.ListenAndServeTLS("", "")
+}
+
+// NoHTTPRedirector is a no-op on Windows
+func NoHTTPRedirector() {
+}
+
+// NoMainListener is a no-op on Windows
+func NoMainListener() {
+}