Queue: Make WorkerPools and Queues flushable (#10001)

* Make WorkerPools and Queues flushable

Adds Flush methods to Queues and the WorkerPool
Further abstracts the WorkerPool
Adds a final step to Flush the queues in the defer from PrintCurrentTest
Fixes an issue with Settings inheritance in queues

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Change to for loop

* Add IsEmpty and begin just making the queues composed WorkerPools

* subsume workerpool into the queues and create a flushable interface

* Add manager command

* Move flushall to queue.Manager and add to testlogger

* As per @guillep2k

* as per @guillep2k

* Just make queues all implement flushable and clean up the wrapped queue flushes

* cope with no timeout

Co-authored-by: Lauris BH <lauris@nix.lv>

routers/admin/admin.go

@@ -404,6 +404,28 @@ func WorkerCancel(ctx *context.Context) {
 	})
 }
 
+// Flush flushes a queue
+func Flush(ctx *context.Context) {
+	qid := ctx.ParamsInt64("qid")
+	mq := queue.GetManager().GetManagedQueue(qid)
+	if mq == nil {
+		ctx.Status(404)
+		return
+	}
+	timeout, err := time.ParseDuration(ctx.Query("timeout"))
+	if err != nil {
+		timeout = -1
+	}
+	ctx.Flash.Info(ctx.Tr("admin.monitor.queue.pool.flush.added", mq.Name))
+	go func() {
+		err := mq.Flush(timeout)
+		if err != nil {
+			log.Error("Flushing failure for %s: Error %v", mq.Name, err)
+		}
+	}()
+	ctx.Redirect(setting.AppSubURL + fmt.Sprintf("/admin/monitor/queue/%d", qid))
+}
+
 // AddWorkers adds workers to a worker group
 func AddWorkers(ctx *context.Context) {
 	qid := ctx.ParamsInt64("qid")
@@ -424,7 +446,7 @@ func AddWorkers(ctx *context.Context) {
 		ctx.Redirect(setting.AppSubURL + fmt.Sprintf("/admin/monitor/queue/%d", qid))
 		return
 	}
-	if mq.Pool == nil {
+	if _, ok := mq.Managed.(queue.ManagedPool); !ok {
 		ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.none"))
 		ctx.Redirect(setting.AppSubURL + fmt.Sprintf("/admin/monitor/queue/%d", qid))
 		return
@@ -442,7 +464,7 @@ func SetQueueSettings(ctx *context.Context) {
 		ctx.Status(404)
 		return
 	}
-	if mq.Pool == nil {
+	if _, ok := mq.Managed.(queue.ManagedPool); !ok {
 		ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.none"))
 		ctx.Redirect(setting.AppSubURL + fmt.Sprintf("/admin/monitor/queue/%d", qid))
 		return
@@ -488,10 +510,10 @@ func SetQueueSettings(ctx *context.Context) {
 			return
 		}
 	} else {
-		timeout = mq.Pool.BoostTimeout()
+		timeout = mq.BoostTimeout()
 	}
 
-	mq.SetSettings(maxNumber, number, timeout)
+	mq.SetPoolSettings(maxNumber, number, timeout)
 	ctx.Flash.Success(ctx.Tr("admin.monitor.queue.settings.changed"))
 	ctx.Redirect(setting.AppSubURL + fmt.Sprintf("/admin/monitor/queue/%d", qid))
 }

routers/private/internal.go

@@ -89,5 +89,9 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/hook/set-default-branch/:owner/:repo/:branch", SetDefaultBranch)
 		m.Get("/serv/none/:keyid", ServNoCommand)
 		m.Get("/serv/command/:keyid/:owner/:repo", ServCommand)
+		m.Post("/manager/shutdown", Shutdown)
+		m.Post("/manager/restart", Restart)
+		m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues)
+
 	}, CheckInternalToken)
 }

routers/private/manager.go

@@ -0,0 +1,41 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	"code.gitea.io/gitea/modules/queue"
+
+	"gitea.com/macaron/macaron"
+)
+
+// FlushQueues flushes all the Queues
+func FlushQueues(ctx *macaron.Context, opts private.FlushOptions) {
+	if opts.NonBlocking {
+		// Save the hammer ctx here - as a new one is created each time you call this.
+		baseCtx := graceful.GetManager().HammerContext()
+		go func() {
+			err := queue.GetManager().FlushAll(baseCtx, opts.Timeout)
+			if err != nil {
+				log.Error("Flushing request timed-out with error: %v", err)
+			}
+		}()
+		ctx.JSON(http.StatusAccepted, map[string]interface{}{
+			"err": "Flushing",
+		})
+		return
+	}
+	err := queue.GetManager().FlushAll(ctx.Req.Request.Context(), opts.Timeout)
+	if err != nil {
+		ctx.JSON(http.StatusRequestTimeout, map[string]interface{}{
+			"err": err,
+		})
+	}
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}

routers/private/manager_unix.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+
+	"gitea.com/macaron/macaron"
+)
+
+// Restart causes the server to perform a graceful restart
+func Restart(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulRestart()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+
+}
+
+// Shutdown causes the server to perform a graceful shutdown
+func Shutdown(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulShutdown()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}

routers/private/manager_windows.go

@@ -0,0 +1,28 @@
+// +build windows
+
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+
+	"gitea.com/macaron/macaron"
+)
+
+// Restart is not implemented for Windows based servers as they can't fork
+func Restart(ctx *macaron.Context) {
+	ctx.JSON(http.StatusNotImplemented, map[string]interface{}{
+		"err": "windows servers cannot be gracefully restarted - shutdown and restart manually",
+	})
+}
+
+// Shutdown causes the server to perform a graceful shutdown
+func Shutdown(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulShutdown()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}

routers/routes/routes.go

@@ -423,6 +423,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Post("/set", admin.SetQueueSettings)
 				m.Post("/add", admin.AddWorkers)
 				m.Post("/cancel/:pid", admin.WorkerCancel)
+				m.Post("/flush", admin.Flush)
 			})
 		})