Pause, Resume, Release&Reopen, Add and Remove Logging from command line (#11777)

* Make LogDescriptions race safe

* Add manager commands for pausing, resuming, adding and removing loggers

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Placate lint

* Ensure that file logger is run!

* Add support for smtp and conn

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add release-and-reopen

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lauris BH <lauris@nix.lv>

modules/graceful/manager_unix.go

@@ -113,7 +113,10 @@ func (g *Manager) handleSignals(ctx context.Context) {
 				log.Info("PID: %d. Received SIGHUP. Attempting GracefulRestart...", pid)
 				g.DoGracefulRestart()
 			case syscall.SIGUSR1:
-				log.Info("PID %d. Received SIGUSR1.", pid)
+				log.Warn("PID %d. Received SIGUSR1. Releasing and reopening logs", pid)
+				if err := log.ReleaseReopen(); err != nil {
+					log.Error("Error whilst releasing and reopening logs: %v", err)
+				}
 			case syscall.SIGUSR2:
 				log.Warn("PID %d. Received SIGUSR2. Hammering...", pid)
 				g.DoImmediateHammer()

modules/log/conn.go

@@ -77,6 +77,13 @@ func (i *connWriter) connect() error {
 	return nil
 }
 
+func (i *connWriter) releaseReopen() error {
+	if i.innerWriter != nil {
+		return i.connect()
+	}
+	return nil
+}
+
 // ConnLogger implements LoggerProvider.
 // it writes messages in keep-live tcp connection.
 type ConnLogger struct {
@@ -119,6 +126,11 @@ func (log *ConnLogger) GetName() string {
 	return "conn"
 }
 
+// ReleaseReopen causes the ConnLogger to reconnect to the server
+func (log *ConnLogger) ReleaseReopen() error {
+	return log.out.(*connWriter).releaseReopen()
+}
+
 func init() {
 	Register("conn", NewConn)
 }

modules/log/console.go

@@ -68,6 +68,20 @@ func (log *ConsoleLogger) Init(config string) error {
 func (log *ConsoleLogger) Flush() {
 }
 
+// ReleaseReopen causes the console logger to reconnect to os.Stdout
+func (log *ConsoleLogger) ReleaseReopen() error {
+	if log.Stderr {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stderr,
+		})
+	} else {
+		log.NewWriterLogger(&nopWriteCloser{
+			w: os.Stdout,
+		})
+	}
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *ConsoleLogger) GetName() string {
 	return "console"

modules/log/event.go

@@ -29,6 +29,7 @@ type EventLogger interface {
 	GetLevel() Level
 	GetStacktraceLevel() Level
 	GetName() string
+	ReleaseReopen() error
 }
 
 // ChannelledLog represents a cached channel to a LoggerProvider
@@ -117,6 +118,11 @@ func (l *ChannelledLog) Flush() {
 	l.flush <- true
 }
 
+// ReleaseReopen this ChannelledLog
+func (l *ChannelledLog) ReleaseReopen() error {
+	return l.loggerProvider.ReleaseReopen()
+}
+
 // GetLevel gets the level of this ChannelledLog
 func (l *ChannelledLog) GetLevel() Level {
 	return l.loggerProvider.GetLevel()
@@ -145,6 +151,7 @@ type MultiChannelledLog struct {
 	level           Level
 	stacktraceLevel Level
 	closed          chan bool
+	paused          chan bool
 }
 
 // NewMultiChannelledLog a new logger instance with given logger provider and config.
@@ -159,6 +166,7 @@ func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog
 		stacktraceLevel: NONE,
 		close:           make(chan bool),
 		closed:          make(chan bool),
+		paused:          make(chan bool),
 	}
 	return m
 }
@@ -229,6 +237,33 @@ func (m *MultiChannelledLog) closeLoggers() {
 	m.closed <- true
 }
 
+// Pause pauses this Logger
+func (m *MultiChannelledLog) Pause() {
+	m.paused <- true
+}
+
+// Resume resumes this Logger
+func (m *MultiChannelledLog) Resume() {
+	m.paused <- false
+}
+
+// ReleaseReopen causes this logger to tell its subloggers to release and reopen
+func (m *MultiChannelledLog) ReleaseReopen() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	var accumulatedErr error
+	for _, logger := range m.loggers {
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v", logger.GetName(), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v & %v", logger.GetName(), err, accumulatedErr)
+			}
+		}
+	}
+	return accumulatedErr
+}
+
 // Start processing the MultiChannelledLog
 func (m *MultiChannelledLog) Start() {
 	m.mutex.Lock()
@@ -238,8 +273,35 @@ func (m *MultiChannelledLog) Start() {
 	}
 	m.started = true
 	m.mutex.Unlock()
+	paused := false
 	for {
+		if paused {
+			select {
+			case paused = <-m.paused:
+				if !paused {
+					m.ResetLevel()
+				}
+			case _, ok := <-m.flush:
+				if !ok {
+					m.closeLoggers()
+					return
+				}
+				m.mutex.Lock()
+				for _, logger := range m.loggers {
+					logger.Flush()
+				}
+				m.mutex.Unlock()
+			case <-m.close:
+				m.closeLoggers()
+				return
+			}
+			continue
+		}
 		select {
+		case paused = <-m.paused:
+			if paused && m.level < INFO {
+				m.level = INFO
+			}
 		case event, ok := <-m.queue:
 			if !ok {
 				m.closeLoggers()
@@ -275,7 +337,7 @@ func (m *MultiChannelledLog) LogEvent(event *Event) error {
 	select {
 	case m.queue <- event:
 		return nil
-	case <-time.After(60 * time.Second):
+	case <-time.After(100 * time.Millisecond):
 		// We're blocked!
 		return ErrTimeout{
 			Name:     m.name,

modules/log/file.go

@@ -249,6 +249,19 @@ func (log *FileLogger) Flush() {
 	_ = log.mw.fd.Sync()
 }
 
+// ReleaseReopen releases and reopens log files
+func (log *FileLogger) ReleaseReopen() error {
+	closingErr := log.mw.fd.Close()
+	startingErr := log.StartLogger()
+	if startingErr != nil {
+		if closingErr != nil {
+			return fmt.Errorf("Error during closing: %v Error during starting: %v", closingErr, startingErr)
+		}
+		return startingErr
+	}
+	return closingErr
+}
+
 // GetName returns the default name for this implementation
 func (log *FileLogger) GetName() string {
 	return "file"

modules/log/log.go

@@ -5,6 +5,7 @@
 package log
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strings"
@@ -192,6 +193,42 @@ func IsFatal() bool {
 	return GetLevel() <= FATAL
 }
 
+// Pause pauses all the loggers
+func Pause() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Pause()
+		logger.Flush()
+		return true
+	})
+}
+
+// Resume resumes all the loggers
+func Resume() {
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		logger.Resume()
+		return true
+	})
+}
+
+// ReleaseReopen releases and reopens logging files
+func ReleaseReopen() error {
+	var accumulatedErr error
+	NamedLoggers.Range(func(key, value interface{}) bool {
+		logger := value.(*Logger)
+		if err := logger.ReleaseReopen(); err != nil {
+			if accumulatedErr == nil {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v", key.(string), err)
+			} else {
+				accumulatedErr = fmt.Errorf("Error reopening %s: %v & %v", key.(string), err, accumulatedErr)
+			}
+		}
+		return true
+	})
+	return accumulatedErr
+}
+
 // Close closes all the loggers
 func Close() {
 	l, ok := NamedLoggers.Load(DEFAULT)

modules/log/smtp.go

@@ -97,6 +97,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
 func (log *SMTPLogger) Flush() {
 }
 
+// ReleaseReopen does nothing
+func (log *SMTPLogger) ReleaseReopen() error {
+	return nil
+}
+
 // GetName returns the default name for this implementation
 func (log *SMTPLogger) GetName() string {
 	return "smtp"

modules/private/manager.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -81,3 +82,110 @@ func FlushQueues(timeout time.Duration, nonBlocking bool) (int, string) {
 
 	return http.StatusOK, "Flushed"
 }
+
+// PauseLogging pauses logging
+func PauseLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/pause-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Paused"
+}
+
+// ResumeLogging resumes logging
+func ResumeLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/resume-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Restarted"
+}
+
+// ReleaseReopenLogging releases and reopens logging files
+func ReleaseReopenLogging() (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/release-and-reopen-logging"
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Logging Restarted"
+}
+
+// LoggerOptions represents the options for the add logger call
+type LoggerOptions struct {
+	Group  string
+	Name   string
+	Mode   string
+	Config map[string]interface{}
+}
+
+// AddLogger adds a logger
+func AddLogger(group, name, mode string, config map[string]interface{}) (int, string) {
+	reqURL := setting.LocalURL + "api/internal/manager/add-logger"
+
+	req := newInternalRequest(reqURL, "POST")
+	req = req.Header("Content-Type", "application/json")
+	jsonBytes, _ := json.Marshal(LoggerOptions{
+		Group:  group,
+		Name:   name,
+		Mode:   mode,
+		Config: config,
+	})
+	req.Body(jsonBytes)
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Added"
+
+}
+
+// RemoveLogger removes a logger
+func RemoveLogger(group, name string) (int, string) {
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(group), url.PathEscape(name))
+
+	req := newInternalRequest(reqURL, "POST")
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return resp.StatusCode, decodeJSONError(resp).Err
+	}
+
+	return http.StatusOK, "Removed"
+}

modules/setting/log.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/log"
 
@@ -20,6 +21,69 @@ import (
 
 var filenameSuffix = ""
 
+var descriptionLock = sync.RWMutex{}
+var logDescriptions = make(map[string]*LogDescription)
+
+// GetLogDescriptions returns a race safe set of descriptions
+func GetLogDescriptions() map[string]*LogDescription {
+	descriptionLock.RLock()
+	defer descriptionLock.RUnlock()
+	descs := make(map[string]*LogDescription, len(logDescriptions))
+	for k, v := range logDescriptions {
+		subLogDescriptions := make([]SubLogDescription, len(v.SubLogDescriptions))
+		for i, s := range v.SubLogDescriptions {
+			subLogDescriptions[i] = s
+		}
+		descs[k] = &LogDescription{
+			Name:               v.Name,
+			SubLogDescriptions: subLogDescriptions,
+		}
+	}
+	return descs
+}
+
+// AddLogDescription adds a set of descriptions to the complete description
+func AddLogDescription(key string, description *LogDescription) {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	logDescriptions[key] = description
+}
+
+// AddSubLogDescription adds a sub log description
+func AddSubLogDescription(key string, subLogDescription SubLogDescription) bool {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	desc, ok := logDescriptions[key]
+	if !ok {
+		return false
+	}
+	for i, sub := range desc.SubLogDescriptions {
+		if sub.Name == subLogDescription.Name {
+			desc.SubLogDescriptions[i] = subLogDescription
+			return true
+		}
+	}
+	desc.SubLogDescriptions = append(desc.SubLogDescriptions, subLogDescription)
+	return true
+}
+
+// RemoveSubLogDescription removes a sub log description
+func RemoveSubLogDescription(key string, name string) bool {
+	descriptionLock.Lock()
+	defer descriptionLock.Unlock()
+	desc, ok := logDescriptions[key]
+	if !ok {
+		return false
+	}
+	for i, sub := range desc.SubLogDescriptions {
+		if sub.Name == name {
+			desc.SubLogDescriptions = append(desc.SubLogDescriptions[:i], desc.SubLogDescriptions[i+1:]...)
+			return true
+		}
+	}
+	return false
+}
+
 type defaultLogOptions struct {
 	levelName      string // LogLevel
 	flags          string
@@ -185,7 +249,7 @@ func generateNamedLogger(key string, options defaultLogOptions) *LogDescription
 		log.Info("%s Log: %s(%s:%s)", strings.Title(key), strings.Title(name), provider, levelName)
 	}
 
-	LogDescriptions[key] = &description
+	AddLogDescription(key, &description)
 
 	return &description
 }
@@ -279,7 +343,7 @@ func newLogService() {
 		log.Info("Gitea Log Mode: %s(%s:%s)", strings.Title(name), strings.Title(provider), levelName)
 	}
 
-	LogDescriptions[log.DEFAULT] = &description
+	AddLogDescription(log.DEFAULT, &description)
 
 	// Finally redirect the default golog to here
 	golog.SetFlags(0)

modules/setting/setting.go

@@ -289,7 +289,6 @@ var (
 	LogLevel           string
 	StacktraceLogLevel string
 	LogRootPath        string
-	LogDescriptions    = make(map[string]*LogDescription)
 	RedirectMacaronLog bool
 	DisableRouterLog   bool
 	RouterLogLevel     log.Level