finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor doctor (#12264)

Browse source 
* Refactor Logger

Refactor Logger to make a logger interface and make it possible to
wrap loggers for specific purposes.

* Refactor Doctor

    Move the gitea doctor functions into its own module.
    Use a logger for its messages instead of returning a results string[]

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update modules/doctor/misc.go

Co-authored-by: 6543 <6543@obermui.de>

* Update modules/doctor/misc.go

Co-authored-by: 6543 <6543@obermui.de>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
zeripath 2020-12-02 04:56:04 +00:00 committed by GitHub
parent 253add883d
commit 4569339a4b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 787 additions and 522 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

558
cmd/doctor.go
View file

@ -5,27 +5,19 @@
package cmd package cmd
import ( import (
"bufio"
"bytes"
"context" "context"
"fmt" "fmt"
"io/ioutil"
golog "log" golog "log"
"os" "os"
"os/exec"
"path/filepath"
"strings" "strings"
"text/tabwriter" "text/tabwriter"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations" "code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/doctor"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"xorm.io/builder"
"xorm.io/xorm" "xorm.io/xorm"
"github.com/urfave/cli" "github.com/urfave/cli"
@ -62,6 +54,10 @@ var CmdDoctor = cli.Command{
Name: "log-file", Name: "log-file",
Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`, Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`,
}, },
cli.BoolFlag{
Name: "color, H",
Usage: "Use color for outputted information",
},
}, },
Subcommands: []cli.Command{ Subcommands: []cli.Command{
cmdRecreateTable, cmdRecreateTable,
@ -86,78 +82,6 @@ You should back-up your database before doing this and ensure that your database
Action: runRecreateTable, Action: runRecreateTable,
} }
type check struct {
title string
name string
isDefault bool
f func(ctx *cli.Context) ([]string, error)
abortIfFailed bool
skipDatabaseInit bool
}
// checklist represents list for all checks
var checklist = []check{
{
// NOTE: this check should be the first in the list
title: "Check paths and basic configuration",
name: "paths",
isDefault: true,
f: runDoctorPathInfo,
abortIfFailed: true,
skipDatabaseInit: true,
},
{
title: "Check Database Version",
name: "check-db-version",
isDefault: true,
f: runDoctorCheckDBVersion,
abortIfFailed: false,
},
{
title: "Check consistency of database",
name: "check-db-consistency",
isDefault: false,
f: runDoctorCheckDBConsistency,
},
{
title: "Check if OpenSSH authorized_keys file is up-to-date",
name: "authorized_keys",
isDefault: true,
f: runDoctorAuthorizedKeys,
},
{
title: "Check if SCRIPT_TYPE is available",
name: "script-type",
isDefault: false,
f: runDoctorScriptType,
},
{
title: "Check if hook files are up-to-date and executable",
name: "hooks",
isDefault: false,
f: runDoctorHooks,
},
{
title: "Recalculate merge bases",
name: "recalculate_merge_bases",
isDefault: false,
f: runDoctorPRMergeBase,
},
{
title: "Recalculate Stars number for all user",
name: "recalculate_stars_number",
isDefault: false,
f: runDoctorUserStarNum,
},
{
title: "Enable push options",
name: "enable-push-options",
isDefault: false,
f: runDoctorEnablePushOptions,
},
// more checks please append here
}
func runRecreateTable(ctx *cli.Context) error { func runRecreateTable(ctx *cli.Context) error {
// Redirect the default golog to here // Redirect the default golog to here
golog.SetFlags(0) golog.SetFlags(0)
@ -211,10 +135,15 @@ func runDoctor(ctx *cli.Context) error {
logFile = "doctor.log" logFile = "doctor.log"
} }
colorize := log.CanColorStdout
if ctx.IsSet("color") {
colorize = ctx.Bool("color")
}
if len(logFile) == 0 { if len(logFile) == 0 {
log.NewLogger(1000, "doctor", "console", `{"level":"NONE","stacktracelevel":"NONE","colorize":"%t"}`) log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize))
} else if logFile == "-" { } else if logFile == "-" {
log.NewLogger(1000, "doctor", "console", `{"level":"trace","stacktracelevel":"NONE"}`) log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"trace","stacktracelevel":"NONE","colorize":%t}`, colorize))
} else { } else {
log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile)) log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile))
} }
@ -225,24 +154,24 @@ func runDoctor(ctx *cli.Context) error {
golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
if ctx.IsSet("list") { if ctx.IsSet("list") {
w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
_, _ = w.Write([]byte("Default\tName\tTitle\n")) _, _ = w.Write([]byte("Default\tName\tTitle\n"))
for _, check := range checklist { for _, check := range doctor.Checks {
if check.isDefault { if check.IsDefault {
_, _ = w.Write([]byte{'*'}) _, _ = w.Write([]byte{'*'})
} }
_, _ = w.Write([]byte{'\t'}) _, _ = w.Write([]byte{'\t'})
_, _ = w.Write([]byte(check.name)) _, _ = w.Write([]byte(check.Name))
_, _ = w.Write([]byte{'\t'}) _, _ = w.Write([]byte{'\t'})
_, _ = w.Write([]byte(check.title)) _, _ = w.Write([]byte(check.Title))
_, _ = w.Write([]byte{'\n'}) _, _ = w.Write([]byte{'\n'})
} }
return w.Flush() return w.Flush()
} }
var checks []check var checks []*doctor.Check
if ctx.Bool("all") { if ctx.Bool("all") {
checks = checklist checks = doctor.Checks
} else if ctx.IsSet("run") { } else if ctx.IsSet("run") {
addDefault := ctx.Bool("default") addDefault := ctx.Bool("default")
names := ctx.StringSlice("run") names := ctx.StringSlice("run")
@ -250,452 +179,37 @@ func runDoctor(ctx *cli.Context) error {
names[i] = strings.ToLower(strings.TrimSpace(name)) names[i] = strings.ToLower(strings.TrimSpace(name))
} }
for _, check := range checklist { for _, check := range doctor.Checks {
if addDefault && check.isDefault { if addDefault && check.IsDefault {
checks = append(checks, check) checks = append(checks, check)
continue continue
} }
for _, name := range names { for _, name := range names {
if name == check.name { if name == check.Name {
checks = append(checks, check) checks = append(checks, check)
break break
} }
} }
} }
} else { } else {
for _, check := range checklist { for _, check := range doctor.Checks {
if check.isDefault { if check.IsDefault {
checks = append(checks, check) checks = append(checks, check)
} }
} }
} }
dbIsInit := false // Now we can set up our own logger to return information about what the doctor is doing
if err := log.NewNamedLogger("doctorouter",
for i, check := range checks { 1000,
if !dbIsInit && !check.skipDatabaseInit { "console",
// Only open database after the most basic configuration check "console",
setting.EnableXORMLog = false fmt.Sprintf(`{"level":"INFO","stacktracelevel":"NONE","colorize":%t,"flags":-1}`, colorize)); err != nil {
if err := initDBDisableConsole(true); err != nil { fmt.Println(err)
fmt.Println(err)
fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.")
return nil
}
dbIsInit = true
}
fmt.Println("[", i+1, "]", check.title)
messages, err := check.f(ctx)
for _, message := range messages {
fmt.Println("-", message)
}
if err != nil {
fmt.Println("Error:", err)
if check.abortIfFailed {
return nil
}
} else {
fmt.Println("OK.")
}
fmt.Println()
}
return nil
}
func runDoctorPathInfo(ctx *cli.Context) ([]string, error) {
res := make([]string, 0, 10)
if fi, err := os.Stat(setting.CustomConf); err != nil || !fi.Mode().IsRegular() {
res = append(res, fmt.Sprintf("Failed to find configuration file at '%s'.", setting.CustomConf))
res = append(res, fmt.Sprintf("If you've never ran Gitea yet, this is normal and '%s' will be created for you on first run.", setting.CustomConf))
res = append(res, "Otherwise check that you are running this command from the correct path and/or provide a `--config` parameter.")
return res, fmt.Errorf("can't proceed without a configuration file")
}
setting.NewContext()
fail := false
check := func(name, path string, is_dir, required, is_write bool) {
res = append(res, fmt.Sprintf("%-25s '%s'", name+":", path))
fi, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) && ctx.Bool("fix") && is_dir {
if err := os.MkdirAll(path, 0777); err != nil {
res = append(res, fmt.Sprintf(" ERROR: %v", err))
fail = true
return
}
fi, err = os.Stat(path)
}
}
if err != nil {
if required {
res = append(res, fmt.Sprintf(" ERROR: %v", err))
fail = true
return
}
res = append(res, fmt.Sprintf(" NOTICE: not accessible (%v)", err))
return
}
if is_dir && !fi.IsDir() {
res = append(res, " ERROR: not a directory")
fail = true
return
} else if !is_dir && !fi.Mode().IsRegular() {
res = append(res, " ERROR: not a regular file")
fail = true
} else if is_write {
if err := runDoctorWritableDir(path); err != nil {
res = append(res, fmt.Sprintf(" ERROR: not writable: %v", err))
fail = true
}
}
}
// Note print paths inside quotes to make any leading/trailing spaces evident
check("Configuration File Path", setting.CustomConf, false, true, false)
check("Repository Root Path", setting.RepoRootPath, true, true, true)
check("Data Root Path", setting.AppDataPath, true, true, true)
check("Custom File Root Path", setting.CustomPath, true, false, false)
check("Work directory", setting.AppWorkPath, true, true, false)
check("Log Root Path", setting.LogRootPath, true, true, true)
if options.IsDynamic() {
// Do not check/report on StaticRootPath if data is embedded in Gitea (-tags bindata)
check("Static File Root Path", setting.StaticRootPath, true, true, false)
}
if fail {
return res, fmt.Errorf("please check your configuration file and try again")
}
return res, nil
}
func runDoctorWritableDir(path string) error {
// There's no platform-independent way of checking if a directory is writable
// https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable
tmpFile, err := ioutil.TempFile(path, "doctors-order")
if err != nil {
return err return err
} }
if err := util.Remove(tmpFile.Name()); err != nil {
fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) logger := log.GetLogger("doctorouter")
} defer logger.Close()
tmpFile.Close() return doctor.RunChecks(logger, ctx.Bool("fix"), checks)
return nil
}
const tplCommentPrefix = `# gitea public key`
func runDoctorAuthorizedKeys(ctx *cli.Context) ([]string, error) {
if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile {
return nil, nil
}
fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys")
f, err := os.Open(fPath)
if err != nil {
if ctx.Bool("fix") {
return []string{fmt.Sprintf("Error whilst opening authorized_keys: %v. Attempting regeneration", err)}, models.RewriteAllPublicKeys()
}
return nil, err
}
defer f.Close()
linesInAuthorizedKeys := map[string]bool{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
linesInAuthorizedKeys[line] = true
}
f.Close()
// now we regenerate and check if there are any lines missing
regenerated := &bytes.Buffer{}
if err := models.RegeneratePublicKeys(regenerated); err != nil {
return nil, err
}
scanner = bufio.NewScanner(regenerated)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
if ok := linesInAuthorizedKeys[line]; ok {
continue
}
if ctx.Bool("fix") {
return []string{"authorized_keys is out of date, attempting regeneration"}, models.RewriteAllPublicKeys()
}
return nil, fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized_keys --fix"`)
}
return nil, nil
}
func runDoctorCheckDBVersion(ctx *cli.Context) ([]string, error) {
if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
if ctx.Bool("fix") {
return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate)
}
return nil, err
}
return nil, nil
}
func iterateRepositories(each func(*models.Repository) ([]string, error)) ([]string, error) {
results := []string{}
err := models.Iterate(
models.DefaultDBContext(),
new(models.Repository),
builder.Gt{"id": 0},
func(idx int, bean interface{}) error {
res, err := each(bean.(*models.Repository))
results = append(results, res...)
return err
},
)
return results, err
}
func iteratePRs(repo *models.Repository, each func(*models.Repository, *models.PullRequest) ([]string, error)) ([]string, error) {
results := []string{}
err := models.Iterate(
models.DefaultDBContext(),
new(models.PullRequest),
builder.Eq{"base_repo_id": repo.ID},
func(idx int, bean interface{}) error {
res, err := each(repo, bean.(*models.PullRequest))
results = append(results, res...)
return err
},
)
return results, err
}
func runDoctorHooks(ctx *cli.Context) ([]string, error) {
// Need to iterate across all of the repositories
return iterateRepositories(func(repo *models.Repository) ([]string, error) {
results, err := repository.CheckDelegateHooks(repo.RepoPath())
if err != nil {
return nil, err
}
if len(results) > 0 && ctx.Bool("fix") {
return []string{fmt.Sprintf("regenerated hooks for %s", repo.FullName())}, repository.CreateDelegateHooks(repo.RepoPath())
}
return results, nil
})
}
func runDoctorPRMergeBase(ctx *cli.Context) ([]string, error) {
numRepos := 0
numPRs := 0
numPRsUpdated := 0
results, err := iterateRepositories(func(repo *models.Repository) ([]string, error) {
numRepos++
return iteratePRs(repo, func(repo *models.Repository, pr *models.PullRequest) ([]string, error) {
numPRs++
results := []string{}
pr.BaseRepo = repo
repoPath := repo.RepoPath()
oldMergeBase := pr.MergeBase
if !pr.HasMerged {
var err error
pr.MergeBase, err = git.NewCommand("merge-base", "--", pr.BaseBranch, pr.GetGitRefName()).RunInDir(repoPath)
if err != nil {
var err2 error
pr.MergeBase, err2 = git.NewCommand("rev-parse", git.BranchPrefix+pr.BaseBranch).RunInDir(repoPath)
if err2 != nil {
results = append(results, fmt.Sprintf("WARN: Unable to get merge base for PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name))
log.Error("Unable to get merge base for PR ID %d, Index %d in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2)
return results, nil
}
}
} else {
parentsString, err := git.NewCommand("rev-list", "--parents", "-n", "1", pr.MergedCommitID).RunInDir(repoPath)
if err != nil {
results = append(results, fmt.Sprintf("WARN: Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name))
log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return results, nil
}
parents := strings.Split(strings.TrimSpace(parentsString), " ")
if len(parents) < 2 {
return results, nil
}
args := append([]string{"merge-base", "--"}, parents[1:]...)
args = append(args, pr.GetGitRefName())
pr.MergeBase, err = git.NewCommand(args...).RunInDir(repoPath)
if err != nil {
results = append(results, fmt.Sprintf("WARN: Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name))
log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return results, nil
}
}
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
if pr.MergeBase != oldMergeBase {
if ctx.Bool("fix") {
if err := pr.UpdateCols("merge_base"); err != nil {
return results, err
}
} else {
results = append(results, fmt.Sprintf("#%d onto %s in %s/%s: MergeBase should be %s but is %s", pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, oldMergeBase, pr.MergeBase))
}
numPRsUpdated++
}
return results, nil
})
})
if ctx.Bool("fix") {
results = append(results, fmt.Sprintf("%d PR mergebases updated of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos))
} else {
if numPRsUpdated > 0 && err == nil {
return results, fmt.Errorf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
}
results = append(results, fmt.Sprintf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos))
}
return results, err
}
func runDoctorUserStarNum(ctx *cli.Context) ([]string, error) {
return nil, models.DoctorUserStarNum()
}
func runDoctorScriptType(ctx *cli.Context) ([]string, error) {
path, err := exec.LookPath(setting.ScriptType)
if err != nil {
return []string{fmt.Sprintf("ScriptType %s is not on the current PATH", setting.ScriptType)}, err
}
return []string{fmt.Sprintf("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)}, nil
}
func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
var results []string
// make sure DB version is uptodate
if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
return nil, fmt.Errorf("model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
}
//find labels without existing repo or org
count, err := models.CountOrphanedLabels()
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
if err = models.DeleteOrphanedLabels(); err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d labels without existing repository/organisation deleted", count))
} else {
results = append(results, fmt.Sprintf("%d labels without existing repository/organisation", count))
}
}
//find issues without existing repository
count, err = models.CountOrphanedIssues()
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
if err = models.DeleteOrphanedIssues(); err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count))
} else {
results = append(results, fmt.Sprintf("%d issues without existing repository", count))
}
}
//find pulls without existing issues
count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count))
} else {
results = append(results, fmt.Sprintf("%d pull requests without existing issue", count))
}
}
//find tracked times without existing issues/pulls
count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id")
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count))
} else {
results = append(results, fmt.Sprintf("%d tracked times without existing issue", count))
}
}
count, err = models.CountNullArchivedRepository()
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
updatedCount, err := models.FixNullArchivedRepository()
if err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d repositories with null is_archived updated", updatedCount))
} else {
results = append(results, fmt.Sprintf("%d repositories with null is_archived", count))
}
}
//ToDo: function to recalc all counters
return results, nil
}
func runDoctorEnablePushOptions(ctx *cli.Context) ([]string, error) {
numRepos := 0
_, err := iterateRepositories(func(repo *models.Repository) ([]string, error) {
numRepos++
r, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return nil, err
}
defer r.Close()
if ctx.Bool("fix") {
_, err := git.NewCommand("config", "receive.advertisePushOptions", "true").RunInDir(r.Path)
return nil, err
}
return nil, nil
})
var prefix string
if !ctx.Bool("fix") {
prefix = "DRY RUN: "
}
return []string{fmt.Sprintf("%sEnabled push options for %d repositories.", prefix, numRepos)}, err
} }

95
modules/doctor/authorizedkeys.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"bufio"
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
const tplCommentPrefix = `# gitea public key`
func checkAuthorizedKeys(logger log.Logger, autofix bool) error {
if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile {
return nil
}
fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys")
f, err := os.Open(fPath)
if err != nil {
if !autofix {
logger.Critical("Unable to open authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to open authorized_keys file. ERROR: %v", err)
}
logger.Warn("Unable to open authorized_keys. (ERROR: %v). Attempting to rewrite...", err)
if err = models.RewriteAllPublicKeys(); err != nil {
logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err)
}
}
defer f.Close()
linesInAuthorizedKeys := map[string]bool{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
linesInAuthorizedKeys[line] = true
}
f.Close()
// now we regenerate and check if there are any lines missing
regenerated := &bytes.Buffer{}
if err := models.RegeneratePublicKeys(regenerated); err != nil {
logger.Critical("Unable to regenerate authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to regenerate authorized_keys file. ERROR: %v", err)
}
scanner = bufio.NewScanner(regenerated)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
if ok := linesInAuthorizedKeys[line]; ok {
continue
}
if !autofix {
logger.Critical(
"authorized_keys file %q is out of date.\nRegenerate it with:\n\t\"%s\"\nor\n\t\"%s\"",
fPath,
"gitea admin regenerate keys",
"gitea doctor --run authorized_keys --fix")
return fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized_keys --fix"`)
}
logger.Warn("authorized_keys is out of date. Attempting rewrite...")
err = models.RewriteAllPublicKeys()
if err != nil {
logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err)
}
}
return nil
}
func init() {
Register(&Check{
Title: "Check if OpenSSH authorized_keys file is up-to-date",
Name: "authorized-keys",
IsDefault: true,
Run: checkAuthorizedKeys,
Priority: 4,
})
}

127
modules/doctor/dbconsistency.go Normal file
View file

@ -0,0 +1,127 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"context"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
)
func checkDBConsistency(logger log.Logger, autofix bool) error {
// make sure DB version is uptodate
if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
return err
}
// find labels without existing repo or org
count, err := models.CountOrphanedLabels()
if err != nil {
logger.Critical("Error: %v whilst counting orphaned labels")
return err
}
if count > 0 {
if autofix {
if err = models.DeleteOrphanedLabels(); err != nil {
logger.Critical("Error: %v whilst deleting orphaned labels")
return err
}
logger.Info("%d labels without existing repository/organisation deleted", count)
} else {
logger.Warn("%d labels without existing repository/organisation", count)
}
}
// find issues without existing repository
count, err = models.CountOrphanedIssues()
if err != nil {
logger.Critical("Error: %v whilst counting orphaned issues")
return err
}
if count > 0 {
if autofix {
if err = models.DeleteOrphanedIssues(); err != nil {
logger.Critical("Error: %v whilst deleting orphaned issues")
return err
}
logger.Info("%d issues without existing repository deleted", count)
} else {
logger.Warn("%d issues without existing repository", count)
}
}
// find pulls without existing issues
count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
if err != nil {
logger.Critical("Error: %v whilst counting orphaned objects")
return err
}
if count > 0 {
if autofix {
if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil {
logger.Critical("Error: %v whilst deleting orphaned objects")
return err
}
logger.Info("%d pull requests without existing issue deleted", count)
} else {
logger.Warn("%d pull requests without existing issue", count)
}
}
// find tracked times without existing issues/pulls
count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id")
if err != nil {
logger.Critical("Error: %v whilst counting orphaned objects")
return err
}
if count > 0 {
if autofix {
if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil {
logger.Critical("Error: %v whilst deleting orphaned objects")
return err
}
logger.Info("%d tracked times without existing issue deleted", count)
} else {
logger.Warn("%d tracked times without existing issue", count)
}
}
// find null archived repositories
count, err = models.CountNullArchivedRepository()
if err != nil {
logger.Critical("Error: %v whilst counting null archived repositories")
return err
}
if count > 0 {
if autofix {
updatedCount, err := models.FixNullArchivedRepository()
if err != nil {
logger.Critical("Error: %v whilst fixing null archived repositories")
return err
}
logger.Info("%d repositories with null is_archived updated", updatedCount)
} else {
logger.Warn("%d repositories with null is_archived", count)
}
}
// TODO: function to recalc all counters
return nil
}
func init() {
Register(&Check{
Title: "Check consistency of database",
Name: "check-db-consistency",
IsDefault: false,
Run: checkDBConsistency,
Priority: 3,
})
}

42
modules/doctor/dbversion.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"context"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
)
func checkDBVersion(logger log.Logger, autofix bool) error {
if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
if !autofix {
logger.Critical("Error: %v during ensure up to date", err)
return err
}
logger.Warn("Got Error: %v during ensure up to date", err)
logger.Warn("Attempting to migrate to the latest DB version to fix this.")
err = models.NewEngine(context.Background(), migrations.Migrate)
if err != nil {
logger.Critical("Error: %v during migration")
}
return err
}
return nil
}
func init() {
Register(&Check{
Title: "Check Database Version",
Name: "check-db-version",
IsDefault: true,
Run: checkDBVersion,
AbortIfFailed: false,
Priority: 2,
})
}

105
modules/doctor/doctor.go Normal file
View file

@ -0,0 +1,105 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"fmt"
"sort"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// Check represents a Doctor check
type Check struct {
Title string
Name string
IsDefault bool
Run func(logger log.Logger, autofix bool) error
AbortIfFailed bool
SkipDatabaseInitialization bool
Priority int
}
type wrappedLevelLogger struct {
log.LevelLogger
}
func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...interface{}) error {
return w.LevelLogger.Log(
skip+1,
level,
" - %s "+format,
append(
[]interface{}{
log.NewColoredValueBytes(
fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])),
level.Color()),
}, v...)...)
}
func initDBDisableConsole(disableConsole bool) error {
setting.NewContext()
setting.InitDBConfig()
setting.NewXORMLogService(disableConsole)
if err := models.SetEngine(); err != nil {
return fmt.Errorf("models.SetEngine: %v", err)
}
return nil
}
// Checks is the list of available commands
var Checks []*Check
// RunChecks runs the doctor checks for the provided list
func RunChecks(logger log.Logger, autofix bool, checks []*Check) error {
wrappedLogger := log.LevelLoggerLogger{
LevelLogger: &wrappedLevelLogger{logger},
}
dbIsInit := false
for i, check := range checks {
if !dbIsInit && !check.SkipDatabaseInitialization {
// Only open database after the most basic configuration check
setting.EnableXORMLog = false
if err := initDBDisableConsole(true); err != nil {
logger.Error("Error whilst initializing the database: %v", err)
logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
return nil
}
dbIsInit = true
}
logger.Info("[%d] %s", log.NewColoredIDValue(i+1), check.Title)
logger.Flush()
if err := check.Run(&wrappedLogger, autofix); err != nil {
if check.AbortIfFailed {
logger.Critical("FAIL")
return err
}
logger.Error("ERROR")
} else {
logger.Info("OK")
logger.Flush()
}
}
return nil
}
// Register registers a command with the list
func Register(command *Check) {
Checks = append(Checks, command)
sort.SliceStable(Checks, func(i, j int) bool {
if Checks[i].Priority == Checks[j].Priority {
return Checks[i].Name < Checks[j].Name
}
if Checks[i].Priority == 0 {
return false
}
return Checks[i].Priority < Checks[j].Priority
})
}

110
modules/doctor/mergebase.go Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"fmt"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func iteratePRs(repo *models.Repository, each func(*models.Repository, *models.PullRequest) error) error {
return models.Iterate(
models.DefaultDBContext(),
new(models.PullRequest),
builder.Eq{"base_repo_id": repo.ID},
func(idx int, bean interface{}) error {
return each(repo, bean.(*models.PullRequest))
},
)
}
func checkPRMergeBase(logger log.Logger, autofix bool) error {
numRepos := 0
numPRs := 0
numPRsUpdated := 0
err := iterateRepositories(func(repo *models.Repository) error {
numRepos++
return iteratePRs(repo, func(repo *models.Repository, pr *models.PullRequest) error {
numPRs++
pr.BaseRepo = repo
repoPath := repo.RepoPath()
oldMergeBase := pr.MergeBase
if !pr.HasMerged {
var err error
pr.MergeBase, err = git.NewCommand("merge-base", "--", pr.BaseBranch, pr.GetGitRefName()).RunInDir(repoPath)
if err != nil {
var err2 error
pr.MergeBase, err2 = git.NewCommand("rev-parse", git.BranchPrefix+pr.BaseBranch).RunInDir(repoPath)
if err2 != nil {
logger.Warn("Unable to get merge base for PR ID %d, #%d onto %s in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2)
return nil
}
}
} else {
parentsString, err := git.NewCommand("rev-list", "--parents", "-n", "1", pr.MergedCommitID).RunInDir(repoPath)
if err != nil {
logger.Warn("Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return nil
}
parents := strings.Split(strings.TrimSpace(parentsString), " ")
if len(parents) < 2 {
return nil
}
args := append([]string{"merge-base", "--"}, parents[1:]...)
args = append(args, pr.GetGitRefName())
pr.MergeBase, err = git.NewCommand(args...).RunInDir(repoPath)
if err != nil {
logger.Warn("Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return nil
}
}
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
if pr.MergeBase != oldMergeBase {
if autofix {
if err := pr.UpdateCols("merge_base"); err != nil {
logger.Critical("Failed to update merge_base. ERROR: %v", err)
return fmt.Errorf("Failed to update merge_base. ERROR: %v", err)
}
} else {
logger.Info("#%d onto %s in %s/%s: MergeBase should be %s but is %s", pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, oldMergeBase, pr.MergeBase)
}
numPRsUpdated++
}
return nil
})
})
if autofix {
logger.Info("%d PR mergebases updated of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
} else {
if numPRsUpdated > 0 && err == nil {
logger.Critical("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
return fmt.Errorf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
}
logger.Warn("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
}
return err
}
func init() {
Register(&Check{
Title: "Recalculate merge bases",
Name: "recalculate-merge-bases",
IsDefault: false,
Run: checkPRMergeBase,
Priority: 7,
})
}

146
modules/doctor/misc.go Normal file
View file

@ -0,0 +1,146 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"fmt"
"os/exec"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"xorm.io/builder"
)
func iterateRepositories(each func(*models.Repository) error) error {
err := models.Iterate(
models.DefaultDBContext(),
new(models.Repository),
builder.Gt{"id": 0},
func(idx int, bean interface{}) error {
return each(bean.(*models.Repository))
},
)
return err
}
func checkScriptType(logger log.Logger, autofix bool) error {
path, err := exec.LookPath(setting.ScriptType)
if err != nil {
logger.Critical("ScriptType \"%q\" is not on the current PATH. Error: %v", setting.ScriptType, err)
return fmt.Errorf("ScriptType \"%q\" is not on the current PATH. Error: %v", setting.ScriptType, err)
}
logger.Info("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)
return nil
}
func checkHooks(logger log.Logger, autofix bool) error {
if err := iterateRepositories(func(repo *models.Repository) error {
results, err := repository.CheckDelegateHooks(repo.RepoPath())
if err != nil {
logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err)
return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err)
}
if len(results) > 0 && autofix {
logger.Warn("Regenerated hooks for %s", repo.FullName())
if err := repository.CreateDelegateHooks(repo.RepoPath()); err != nil {
logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err)
return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err)
}
}
for _, result := range results {
logger.Warn(result)
}
return nil
}); err != nil {
logger.Critical("Errors noted whilst checking delegate hooks.")
return err
}
return nil
}
func checkUserStarNum(logger log.Logger, autofix bool) error {
if err := models.DoctorUserStarNum(); err != nil {
logger.Critical("Unable update User Stars numbers")
return err
}
return nil
}
func checkEnablePushOptions(logger log.Logger, autofix bool) error {
numRepos := 0
numNeedUpdate := 0
if err := iterateRepositories(func(repo *models.Repository) error {
numRepos++
r, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return err
}
defer r.Close()
if autofix {
_, err := git.NewCommand("config", "receive.advertisePushOptions", "true").RunInDir(r.Path)
return err
}
value, err := git.NewCommand("config", "receive.advertisePushOptions").RunInDir(r.Path)
if err != nil {
return err
}
result, valid := git.ParseBool(strings.TrimSpace(value))
if !result || !valid {
numNeedUpdate++
logger.Info("%s: does not have receive.advertisePushOptions set correctly: %q", repo.FullName(), value)
}
return nil
}); err != nil {
logger.Critical("Unable to EnablePushOptions: %v", err)
return err
}
if autofix {
logger.Info("Enabled push options for %d repositories.", numRepos)
} else {
logger.Info("Checked %d repositories, %d need updates.", numRepos, numNeedUpdate)
}
return nil
}
func init() {
Register(&Check{
Title: "Check if SCRIPT_TYPE is available",
Name: "script-type",
IsDefault: false,
Run: checkScriptType,
Priority: 5,
})
Register(&Check{
Title: "Check if hook files are up-to-date and executable",
Name: "hooks",
IsDefault: false,
Run: checkHooks,
Priority: 6,
})
Register(&Check{
Title: "Recalculate Stars number for all user",
Name: "recalculate-stars-number",
IsDefault: false,
Run: checkUserStarNum,
Priority: 6,
})
Register(&Check{
Title: "Enable push options",
Name: "enable-push-options",
IsDefault: false,
Run: checkEnablePushOptions,
Priority: 7,
})
}

126
modules/doctor/paths.go Normal file
View file

@ -0,0 +1,126 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"fmt"
"io/ioutil"
"os"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
)
type configurationFile struct {
Name string
Path string
IsDirectory bool
Required bool
Writable bool
}
func checkConfigurationFile(logger log.Logger, autofix bool, fileOpts configurationFile) error {
logger.Info(`%-26s %q`, log.NewColoredValue(fileOpts.Name+":", log.Reset), fileOpts.Path)
fi, err := os.Stat(fileOpts.Path)
if err != nil {
if os.IsNotExist(err) && autofix && fileOpts.IsDirectory {
if err := os.MkdirAll(fileOpts.Path, 0777); err != nil {
logger.Error(" Directory does not exist and could not be created. ERROR: %v", err)
return fmt.Errorf("Configuration directory: \"%q\" does not exist and could not be created. ERROR: %v", fileOpts.Path, err)
}
fi, err = os.Stat(fileOpts.Path)
}
}
if err != nil {
if fileOpts.Required {
logger.Error(" Is REQUIRED but is not accessible. ERROR: %v", err)
return fmt.Errorf("Configuration file \"%q\" is not accessible but is required. Error: %v", fileOpts.Path, err)
}
logger.Warn(" NOTICE: is not accessible (Error: %v)", err)
// this is a non-critical error
return nil
}
if fileOpts.IsDirectory && !fi.IsDir() {
logger.Error(" ERROR: not a directory")
return fmt.Errorf("Configuration directory \"%q\" is not a directory. Error: %v", fileOpts.Path, err)
} else if !fileOpts.IsDirectory && !fi.Mode().IsRegular() {
logger.Error(" ERROR: not a regular file")
return fmt.Errorf("Configuration file \"%q\" is not a regular file. Error: %v", fileOpts.Path, err)
} else if fileOpts.Writable {
if err := isWritableDir(fileOpts.Path); err != nil {
logger.Error(" ERROR: is required to be writable but is not writable: %v", err)
return fmt.Errorf("Configuration file \"%q\" is required to be writable but is not. Error: %v", fileOpts.Path, err)
}
}
return nil
}
func checkConfigurationFiles(logger log.Logger, autofix bool) error {
if fi, err := os.Stat(setting.CustomConf); err != nil || !fi.Mode().IsRegular() {
logger.Error("Failed to find configuration file at '%s'.", setting.CustomConf)
logger.Error("If you've never ran Gitea yet, this is normal and '%s' will be created for you on first run.", setting.CustomConf)
logger.Error("Otherwise check that you are running this command from the correct path and/or provide a `--config` parameter.")
logger.Critical("Cannot proceed without a configuration file")
return err
}
setting.NewContext()
configurationFiles := []configurationFile{
{"Configuration File Path", setting.CustomConf, false, true, false},
{"Repository Root Path", setting.RepoRootPath, true, true, true},
{"Data Root Path", setting.AppDataPath, true, true, true},
{"Custom File Root Path", setting.CustomPath, true, false, false},
{"Work directory", setting.AppWorkPath, true, true, false},
{"Log Root Path", setting.LogRootPath, true, true, true},
}
if options.IsDynamic() {
configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false})
}
numberOfErrors := 0
for _, configurationFile := range configurationFiles {
if err := checkConfigurationFile(logger, autofix, configurationFile); err != nil {
numberOfErrors++
}
}
if numberOfErrors > 0 {
logger.Critical("Please check your configuration files and try again.")
return fmt.Errorf("%d configuration files with errors", numberOfErrors)
}
return nil
}
func isWritableDir(path string) error {
// There's no platform-independent way of checking if a directory is writable
// https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable
tmpFile, err := ioutil.TempFile(path, "doctors-order")
if err != nil {
return err
}
if err := os.Remove(tmpFile.Name()); err != nil {
fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name())
}
tmpFile.Close()
return nil
}
func init() {
Register(&Check{
Title: "Check paths and basic configuration",
Name: "paths",
IsDefault: true,
Run: checkConfigurationFiles,
AbortIfFailed: true,
SkipDatabaseInitialization: true,
Priority: 1,
})
}