Improve install code to avoid low-level mistakes. (#17779)

* Improve install code to avoid low-level mistakes.

If a user tries to do a re-install in a Gitea database, they gets a warning and double check.
When Gitea runs, it never create empty app.ini automatically.

Also some small (related) refactoring:

* Refactor db.InitEngine related logic make it more clean (especially for the install code)
* Move some i18n strings out from setting.go to make the setting.go can be easily maintained.
* Show errors in CLI code if an incorrect app.ini is used.
* APP_DATA_PATH is created when installing, and checked when starting (no empty directory is created any more).

cmd/cmd.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -57,15 +58,17 @@ func confirm() (bool, error) {
 }
 
 func initDB(ctx context.Context) error {
-	return initDBDisableConsole(ctx, false)
+	setting.LoadFromExisting()
-}
-
-func initDBDisableConsole(ctx context.Context, disableConsole bool) error {
-	setting.NewContext()
 	setting.InitDBConfig()
-	setting.NewXORMLogService(disableConsole)
+	setting.NewXORMLogService(false)
+
+	if setting.Database.Type == "" {
+		log.Fatal(`Database settings are missing from the configuration file: %q.
+Ensure you are running in the correct environment or set the correct configuration file with -c.
+If this is the intended configuration file complete the [database] section.`, setting.CustomConf)
+	}
 	if err := db.InitEngine(ctx); err != nil {
-		return fmt.Errorf("models.SetEngine: %v", err)
+		return fmt.Errorf("unable to initialise the database using the configuration in %q. Error: %v", setting.CustomConf, err)
 	}
 	return nil
 }

cmd/convert.go

@@ -35,7 +35,6 @@ func runConvert(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if !setting.Database.UseMySQL {
 		fmt.Println("This command can only be used with a MySQL database")

cmd/doctor.go

@@ -87,7 +87,7 @@ func runRecreateTable(ctx *cli.Context) error {
 	golog.SetPrefix("")
 	golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
 
-	setting.NewContext()
+	setting.LoadFromExisting()
 	setting.InitDBConfig()
 
 	setting.EnableXORMLog = ctx.Bool("debug")

cmd/dump.go

@@ -159,7 +159,8 @@ func runDump(ctx *cli.Context) error {
 			fatal("Deleting default logger failed. Can not write to stdout: %v", err)
 		}
 	}
-	setting.NewContext()
+	setting.LoadFromExisting()
+
 	// make sure we are logging to the console no matter what the configuration tells us do to
 	if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
 		fatal("Setting logging mode to console failed: %v", err)

cmd/dump_repo.go

@@ -88,7 +88,6 @@ func runDumpRepository(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	var (
 		serviceType structs.GitServiceType

cmd/embedded.go

@@ -115,7 +115,7 @@ func initEmbeddedExtractor(c *cli.Context) error {
 	log.DelNamedLogger(log.DEFAULT)
 
 	// Read configuration file
-	setting.NewContext()
+	setting.LoadAllowEmpty()
 
 	pats, err := getPatterns(c.Args())
 	if err != nil {

cmd/mailer.go

@@ -18,7 +18,7 @@ func runSendMail(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.NewContext()
+	setting.LoadFromExisting()
 
 	if err := argsSet(c, "title"); err != nil {
 		return err

cmd/migrate.go

@@ -36,7 +36,6 @@ func runMigrate(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
 		log.Fatal("Failed to initialize ORM engine: %v", err)

cmd/migrate_storage.go

@@ -121,7 +121,6 @@ func runMigrateStorage(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
 		log.Fatal("Failed to initialize ORM engine: %v", err)

cmd/restore_repo.go

@@ -50,7 +50,7 @@ func runRestoreRepository(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.NewContext()
+	setting.LoadFromExisting()
 
 	statusCode, errStr := private.RestoreRepo(
 		ctx,

cmd/serv.go

@@ -58,7 +58,7 @@ func setup(logPath string, debug bool) {
 	} else {
 		_ = log.NewLogger(1000, "console", "console", `{"level":"fatal","stacktracelevel":"NONE","stderr":true}`)
 	}
-	setting.NewContext()
+	setting.LoadFromExisting()
 	if debug {
 		setting.RunMode = "dev"
 	}

cmd/web.go

@@ -124,6 +124,10 @@ func runWeb(ctx *cli.Context) error {
 		}
 		c := install.Routes()
 		err := listen(c, false)
+		if err != nil {
+			log.Critical("Unable to open listener for installer. Is Gitea already running?")
+			graceful.GetManager().DoGracefulShutdown()
+		}
 		select {
 		case <-graceful.GetManager().IsShutdown():
 			<-graceful.GetManager().Done()
@@ -145,7 +149,15 @@ func runWeb(ctx *cli.Context) error {
 
 	log.Info("Global init")
 	// Perform global initialization
-	routers.GlobalInit(graceful.GetManager().HammerContext())
+	setting.LoadFromExisting()
+	routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
+
+	// We check that AppDataPath exists here (it should have been created during installation)
+	// We can't check it in `GlobalInitInstalled`, because some integration tests
+	// use cmd -> GlobalInitInstalled, but the AppDataPath doesn't exist during those tests.
+	if _, err := os.Stat(setting.AppDataPath); err != nil {
+		log.Fatal("Can not find APP_DATA_PATH '%s'", setting.AppDataPath)
+	}
 
 	// Override the provided port number within the configuration
 	if ctx.IsSet("port") {

contrib/environment-to-ini/environment-to-ini.go

@@ -156,6 +156,7 @@ func runEnvironmentToIni(c *cli.Context) error {
 		destination = setting.CustomConf
 	}
 	if destination != setting.CustomConf || changed {
+		log.Info("Settings saved to: %q", destination)
 		err = cfg.SaveTo(destination)
 		if err != nil {
 			return err

contrib/pr/checkout.go

@@ -49,7 +49,7 @@ func runPR() {
 		log.Fatal(err)
 	}
 	setting.SetCustomPathAndConf("", "", "")
-	setting.NewContext()
+	setting.LoadAllowEmpty()
 
 	setting.RepoRootPath, err = os.MkdirTemp(os.TempDir(), "repos")
 	if err != nil {

integrations/integration_test.go

@@ -164,8 +164,8 @@ func initIntegrationTest() {
 	}
 
 	setting.SetCustomPathAndConf("", "", "")
-	setting.NewContext()
+	setting.LoadForTest()
-	util.RemoveAll(models.LocalCopyPath())
+	_ = util.RemoveAll(models.LocalCopyPath())
 	git.CheckLFSVersion()
 	setting.InitDBConfig()
 	if err := storage.Init(); err != nil {
@@ -240,7 +240,8 @@ func initIntegrationTest() {
 		}
 		defer db.Close()
 	}
-	routers.GlobalInit(graceful.GetManager().HammerContext())
+
+	routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
 }
 
 func prepareTestEnv(t testing.TB, skip ...int) func() {
@@ -254,6 +255,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 
 	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+
 	return deferFn
 }
 

integrations/migration-test/migration_test.go

@@ -56,7 +56,7 @@ func initMigrationTest(t *testing.T) func() {
 		setting.CustomConf = giteaConf
 	}
 
-	setting.NewContext()
+	setting.LoadForTest()
 
 	assert.True(t, len(setting.RepoRootPath) != 0)
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))

models/db/engine.go

@@ -8,7 +8,6 @@ package db
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"io"
 	"reflect"
@@ -92,8 +91,8 @@ func init() {
 	}
 }
 
-// NewEngine returns a new xorm engine from the configuration
+// newXORMEngine returns a new XORM engine from the configuration
-func NewEngine() (*xorm.Engine, error) {
+func newXORMEngine() (*xorm.Engine, error) {
 	connStr, err := setting.DBConnStr()
 	if err != nil {
 		return nil, err
@@ -126,40 +125,49 @@ func SyncAllTables() error {
 	return x.StoreEngine("InnoDB").Sync2(tables...)
 }
 
-// InitEngine sets the xorm.Engine
+// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext
-func InitEngine(ctx context.Context) (err error) {
+func InitEngine(ctx context.Context) error {
-	x, err = NewEngine()
+	xormEngine, err := newXORMEngine()
 	if err != nil {
-		return fmt.Errorf("Failed to connect to database: %v", err)
+		return fmt.Errorf("failed to connect to database: %v", err)
 	}
 
-	x.SetMapper(names.GonicMapper{})
+	xormEngine.SetMapper(names.GonicMapper{})
 	// WARNING: for serv command, MUST remove the output to os.stdout,
 	// so use log file to instead print to stdout.
-	x.SetLogger(NewXORMLogger(setting.Database.LogSQL))
+	xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL))
-	x.ShowSQL(setting.Database.LogSQL)
+	xormEngine.ShowSQL(setting.Database.LogSQL)
-	x.SetMaxOpenConns(setting.Database.MaxOpenConns)
+	xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns)
-	x.SetMaxIdleConns(setting.Database.MaxIdleConns)
+	xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns)
-	x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	xormEngine.SetDefaultContext(ctx)
 
+	SetDefaultEngine(ctx, xormEngine)
+	return nil
+}
+
+// SetDefaultEngine sets the default engine for db
+func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) {
+	x = eng
 	DefaultContext = &Context{
 		Context: ctx,
 		e:       x,
 	}
-	x.SetDefaultContext(ctx)
-	return nil
 }
 
-// SetEngine is used by unit test code
+// UnsetDefaultEngine closes and unsets the default engine
-func SetEngine(eng *xorm.Engine) {
+// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now,
-	x = eng
+// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close
-	DefaultContext = &Context{
+// Global database engine related functions are all racy and there is no graceful close right now.
-		Context: context.Background(),
+func UnsetDefaultEngine() {
-		e:       x,
+	if x != nil {
+		_ = x.Close()
+		x = nil
 	}
+	DefaultContext = nil
 }
 
-// InitEngineWithMigration initializes a new xorm.Engine
+// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext
 // This function must never call .Sync2() if the provided migration function fails.
 // When called from the "doctor" command, the migration function is a version check
 // that prevents the doctor from fixing anything in the database if the migration level
@@ -226,14 +234,6 @@ func NamesToBean(names ...string) ([]interface{}, error) {
 	return beans, nil
 }
 
-// Ping tests if database is alive
-func Ping() error {
-	if x != nil {
-		return x.Ping()
-	}
-	return errors.New("database not configured")
-}
-
 // DumpDatabase dumps all data from database according the special database SQL syntax to file system.
 func DumpDatabase(filePath, dbType string) error {
 	var tbs []*schemas.Table
@@ -291,11 +291,3 @@ func GetMaxID(beanOrTableName interface{}) (maxID int64, err error) {
 	_, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID)
 	return
 }
-
-// FindByMaxID filled results as the condition from database
-func FindByMaxID(maxID int64, limit int, results interface{}) error {
-	return x.Where("id <= ?", maxID).
-		OrderBy("id DESC").
-		Limit(limit).
-		Find(results)
-}

models/db/install/db.go

@@ -0,0 +1,65 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package install
+
+import (
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func getXORMEngine() *xorm.Engine {
+	return db.DefaultContext.(*db.Context).Engine().(*xorm.Engine)
+}
+
+// CheckDatabaseConnection checks the database connection
+func CheckDatabaseConnection() error {
+	e := db.GetEngine(db.DefaultContext)
+	_, err := e.Exec("SELECT 1")
+	return err
+}
+
+// GetMigrationVersion gets the database migration version
+func GetMigrationVersion() (int64, error) {
+	var installedDbVersion int64
+	x := getXORMEngine()
+	exist, err := x.IsTableExist("version")
+	if err != nil {
+		return 0, err
+	}
+	if !exist {
+		return 0, nil
+	}
+	_, err = x.Table("version").Cols("version").Get(&installedDbVersion)
+	if err != nil {
+		return 0, err
+	}
+	return installedDbVersion, nil
+}
+
+// HasPostInstallationUsers checks whether there are users after installation
+func HasPostInstallationUsers() (bool, error) {
+	x := getXORMEngine()
+	exist, err := x.IsTableExist("user")
+	if err != nil {
+		return false, err
+	}
+	if !exist {
+		return false, nil
+	}
+
+	// if there are 2 or more users in database, we consider there are users created after installation
+	threshold := 2
+	if !setting.IsProd {
+		// to debug easily, with non-prod RUN_MODE, we only check the count to 1
+		threshold = 1
+	}
+	res, err := x.Table("user").Cols("id").Limit(threshold).Query()
+	if err != nil {
+		return false, err
+	}
+	return len(res) >= threshold, nil
+}

models/migrations/migrations_test.go

@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"os"
@@ -57,7 +58,7 @@ func TestMain(m *testing.M) {
 	}
 
 	setting.SetCustomPathAndConf("", "", "")
-	setting.NewContext()
+	setting.LoadForTest()
 	git.CheckLFSVersion()
 	setting.InitDBConfig()
 	setting.NewLogServices(true)
@@ -85,21 +86,11 @@ func removeAllWithRetry(dir string) error {
 	return err
 }
 
-// newEngine sets the xorm.Engine
+func newXORMEngine() (*xorm.Engine, error) {
-func newEngine() (*xorm.Engine, error) {
+	if err := db.InitEngine(context.Background()); err != nil {
-	x, err := db.NewEngine()
+		return nil, err
-	if err != nil {
-		return x, fmt.Errorf("Failed to connect to database: %v", err)
 	}
-
+	x := unittest.GetXORMEngine()
-	x.SetMapper(names.GonicMapper{})
-	// WARNING: for serv command, MUST remove the output to os.stdout,
-	// so use log file to instead print to stdout.
-	x.SetLogger(db.NewXORMLogger(setting.Database.LogSQL))
-	x.ShowSQL(setting.Database.LogSQL)
-	x.SetMaxOpenConns(setting.Database.MaxOpenConns)
-	x.SetMaxIdleConns(setting.Database.MaxIdleConns)
-	x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
 	return x, nil
 }
 
@@ -213,7 +204,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 		return nil, deferFn
 	}
 
-	x, err := newEngine()
+	x, err := newXORMEngine()
 	assert.NoError(t, err)
 	if x != nil {
 		oldDefer := deferFn

models/ssh_key_test.go

@@ -16,7 +16,7 @@ import (
 
 func init() {
 	setting.SetCustomPathAndConf("", "", "")
-	setting.NewContext()
+	setting.LoadForTest()
 }
 
 func Test_SSHParsePublicKey(t *testing.T) {

models/unittest/fixtures.go

@@ -18,7 +18,8 @@ import (
 
 var fixtures *testfixtures.Loader
 
-func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
+// GetXORMEngine gets the XORM engine
+func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 	if len(engine) == 1 {
 		return engine[0]
 	}
@@ -27,7 +28,7 @@ func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 
 // InitFixtures initialize test fixtures for a test database
 func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
-	e := getXORMEngine(engine...)
+	e := GetXORMEngine(engine...)
 	var testfiles func(*testfixtures.Loader) error
 	if opts.Dir != "" {
 		testfiles = testfixtures.Directory(opts.Dir)
@@ -69,7 +70,7 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 
 // LoadFixtures load fixtures for a test database
 func LoadFixtures(engine ...*xorm.Engine) error {
-	e := getXORMEngine(engine...)
+	e := GetXORMEngine(engine...)
 	var err error
 	// Database transaction conflicts could occur and result in ROLLBACK
 	// As a simple workaround, we just retry 20 times.

models/unittest/testdb.go

@@ -5,6 +5,7 @@
 package unittest
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
@@ -124,7 +125,7 @@ func CreateTestEngine(opts FixturesOptions) error {
 		return err
 	}
 	x.SetMapper(names.GonicMapper{})
-	db.SetEngine(x)
+	db.SetDefaultEngine(context.Background(), x)
 
 	if err = db.SyncAllTables(); err != nil {
 		return err

modules/doctor/doctor.go

@@ -44,7 +44,7 @@ func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...
 }
 
 func initDBDisableConsole(ctx context.Context, disableConsole bool) error {
-	setting.NewContext()
+	setting.LoadFromExisting()
 	setting.InitDBConfig()
 
 	setting.NewXORMLogService(disableConsole)

modules/doctor/paths.go

@@ -67,7 +67,7 @@ func checkConfigurationFiles(logger log.Logger, autofix bool) error {
 		return err
 	}
 
-	setting.NewContext()
+	setting.LoadFromExisting()
 
 	configurationFiles := []configurationFile{
 		{"Configuration File Path", setting.CustomConf, false, true, false},

modules/private/internal.go

@@ -13,14 +13,18 @@ import (
 
 	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 )
 
 func newRequest(ctx context.Context, url, method string) *httplib.Request {
+	if setting.InternalToken == "" {
+		log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q.
+Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf)
+	}
 	return httplib.NewRequest(url, method).
 		SetContext(ctx).
-		Header("Authorization",
+		Header("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken))
-			fmt.Sprintf("Bearer %s", setting.InternalToken))
 }
 
 // Response internal request response
@@ -44,9 +48,6 @@ func newInternalRequest(ctx context.Context, url, method string) *httplib.Reques
 	})
 	if setting.Protocol == setting.UnixSocket {
 		req.SetTransport(&http.Transport{
-			Dial: func(_, _ string) (net.Conn, error) {
-				return net.Dial("unix", setting.HTTPAddr)
-			},
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 				var d net.Dialer
 				return d.DialContext(ctx, "unix", setting.HTTPAddr)

modules/setting/directory.go

@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"fmt"
+	"os"
+)
+
+// PrepareAppDataPath creates app data directory if necessary
+func PrepareAppDataPath() error {
+	// FIXME: There are too many calls to MkdirAll in old code. It is incorrect.
+	// For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs,
+	// then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem.
+	// The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories.
+	// For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK).
+	// Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future.
+
+	st, err := os.Stat(AppDataPath)
+
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(AppDataPath, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("unable to create the APP_DATA_PATH directory: %q, Error: %v", AppDataPath, err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("unable to use APP_DATA_PATH %q. Error: %v", AppDataPath, err)
+	}
+
+	if !st.IsDir() /* also works for symlink */ {
+		return fmt.Errorf("the APP_DATA_PATH %q is not a directory (or symlink to a directory) and can't be used", AppDataPath)
+	}
+
+	return nil
+}

modules/setting/i18n.go

@@ -0,0 +1,51 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+// defaultI18nLangNames must be a slice, we need the order
+var defaultI18nLangNames = []string{
+	"en-US", "English",
+	"zh-CN", "简体中文",
+	"zh-HK", "繁體中文（香港）",
+	"zh-TW", "繁體中文（台灣）",
+	"de-DE", "Deutsch",
+	"fr-FR", "français",
+	"nl-NL", "Nederlands",
+	"lv-LV", "latviešu",
+	"ru-RU", "русский",
+	"uk-UA", "Українська",
+	"ja-JP", "日本語",
+	"es-ES", "español",
+	"pt-BR", "português do Brasil",
+	"pt-PT", "Português de Portugal",
+	"pl-PL", "polski",
+	"bg-BG", "български",
+	"it-IT", "italiano",
+	"fi-FI", "suomi",
+	"tr-TR", "Türkçe",
+	"cs-CZ", "čeština",
+	"sr-SP", "српски",
+	"sv-SE", "svenska",
+	"ko-KR", "한국어",
+	"el-GR", "ελληνικά",
+	"fa-IR", "فارسی",
+	"hu-HU", "magyar nyelv",
+	"id-ID", "bahasa Indonesia",
+	"ml-IN", "മലയാളം",
+}
+
+func defaultI18nLangs() (res []string) {
+	for i := 0; i < len(defaultI18nLangNames); i += 2 {
+		res = append(res, defaultI18nLangNames[i])
+	}
+	return
+}
+
+func defaultI18nNames() (res []string) {
+	for i := 0; i < len(defaultI18nLangNames); i += 2 {
+		res = append(res, defaultI18nLangNames[i+1])
+	}
+	return
+}

modules/setting/setting.go

@@ -546,9 +546,27 @@ func SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath string)
 	}
 }
 
-// NewContext initializes configuration context.
+// LoadFromExisting initializes setting options from an existing config file (app.ini)
+func LoadFromExisting() {
+	loadFromConf(false)
+}
+
+// LoadAllowEmpty initializes setting options, it's also fine that if the config file (app.ini) doesn't exist
+func LoadAllowEmpty() {
+	loadFromConf(true)
+}
+
+// LoadForTest initializes setting options for tests
+func LoadForTest() {
+	loadFromConf(true)
+	if err := PrepareAppDataPath(); err != nil {
+		log.Fatal("Can not prepare APP_DATA_PATH: %v", err)
+	}
+}
+
+// loadFromConf initializes configuration context.
 // NOTE: do not print any log except error.
-func NewContext() {
+func loadFromConf(allowEmpty bool) {
 	Cfg = ini.Empty()
 
 	if WritePIDFile && len(PIDFile) > 0 {
@@ -563,9 +581,10 @@ func NewContext() {
 		if err := Cfg.Append(CustomConf); err != nil {
 			log.Fatal("Failed to load custom conf '%s': %v", CustomConf, err)
 		}
-	} else {
+	} else if !allowEmpty {
-		log.Warn("Custom config '%s' not found, ignore this if you're running first time", CustomConf)
+		log.Fatal("Unable to find configuration file: %q.\nEnsure you are running in the correct environment or set the correct configuration file with -c.", CustomConf)
-	}
+	} // else: no config file, a config file might be created at CustomConf later (might not)
+
 	Cfg.NameMapper = ini.SnackCase
 
 	homeDir, err := com.HomeDir()
@@ -698,18 +717,7 @@ func NewContext() {
 	StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(StaticRootPath)
 	StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour)
 	AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data"))
-	if _, err = os.Stat(AppDataPath); err != nil {
+
-		// FIXME: There are too many calls to MkdirAll in old code. It is incorrect.
-		// For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs,
-		// then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem.
-		// The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories.
-		// For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK).
-		// Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future.
-		err = os.MkdirAll(AppDataPath, os.ModePerm)
-		if err != nil {
-			log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath)
-		}
-	}
 	EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
 	EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
 	PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof"))
@@ -864,6 +872,10 @@ func NewContext() {
 	SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20)
 
 	InternalToken = loadInternalToken(sec)
+	if InstallLock && InternalToken == "" {
+		// if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate
+		generateSaveInternalToken()
+	}
 
 	cfgdata := sec.Key("PASSWORD_COMPLEXITY").Strings(",")
 	if len(cfgdata) == 0 {
@@ -975,19 +987,11 @@ func NewContext() {
 
 	Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
 	if len(Langs) == 0 {
-		Langs = []string{
+		Langs = defaultI18nLangs()
-			"en-US", "zh-CN", "zh-HK", "zh-TW", "de-DE", "fr-FR", "nl-NL", "lv-LV",
-			"ru-RU", "uk-UA", "ja-JP", "es-ES", "pt-BR", "pt-PT", "pl-PL", "bg-BG",
-			"it-IT", "fi-FI", "tr-TR", "cs-CZ", "sr-SP", "sv-SE", "ko-KR", "el-GR",
-			"fa-IR", "hu-HU", "id-ID", "ml-IN"}
 	}
 	Names = Cfg.Section("i18n").Key("NAMES").Strings(",")
 	if len(Names) == 0 {
-		Names = []string{"English", "简体中文", "繁體中文（香港）", "繁體中文（台灣）", "Deutsch",
+		Names = defaultI18nNames()
-			"français", "Nederlands", "latviešu", "русский", "Українська", "日本語",
-			"español", "português do Brasil", "Português de Portugal", "polski", "български",
-			"italiano", "suomi", "Türkçe", "čeština", "српски", "svenska", "한국어", "ελληνικά",
-			"فارسی", "magyar nyelv", "bahasa Indonesia", "മലയാളം"}
 	}
 
 	ShowFooterBranding = Cfg.Section("other").Key("SHOW_FOOTER_BRANDING").MustBool(false)
@@ -1054,8 +1058,8 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) {
 
 func loadInternalToken(sec *ini.Section) string {
 	uri := sec.Key("INTERNAL_TOKEN_URI").String()
-	if len(uri) == 0 {
+	if uri == "" {
-		return loadOrGenerateInternalToken(sec)
+		return sec.Key("INTERNAL_TOKEN").String()
 	}
 	tempURI, err := url.Parse(uri)
 	if err != nil {
@@ -1092,21 +1096,17 @@ func loadInternalToken(sec *ini.Section) string {
 	return ""
 }
 
-func loadOrGenerateInternalToken(sec *ini.Section) string {
+// generateSaveInternalToken generates and saves the internal token to app.ini
-	var err error
+func generateSaveInternalToken() {
-	token := sec.Key("INTERNAL_TOKEN").String()
+	token, err := generate.NewInternalToken()
-	if len(token) == 0 {
-		token, err = generate.NewInternalToken()
 	if err != nil {
 		log.Fatal("Error generate internal token: %v", err)
 	}
 
-		// Save secret
+	InternalToken = token
 	CreateOrAppendToCustomConf(func(cfg *ini.File) {
 		cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token)
 	})
-	}
-	return token
 }
 
 // MakeAbsoluteAssetURL returns the absolute asset url prefix without a trailing slash
@@ -1186,6 +1186,8 @@ func CreateOrAppendToCustomConf(callback func(cfg *ini.File)) {
 
 	callback(cfg)
 
+	log.Info("Settings saved to: %q", CustomConf)
+
 	if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
 		log.Fatal("failed to create '%s': %v", CustomConf, err)
 		return

options/locale/locale_en-US.ini

@@ -138,6 +138,11 @@ ssl_mode = SSL
 charset = Charset
 path = Path
 sqlite_helper = File path for the SQLite3 database.<br>Enter an absolute path if you run Gitea as a service.
+reinstall_error = You are trying to install into an existing Gitea database
+reinstall_confirm_message = Re-installing with an existing Gitea database can cause multiple problems. In most cases, you should use your existing "app.ini" to run Gitea. If you know what you are doing, confirm the following:
+reinstall_confirm_check_1 = The data encrypted by the SECRET_KEY in app.ini may be lost: users may not be able to log in with 2FA/OTP & mirrors may not function correctly. By checking this box you confirm that the current app.ini file contains the correct the SECRET_KEY.
+reinstall_confirm_check_2 = The repositories and settings may need to be re-synchronized. By checking this box you confirm that you will resynchronize the hooks for the repositories and authorized_keys file manually. You confirm that you will ensure that repository and mirror settings are correct.
+reinstall_confirm_check_3 = You confirm that you are absolutely sure that this Gitea is running with the correct app.ini location and that you are sure that you have have to re-install. You confirm that you acknowledge the above risks.
 err_empty_db_path = The SQLite3 database path cannot be empty.
 no_admin_and_disable_registration = You cannot disable user self-registration without creating an administrator account.
 err_empty_admin_password = The administrator password cannot be empty.
@@ -203,8 +208,12 @@ install_btn_confirm = Install Gitea
 test_git_failed = Could not test 'git' command: %v
 sqlite3_not_available = This Gitea version does not support SQLite3. Please download the official binary version from %s (not the 'gobuild' version).
 invalid_db_setting = The database settings are invalid: %v
+invalid_db_table = The database table '%s' is invalid: %v
 invalid_repo_path = The repository root path is invalid: %v
+invalid_app_data_path = The app data path is invalid: %v
 run_user_not_match = The 'run as' username is not the current username: %s -> %s
+internal_token_failed = Failed to generate internal token: %v
+secret_key_failed = Failed to generate secret key: %v
 save_config_failed = Failed to save configuration: %v
 invalid_admin_setting = Administrator account setting is invalid: %v
 install_success = Welcome! Thank you for choosing Gitea. Have fun and take care!

routers/init.go

@@ -95,9 +95,8 @@ func syncAppPathForGit(ctx context.Context) error {
 	return nil
 }
 
-// GlobalInit is for global configuration reload-able.
+// GlobalInitInstalled is for global installed configuration.
-func GlobalInit(ctx context.Context) {
+func GlobalInitInstalled(ctx context.Context) {
-	setting.NewContext()
 	if !setting.InstallLock {
 		log.Fatal("Gitea is not installed")
 	}

routers/install/install.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models/db"
+	db_install "code.gitea.io/gitea/models/db/install"
 	"code.gitea.io/gitea/models/migrations"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
@@ -161,10 +162,81 @@ func Install(ctx *context.Context) {
 	ctx.HTML(http.StatusOK, tplInstall)
 }
 
+func checkDatabase(ctx *context.Context, form *forms.InstallForm) bool {
+	var err error
+
+	if (setting.Database.Type == "sqlite3") &&
+		len(setting.Database.Path) == 0 {
+		ctx.Data["Err_DbPath"] = true
+		ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, form)
+		return false
+	}
+
+	// Check if the user is trying to re-install in an installed database
+	db.UnsetDefaultEngine()
+	defer db.UnsetDefaultEngine()
+
+	if err = db.InitEngine(ctx); err != nil {
+		if strings.Contains(err.Error(), `Unknown database type: sqlite3`) {
+			ctx.Data["Err_DbType"] = true
+			ctx.RenderWithErr(ctx.Tr("install.sqlite3_not_available", "https://docs.gitea.io/en-us/install-from-binary/"), tplInstall, form)
+		} else {
+			ctx.Data["Err_DbSetting"] = true
+			ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, form)
+		}
+		return false
+	}
+
+	err = db_install.CheckDatabaseConnection()
+	if err != nil {
+		ctx.Data["Err_DbSetting"] = true
+		ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, form)
+		return false
+	}
+
+	hasPostInstallationUser, err := db_install.HasPostInstallationUsers()
+	if err != nil {
+		ctx.Data["Err_DbSetting"] = true
+		ctx.RenderWithErr(ctx.Tr("install.invalid_db_table", "user", err), tplInstall, form)
+		return false
+	}
+	dbMigrationVersion, err := db_install.GetMigrationVersion()
+	if err != nil {
+		ctx.Data["Err_DbSetting"] = true
+		ctx.RenderWithErr(ctx.Tr("install.invalid_db_table", "version", err), tplInstall, form)
+		return false
+	}
+
+	if hasPostInstallationUser && dbMigrationVersion > 0 {
+		log.Error("The database is likely to have been used by Gitea before, database migration version=%d", dbMigrationVersion)
+		confirmed := form.ReinstallConfirmFirst && form.ReinstallConfirmSecond && form.ReinstallConfirmThird
+		if !confirmed {
+			ctx.Data["Err_DbInstalledBefore"] = true
+			ctx.RenderWithErr(ctx.Tr("install.reinstall_error"), tplInstall, form)
+			return false
+		}
+
+		log.Info("User confirmed reinstallation of Gitea into a pre-existing database")
+	}
+
+	if hasPostInstallationUser || dbMigrationVersion > 0 {
+		log.Info("Gitea will be installed in a database with: hasPostInstallationUser=%v, dbMigrationVersion=%v", hasPostInstallationUser, dbMigrationVersion)
+	}
+
+	return true
+}
+
 // SubmitInstall response for submit install items
 func SubmitInstall(ctx *context.Context) {
-	form := *web.GetForm(ctx).(*forms.InstallForm)
 	var err error
+
+	form := *web.GetForm(ctx).(*forms.InstallForm)
+
+	// fix form values
+	if form.AppURL != "" && form.AppURL[len(form.AppURL)-1] != '/' {
+		form.AppURL += "/"
+	}
+
 	ctx.Data["CurDbOption"] = form.DbType
 
 	if ctx.HasError() {
@@ -186,9 +258,9 @@ func SubmitInstall(ctx *context.Context) {
 		return
 	}
 
-	// Pass basic check, now test configuration.
+	// ---- Basic checks are passed, now test configuration.
-	// Test database setting.
 
+	// Test database setting.
 	setting.Database.Type = setting.GetDBTypeByName(form.DbType)
 	setting.Database.Host = form.DbHost
 	setting.Database.User = form.DbUser
@@ -201,22 +273,13 @@ func SubmitInstall(ctx *context.Context) {
 	setting.Database.LogSQL = !setting.IsProd
 	setting.PasswordHashAlgo = form.PasswordAlgorithm
 
-	if (setting.Database.Type == "sqlite3") &&
+	if !checkDatabase(ctx, &form) {
-		len(setting.Database.Path) == 0 {
-		ctx.Data["Err_DbPath"] = true
-		ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, &form)
 		return
 	}
 
-	// Set test engine.
+	// Prepare AppDataPath, it is very important for Gitea
-	if err = db.InitEngineWithMigration(ctx, migrations.Migrate); err != nil {
+	if err = setting.PrepareAppDataPath(); err != nil {
-		if strings.Contains(err.Error(), `Unknown database type: sqlite3`) {
+		ctx.RenderWithErr(ctx.Tr("install.invalid_app_data_path", err), tplInstall, &form)
-			ctx.Data["Err_DbType"] = true
-			ctx.RenderWithErr(ctx.Tr("install.sqlite3_not_available", "https://docs.gitea.io/en-us/install-from-binary/"), tplInstall, &form)
-		} else {
-			ctx.Data["Err_DbSetting"] = true
-			ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, &form)
-		}
 		return
 	}
 
@@ -299,9 +362,14 @@ func SubmitInstall(ctx *context.Context) {
 		}
 	}
 
-	if form.AppURL[len(form.AppURL)-1] != '/' {
+	// Init the engine with migration
-		form.AppURL += "/"
+	if err = db.InitEngineWithMigration(ctx, migrations.Migrate); err != nil {
+		db.UnsetDefaultEngine()
+		ctx.Data["Err_DbSetting"] = true
+		ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, &form)
+		return
 	}
+	db.UnsetDefaultEngine()
 
 	// Save settings.
 	cfg := ini.Empty()
@@ -344,12 +412,12 @@ func SubmitInstall(ctx *context.Context) {
 	if form.LFSRootPath != "" {
 		cfg.Section("server").Key("LFS_START_SERVER").SetValue("true")
 		cfg.Section("server").Key("LFS_CONTENT_PATH").SetValue(form.LFSRootPath)
-		var secretKey string
+		var lfsJwtSecret string
-		if secretKey, err = generate.NewJwtSecretBase64(); err != nil {
+		if lfsJwtSecret, err = generate.NewJwtSecretBase64(); err != nil {
 			ctx.RenderWithErr(ctx.Tr("install.lfs_jwt_secret_failed", err), tplInstall, &form)
 			return
 		}
-		cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(secretKey)
+		cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(lfsJwtSecret)
 	} else {
 		cfg.Section("server").Key("LFS_START_SERVER").SetValue("false")
 	}
@@ -390,16 +458,30 @@ func SubmitInstall(ctx *context.Context) {
 	cfg.Section("log").Key("ROUTER").SetValue("console")
 
 	cfg.Section("security").Key("INSTALL_LOCK").SetValue("true")
+
+	var internalToken string
+	if internalToken, err = generate.NewInternalToken(); err != nil {
+		ctx.RenderWithErr(ctx.Tr("install.internal_token_failed", err), tplInstall, &form)
+		return
+	}
+	cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(internalToken)
+
+	// if there is already a SECRET_KEY, we should not overwrite it, otherwise the encrypted data will not be able to be decrypted
+	if setting.SecretKey == "" {
 		var secretKey string
 		if secretKey, err = generate.NewSecretKey(); err != nil {
 			ctx.RenderWithErr(ctx.Tr("install.secret_key_failed", err), tplInstall, &form)
 			return
 		}
 		cfg.Section("security").Key("SECRET_KEY").SetValue(secretKey)
+	}
+
 	if len(form.PasswordAlgorithm) > 0 {
 		cfg.Section("security").Key("PASSWORD_HASH_ALGO").SetValue(form.PasswordAlgorithm)
 	}
 
+	log.Info("Save settings to custom config file %s", setting.CustomConf)
+
 	err = os.MkdirAll(filepath.Dir(setting.CustomConf), os.ModePerm)
 	if err != nil {
 		ctx.RenderWithErr(ctx.Tr("install.save_config_failed", err), tplInstall, &form)
@@ -411,8 +493,10 @@ func SubmitInstall(ctx *context.Context) {
 		return
 	}
 
-	// Re-read settings
+	// ---- All checks are passed
-	ReloadSettings(ctx)
+
+	// Reload settings (and re-initialize database connection)
+	reloadSettings(ctx)
 
 	// Create admin account
 	if len(form.AdminName) > 0 {

routers/install/setting.go

@@ -16,17 +16,17 @@ import (
 
 // PreloadSettings preloads the configuration to check if we need to run install
 func PreloadSettings(ctx context.Context) bool {
-	setting.NewContext()
+	setting.LoadAllowEmpty()
 	if !setting.InstallLock {
 		log.Info("AppPath: %s", setting.AppPath)
 		log.Info("AppWorkPath: %s", setting.AppWorkPath)
 		log.Info("Custom path: %s", setting.CustomPath)
 		log.Info("Log path: %s", setting.LogRootPath)
 		log.Info("Configuration file: %s", setting.CustomConf)
-		log.Info("Preparing to run install page")
+		log.Info("Prepare to run install page")
 		translation.InitLocales()
 		if setting.EnableSQLite3 {
-			log.Info("SQLite3 Supported")
+			log.Info("SQLite3 is supported")
 		}
 		setting.InitDBConfig()
 		setting.NewServicesForInstall()
@@ -36,9 +36,9 @@ func PreloadSettings(ctx context.Context) bool {
 	return !setting.InstallLock
 }
 
-// ReloadSettings rereads the settings and starts up the database
+// reloadSettings reloads the existing settings and starts up the database
-func ReloadSettings(ctx context.Context) {
+func reloadSettings(ctx context.Context) {
-	setting.NewContext()
+	setting.LoadFromExisting()
 	setting.InitDBConfig()
 	if setting.InstallLock {
 		if err := common.InitDBEngine(ctx); err == nil {

services/forms/user_form.go

@@ -67,6 +67,11 @@ type InstallForm struct {
 	AdminPasswd        string `binding:"OmitEmpty;MaxSize(255)" locale:"install.admin_password"`
 	AdminConfirmPasswd string
 	AdminEmail         string `binding:"OmitEmpty;MinSize(3);MaxSize(254);Include(@)" locale:"install.admin_email"`
+
+	// ReinstallConfirmFirst we can not use 1/2/3 or A/B/C here, there is a framework bug, can not parse "reinstall_confirm_1" or "reinstall_confirm_a"
+	ReinstallConfirmFirst  bool
+	ReinstallConfirmSecond bool
+	ReinstallConfirmThird  bool
 }
 
 // Validate validates the fields

templates/base/head.tmpl

@@ -51,6 +51,8 @@
 				copy_error: '{{.i18n.Tr "copy_error"}}',
 			}
 		};
+		{{/* in case some pages don't render the pageData, we make sure it is an object to prevent null access */}}
+		window.config.pageData = window.config.pageData || {};
 	</script>
 	<link rel="icon" href="{{AssetUrlPrefix}}/img/logo.svg" type="image/svg+xml">
 	<link rel="alternate icon" href="{{AssetUrlPrefix}}/img/favicon.png" type="image/png">

templates/install.tmpl

@@ -91,6 +91,30 @@
 						</div>
 					</div>
 
+					{{if .Err_DbInstalledBefore}}
+					<div>
+						<p class="reinstall-message">{{.i18n.Tr "install.reinstall_confirm_message"}}</p>
+						<div class="reinstall-confirm">
+							<div class="ui checkbox">
+								<label>{{.i18n.Tr "install.reinstall_confirm_check_1"}}</label>
+								<input name="reinstall_confirm_first" type="checkbox">
+							</div>
+						</div>
+						<div class="reinstall-confirm">
+							<div class="ui checkbox">
+								<label>{{.i18n.Tr "install.reinstall_confirm_check_2"}}</label>
+								<input name="reinstall_confirm_second" type="checkbox">
+							</div>
+						</div>
+						<div class="reinstall-confirm">
+							<div class="ui checkbox">
+								<label>{{.i18n.Tr "install.reinstall_confirm_check_3"}}</label>
+								<input name="reinstall_confirm_third" type="checkbox">
+							</div>
+						</div>
+					</div>
+					{{end}}
+
 					<!-- General Settings -->
 					<h4 class="ui dividing header">{{.i18n.Tr "install.general_title"}}</h4>
 					<div class="inline required field {{if .Err_AppName}}error{{end}}">

web_src/less/_install.less

@@ -4,7 +4,7 @@
   form {
     @input-padding: 320px !important;
 
-    label {
+    .inline.field label {
       text-align: right;
       width: @input-padding;
     }
@@ -20,19 +20,32 @@
         margin-left: @input-padding+15px;
       }
 
-      &.optional .title {
+      &.optional {
+        .title {
           margin-left: 38%;
         }
-    }
-  }
-
-  .ui {
         .checkbox {
           margin-left: 40% !important;
-
           label {
             width: auto !important;
           }
         }
       }
+    }
+  }
+
+  .ui {
+    .reinstall-message {
+      width: 70%;
+      margin: 20px auto;
+      color: red;
+      text-align: left;
+      font-weight: bold;
+    }
+    .reinstall-confirm {
+      width: 70%;
+      text-align: left;
+      margin: 10px auto;
+    }
+  }
 }