finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move database settings from models to setting (#7806)

Browse source 
* move database settings from models to setting

* update docs

* fix checkout pr

* fix tests

* fix lint

* remove unsupported tidb options

* correct wrong variable name

* remove tidb totally
This commit is contained in:
Lunny Xiao 2019-08-24 17:24:45 +08:00 • committed by Antoine GIRARD
parent 26af3401c3
commit f83db078f0
35 changed files with 423 additions and 376 deletions
Download patch file Download diff file Expand all files Collapse all files

8
models/convert.go
View file

@ -4,11 +4,15 @@
package models
import "fmt"
import (
"fmt"
"code.gitea.io/gitea/modules/setting"
)
// ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql
func ConvertUtf8ToUtf8mb4() error {
_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", DbCfg.Name))
_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", setting.Database.Name))
if err != nil {
return err
}

8
models/migrations/migrations.go
View file

@ -296,7 +296,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
// TODO: This will not work if there are foreign keys
switch {
case setting.UseSQLite3:
case setting.Database.UseSQLite3:
// First drop the indexes on the columns
res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName))
if errIndex != nil {
@ -372,7 +372,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
return err
}
case setting.UsePostgreSQL:
case setting.Database.UsePostgreSQL:
cols := ""
for _, col := range columnNames {
if cols != "" {
@ -383,7 +383,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
}
case setting.UseMySQL, setting.UseTiDB:
case setting.Database.UseMySQL:
// Drop indexes on columns first
sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','"))
res, err := sess.Query(sql)
@ -409,7 +409,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
}
case setting.UseMSSQL:
case setting.Database.UseMSSQL:
cols := ""
for _, col := range columnNames {
if cols != "" {

2
models/migrations/v19.go
View file

@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
}
)
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
user := new(User)

2
models/migrations/v22.go
View file

@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
}
)
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
user := new(User)

2
models/migrations/v26.go
View file

@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
)
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
user := new(User)

2
models/migrations/v27.go
View file

@ -41,8 +41,6 @@ func convertIntervalToDuration(x *xorm.Engine) (err error) {
_, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT")
case "postgres":
_, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" SET DATA TYPE bigint")
case "tidb":
_, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT")
case "mssql":
_, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" BIGINT")
case "sqlite3":

4
models/migrations/v33.go
View file

@ -15,9 +15,9 @@ import (
func removeActionColumns(x *xorm.Engine) error {
switch {
case setting.UseSQLite3:
case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB:
case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE action DROP COLUMN act_user_name"); err != nil {
return fmt.Errorf("DROP COLUMN act_user_name: %v", err)
} else if _, err = x.Exec("ALTER TABLE action DROP COLUMN repo_user_name"); err != nil {

4
models/migrations/v45.go
View file

@ -13,9 +13,9 @@ import (
func removeIndexColumnFromRepoUnitTable(x *xorm.Engine) (err error) {
switch {
case setting.UseSQLite3:
case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB:
case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE repo_unit DROP COLUMN `index`"); err != nil {
// Ignoring this error in case we run this migration second time (after migration reordering)
log.Warn("DROP COLUMN index: %v", err)

4
models/migrations/v50.go
View file

@ -40,9 +40,9 @@ func migrateProtectedBranchStruct(x *xorm.Engine) error {
}
switch {
case setting.UseSQLite3:
case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB:
case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE protected_branch DROP COLUMN can_push"); err != nil {
// Ignoring this error in case we run this migration second time (after migration reordering)
log.Warn("DROP COLUMN can_push (skipping): %v", err)

4
models/migrations/v67.go
View file

@ -80,7 +80,7 @@ func removeStaleWatches(x *xorm.Engine) error {
}
repoCache := make(map[int64]*Repository)
err := sess.BufferSize(setting.IterateBufferSize).Iterate(new(Watch),
err := sess.BufferSize(setting.Database.IterateBufferSize).Iterate(new(Watch),
func(idx int, bean interface{}) error {
watch := bean.(*Watch)
@ -117,7 +117,7 @@ func removeStaleWatches(x *xorm.Engine) error {
}
repoCache = make(map[int64]*Repository)
err = sess.BufferSize(setting.IterateBufferSize).
err = sess.BufferSize(setting.Database.IterateBufferSize).
Distinct("issue_watch.user_id", "issue.repo_id").
Join("INNER", "issue", "issue_watch.issue_id = issue.id").
Where("issue_watch.is_watching = ?", true).

2
models/migrations/v81.go
View file

@ -14,8 +14,6 @@ func changeU2FCounterType(x *xorm.Engine) error {
var err error
switch x.Dialect().DriverName() {
case "tidb":
fallthrough
case "mysql":
_, err = x.Exec("ALTER TABLE `u2f_registration` MODIFY `counter` BIGINT")
case "postgres":

148
models/models.go
View file

@ -9,12 +9,6 @@ import (
"database/sql"
"errors"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/modules/setting"
@ -52,24 +46,11 @@ type Engine interface {
}
var (
x *xorm.Engine
supportedDatabases = []string{"mysql", "postgres", "mssql"}
tables []interface{}
x *xorm.Engine
tables []interface{}
// HasEngine specifies if we have a xorm.Engine
HasEngine bool
// DbCfg holds the database settings
DbCfg struct {
Type, Host, Name, User, Passwd, Path, SSLMode, Charset string
Timeout int
}
// EnableSQLite3 use SQLite3
EnableSQLite3 bool
// EnableTiDB enable TiDB
EnableTiDB bool
)
func init() {
@ -139,120 +120,13 @@ func init() {
}
}
// LoadConfigs loads the database settings
func LoadConfigs() {
sec := setting.Cfg.Section("database")
DbCfg.Type = sec.Key("DB_TYPE").String()
switch DbCfg.Type {
case "sqlite3":
setting.UseSQLite3 = true
case "mysql":
setting.UseMySQL = true
case "postgres":
setting.UsePostgreSQL = true
case "tidb":
setting.UseTiDB = true
case "mssql":
setting.UseMSSQL = true
}
DbCfg.Host = sec.Key("HOST").String()
DbCfg.Name = sec.Key("NAME").String()
DbCfg.User = sec.Key("USER").String()
if len(DbCfg.Passwd) == 0 {
DbCfg.Passwd = sec.Key("PASSWD").String()
}
DbCfg.SSLMode = sec.Key("SSL_MODE").MustString("disable")
DbCfg.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
DbCfg.Path = sec.Key("PATH").MustString(filepath.Join(setting.AppDataPath, "gitea.db"))
DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "5432"
if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") {
idx := strings.LastIndex(info, ":")
host = info[:idx]
port = info[idx+1:]
} else if len(info) > 0 {
host = info
}
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) {
host, port := parsePostgreSQLHostPort(dbHost)
if host[0] == '/' { // looks like a unix socket
connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host)
} else {
connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode)
}
return
}
// ParseMSSQLHostPort splits the host into host and port
func ParseMSSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "1433"
if strings.Contains(info, ":") {
host = strings.Split(info, ":")[0]
port = strings.Split(info, ":")[1]
} else if strings.Contains(info, ",") {
host = strings.Split(info, ",")[0]
port = strings.TrimSpace(strings.Split(info, ",")[1])
} else if len(info) > 0 {
host = info
}
return host, port
}
func getEngine() (*xorm.Engine, error) {
connStr := ""
var Param = "?"
if strings.Contains(DbCfg.Name, Param) {
Param = "&"
}
switch DbCfg.Type {
case "mysql":
connType := "tcp"
if DbCfg.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
tls := DbCfg.SSLMode
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
tls = "false"
}
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, DbCfg.Charset, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(DbCfg.Host, DbCfg.User, DbCfg.Passwd, DbCfg.Name, Param, DbCfg.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(DbCfg.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, DbCfg.Name, DbCfg.User, DbCfg.Passwd)
case "sqlite3":
if !EnableSQLite3 {
return nil, errors.New("this binary version does not build support for SQLite3")
}
if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create directories: %v", err)
}
connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", DbCfg.Path, DbCfg.Timeout)
case "tidb":
if !EnableTiDB {
return nil, errors.New("this binary version does not build support for TiDB")
}
if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create directories: %v", err)
}
connStr = "goleveldb://" + DbCfg.Path
default:
return nil, fmt.Errorf("Unknown database type: %s", DbCfg.Type)
connStr, err := setting.DBConnStr()
if err != nil {
return nil, err
}
return xorm.NewEngine(DbCfg.Type, connStr)
return xorm.NewEngine(setting.Database.Type, connStr)
}
// NewTestEngine sets a new test xorm.Engine
@ -280,11 +154,11 @@ func SetEngine() (err error) {
x.SetMapper(core.GonicMapper{})
// WARNING: for serv command, MUST remove the output to os.stdout,
// so use log file to instead print to stdout.
x.SetLogger(NewXORMLogger(setting.LogSQL))
x.ShowSQL(setting.LogSQL)
if DbCfg.Type == "mysql" {
x.SetMaxIdleConns(0)
x.SetConnMaxLifetime(3 * time.Second)
x.SetLogger(NewXORMLogger(setting.Database.LogSQL))
x.ShowSQL(setting.Database.LogSQL)
if setting.Database.UseMySQL {
x.SetMaxIdleConns(setting.Database.MaxIdleConns)
x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
}
return nil

16
models/models_sqlite.go
View file

@ -1,16 +0,0 @@
// +build sqlite
// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
_ "github.com/mattn/go-sqlite3"
)
func init() {
EnableSQLite3 = true
supportedDatabases = append(supportedDatabases, "sqlite3")
}

91
models/models_test.go
View file

@ -1,5 +1,4 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@ -11,99 +10,19 @@ import (
"path/filepath"
"testing"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
)
func Test_parsePostgreSQLHostPort(t *testing.T) {
tests := []struct {
HostPort string
Host string
Port string
}{
{
HostPort: "127.0.0.1:1234",
Host: "127.0.0.1",
Port: "1234",
},
{
HostPort: "127.0.0.1",
Host: "127.0.0.1",
Port: "5432",
},
{
HostPort: "[::1]:1234",
Host: "[::1]",
Port: "1234",
},
{
HostPort: "[::1]",
Host: "[::1]",
Port: "5432",
},
{
HostPort: "/tmp/pg.sock:1234",
Host: "/tmp/pg.sock",
Port: "1234",
},
{
HostPort: "/tmp/pg.sock",
Host: "/tmp/pg.sock",
Port: "5432",
},
}
for _, test := range tests {
host, port := parsePostgreSQLHostPort(test.HostPort)
assert.Equal(t, test.Host, host)
assert.Equal(t, test.Port, port)
}
}
func Test_getPostgreSQLConnectionString(t *testing.T) {
tests := []struct {
Host string
Port string
User string
Passwd string
Name string
Param string
SSLMode string
Output string
}{
{
Host: "/tmp/pg.sock",
Port: "4321",
User: "testuser",
Passwd: "space space !#$%^^%^```-=?=",
Name: "gitea",
Param: "",
SSLMode: "false",
Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock",
},
{
Host: "localhost",
Port: "1234",
User: "pgsqlusername",
Passwd: "I love Gitea!",
Name: "gitea",
Param: "",
SSLMode: "true",
Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true",
},
}
for _, test := range tests {
connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode)
assert.Equal(t, test.Output, connStr)
}
}
func TestDumpDatabase(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())
dir, err := ioutil.TempDir(os.TempDir(), "dump")
assert.NoError(t, err)
for _, dbType := range supportedDatabases {
for _, dbName := range setting.SupportedDatabases {
dbType := setting.GetDBTypeByName(dbName)
assert.NoError(t, DumpDatabase(filepath.Join(dir, dbType+".sql"), dbType))
}
}

6
models/repo.go
View file

@ -2201,7 +2201,7 @@ func GitFsck() {
log.Trace("Doing: GitFsck")
if err := x.
Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.IterateBufferSize).
Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
@ -2225,7 +2225,7 @@ func GitFsck() {
func GitGcRepos() error {
args := append([]string{"gc"}, setting.Git.GCArgs...)
return x.
Where("id > 0").BufferSize(setting.IterateBufferSize).
Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
@ -2568,7 +2568,7 @@ func (repo *Repository) generateRandomAvatar(e Engine) error {
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
func RemoveRandomAvatars() error {
return x.
Where("id > 0").BufferSize(setting.IterateBufferSize).
Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository),
func(idx int, bean interface{}) error {
repository := bean.(*Repository)

2
models/unit_tests.go
View file

@ -49,7 +49,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
setting.RunUser = "runuser"
setting.SSH.Port = 3000
setting.SSH.Domain = "try.gitea.io"
setting.UseSQLite3 = true
setting.Database.UseSQLite3 = true
setting.RepoRootPath, err = ioutil.TempDir(os.TempDir(), "repos")
if err != nil {
fatalTestError("TempDir: %v\n", err)

5
models/user.go
View file

@ -40,7 +40,6 @@ import (
"golang.org/x/crypto/scrypt"
"golang.org/x/crypto/ssh"
"xorm.io/builder"
"xorm.io/core"
)
// UserType defines the user type
@ -1432,9 +1431,9 @@ func (opts *SearchUserOptions) toConds() builder.Cond {
if opts.OwnerID > 0 {
var exprCond builder.Cond
if DbCfg.Type == core.MYSQL {
if setting.Database.UseMySQL {
exprCond = builder.Expr("org_user.org_id = user.id")
} else if DbCfg.Type == core.MSSQL {
} else if setting.Database.UseMSSQL {
exprCond = builder.Expr("org_user.org_id = [user].id")
} else {
exprCond = builder.Expr("org_user.org_id = \"user\".id")

8
models/user_heatmap.go
View file

@ -21,13 +21,13 @@ func GetUserHeatmapDataByUser(user *User) ([]*UserHeatmapData, error) {
var groupBy string
var groupByName = "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
switch {
case setting.UseSQLite3:
case setting.Database.UseSQLite3:
groupBy = "strftime('%s', strftime('%Y-%m-%d', created_unix, 'unixepoch'))"
case setting.UseMySQL:
case setting.Database.UseMySQL:
groupBy = "UNIX_TIMESTAMP(DATE(FROM_UNIXTIME(created_unix)))"
case setting.UsePostgreSQL:
case setting.Database.UsePostgreSQL:
groupBy = "extract(epoch from date_trunc('day', to_timestamp(created_unix)))"
case setting.UseMSSQL:
case setting.Database.UseMSSQL:
groupBy = "datediff(SECOND, '19700101', dateadd(DAY, 0, datediff(day, 0, dateadd(s, created_unix, '19700101'))))"
groupByName = groupBy
}