Remove legacy unknwon/com package (#19298)

Follows: #19284
* The `CopyDir` is only used inside test code
* Rewrite `ToSnakeCase` with more test cases
* The `RedisCacher` only put strings into cache, here we use internal `toStr` to replace the legacy `ToStr`
* The `UniqueQueue` can use string as ID directly, no need to call `ToStr`

cmd/manager_logging.go

@@ -11,6 +11,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
+
 	"github.com/urfave/cli"
 )
 

contrib/pr/checkout.go

@@ -112,7 +112,7 @@ func runPR() {
 	unittest.LoadFixtures()
 	util.RemoveAll(setting.RepoRootPath)
 	util.RemoveAll(models.LocalCopyPath())
-	util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
+	unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
 
 	log.Printf("[PR] Setting up router\n")
 	// routers.GlobalInit()

go.mod

@@ -78,7 +78,6 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tstranex/u2f v1.0.0
-	github.com/unknwon/com v1.0.1
 	github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361
 	github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae
 	github.com/unrolled/render v1.4.1
@@ -251,6 +250,7 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/toqueteos/webbrowser v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/unknwon/com v1.0.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect

integrations/integration_test.go

@@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
 	assert.NoError(t, unittest.LoadFixtures())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
@@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) {
 	assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1))
 	assert.NoError(t, unittest.LoadFixtures())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

integrations/migration-test/migration_test.go

@@ -21,6 +21,7 @@ import (
 	"code.gitea.io/gitea/integrations"
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
@@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() {
 
 	assert.True(t, len(setting.RepoRootPath) != 0)
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

models/migrations/migrations_test.go

@@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 	deferFn := PrintCurrentTest(t, ourSkip)
 	assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
 
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
 		setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {

models/unittest/fscopy.go

@@ -0,0 +1,103 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package unittest
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// Copy copies file from source to target path.
+func Copy(src, dest string) error {
+	// Gather file information to set back later.
+	si, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+
+	// Handle symbolic link.
+	if si.Mode()&os.ModeSymlink != 0 {
+		target, err := os.Readlink(src)
+		if err != nil {
+			return err
+		}
+		// NOTE: os.Chmod and os.Chtimes don't recognize symbolic link,
+		// which will lead "no such file or directory" error.
+		return os.Symlink(target, dest)
+	}
+
+	sr, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sr.Close()
+
+	dw, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer dw.Close()
+
+	if _, err = io.Copy(dw, sr); err != nil {
+		return err
+	}
+
+	// Set back file information.
+	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
+		return err
+	}
+	return os.Chmod(dest, si.Mode())
+}
+
+// CopyDir copy files recursively from source to target directory.
+//
+// The filter accepts a function that process the path info.
+// and should return true for need to filter.
+//
+// It returns error when error occurs in underlying functions.
+func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
+	// Check if target directory exists.
+	if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) {
+		return errors.New("file or directory already exists: " + destPath)
+	}
+
+	err := os.MkdirAll(destPath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	// Gather directory info.
+	infos, err := util.StatDir(srcPath, true)
+	if err != nil {
+		return err
+	}
+
+	var filter func(filePath string) bool
+	if len(filters) > 0 {
+		filter = filters[0]
+	}
+
+	for _, info := range infos {
+		if filter != nil && filter(info) {
+			continue
+		}
+
+		curPath := path.Join(destPath, info)
+		if strings.HasSuffix(info, "/") {
+			err = os.MkdirAll(curPath, os.ModePerm)
+		} else {
+			err = Copy(path.Join(srcPath, info), curPath)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/unittest/testdb.go

@@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) {
 	if err = util.RemoveAll(repoRootPath); err != nil {
 		fatalTestError("util.RemoveAll: %v\n", err)
 	}
-	if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
+	if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
 		fatalTestError("util.CopyDir: %v\n", err)
 	}
 
@@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) {
 	assert.NoError(t, PrepareTestDatabase())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 	metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta")
-	assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath))
+	assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
 
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	assert.NoError(t, err)

modules/cache/cache_redis.go

@@ -6,11 +6,11 @@ package cache
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/nosql"
-	"code.gitea.io/gitea/modules/util"
 
 	"gitea.com/go-chi/cache"
 	"github.com/go-redis/redis/v8"
@@ -24,20 +24,37 @@ type RedisCacher struct {
 	occupyMode bool
 }
 
-// Put puts value into cache with key and expire time.
+// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
+func toStr(v interface{}) string {
+	if v == nil {
+		return ""
+	}
+	switch v := v.(type) {
+	case string:
+		return v
+	case []byte:
+		return string(v)
+	case int:
+		return strconv.FormatInt(int64(v), 10)
+	case int64:
+		return strconv.FormatInt(v, 10)
+	default:
+		return fmt.Sprint(v) // as what the old com.ToStr does in most cases
+	}
+}
+
+// Put puts value (string type) into cache with key and expire time.
 // If expired is 0, it lives forever.
 func (c *RedisCacher) Put(key string, val interface{}, expire int64) error {
+	// this function is not well-designed, it only puts string values into cache
 	key = c.prefix + key
 	if expire == 0 {
-		if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil {
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
 			return err
 		}
 	} else {
-		dur, err := time.ParseDuration(util.ToStr(expire) + "s")
+		dur := time.Duration(expire) * time.Second
-		if err != nil {
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
-			return err
-		}
-		if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil {
 			return err
 		}
 	}

modules/context/csrf.go

@@ -22,8 +22,10 @@ import (
 	"encoding/base32"
 	"fmt"
 	"net/http"
+	"strconv"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web/middleware"
@@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF {
 	}
 
 	x.ID = "0"
-	uid := ctx.Session.Get(opt.SessionKey)
+	uidAny := ctx.Session.Get(opt.SessionKey)
-	if uid != nil {
+	if uidAny != nil {
-		x.ID = util.ToStr(uid)
+		switch uidVal := uidAny.(type) {
+		case string:
+			x.ID = uidVal
+		case int64:
+			x.ID = strconv.FormatInt(uidVal, 10)
+		default:
+			log.Error("invalid uid type in session: %T", uidAny)
+		}
 	}
 
 	needsNew := false

modules/nosql/manager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/process"
+
 	"github.com/go-redis/redis/v8"
 	"github.com/syndtr/goleveldb/leveldb"
 )

modules/sync/unique_queue.go

@@ -5,8 +5,6 @@
 
 package sync
 
-import "code.gitea.io/gitea/modules/util"
-
 // UniqueQueue is a queue which guarantees only one instance of same
 // identity is in the line. Instances with same identity will be
 // discarded if there is already one in the line.
@@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} {
 }
 
 // IDs returns the current ids in the pool
-func (q *UniqueQueue) IDs() []interface{} {
+func (q *UniqueQueue) IDs() []string {
 	q.table.lock.Lock()
 	defer q.table.lock.Unlock()
-	ids := make([]interface{}, 0, len(q.table.pool))
+	ids := make([]string, 0, len(q.table.pool))
 	for id := range q.table.pool {
 		ids = append(ids, id)
 	}
@@ -70,20 +68,19 @@ func (q *UniqueQueue) Queue() <-chan string {
 
 // Exist returns true if there is an instance with given identity
 // exists in the queue.
-func (q *UniqueQueue) Exist(id interface{}) bool {
+func (q *UniqueQueue) Exist(id string) bool {
-	return q.table.IsRunning(util.ToStr(id))
+	return q.table.IsRunning(id)
 }
 
 // AddFunc adds new instance to the queue with a custom runnable function,
 // the queue is blocked until the function exits.
-func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
+func (q *UniqueQueue) AddFunc(id string, fn func()) {
-	idStr := util.ToStr(id)
 	q.table.lock.Lock()
-	if _, ok := q.table.pool[idStr]; ok {
+	if _, ok := q.table.pool[id]; ok {
 		q.table.lock.Unlock()
 		return
 	}
-	q.table.pool[idStr] = struct{}{}
+	q.table.pool[id] = struct{}{}
 	if fn != nil {
 		fn()
 	}
@@ -91,17 +88,17 @@ func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
 	select {
 	case <-q.closed:
 		return
-	case q.queue <- idStr:
+	case q.queue <- id:
 		return
 	}
 }
 
 // Add adds new instance to the queue.
-func (q *UniqueQueue) Add(id interface{}) {
+func (q *UniqueQueue) Add(id string) {
 	q.AddFunc(id, nil)
 }
 
 // Remove removes instance from the queue.
-func (q *UniqueQueue) Remove(id interface{}) {
+func (q *UniqueQueue) Remove(id string) {
-	q.table.Stop(util.ToStr(id))
+	q.table.Stop(id)
 }

modules/util/legacy.go

@@ -9,29 +9,37 @@ import (
 	"crypto/cipher"
 	"crypto/rand"
 	"errors"
-
+	"io"
-	"github.com/unknwon/com" //nolint:depguard
+	"os"
 )
 
 // CopyFile copies file from source to target path.
 func CopyFile(src, dest string) error {
-	return com.Copy(src, dest)
+	si, err := os.Lstat(src)
+	if err != nil {
+		return err
 	}
 
-// CopyDir copy files recursively from source to target directory.
+	sr, err := os.Open(src)
-// It returns error when error occurs in underlying functions.
+	if err != nil {
-func CopyDir(srcPath, destPath string) error {
+		return err
-	return com.CopyDir(srcPath, destPath)
+	}
+	defer sr.Close()
+
+	dw, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer dw.Close()
+
+	if _, err = io.Copy(dw, sr); err != nil {
+		return err
 	}
 
-// ToStr converts any interface to string. should be replaced.
+	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
-func ToStr(value interface{}, args ...int) string {
+		return err
-	return com.ToStr(value, args...)
 	}
-
+	return os.Chmod(dest, si.Mode())
-// ToSnakeCase converts a string to snake_case. should be replaced.
-func ToSnakeCase(str string) string {
-	return com.ToSnakeCase(str)
 }
 
 // AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced.

modules/util/legacy_test.go

@@ -7,12 +7,38 @@ package util
 import (
 	"crypto/aes"
 	"crypto/rand"
+	"fmt"
+	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/unknwon/com" //nolint:depguard
 )
 
+func TestCopyFile(t *testing.T) {
+	testContent := []byte("hello")
+
+	tmpDir := os.TempDir()
+	now := time.Now()
+	srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro())
+	dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro())
+
+	_ = os.Remove(srcFile)
+	_ = os.Remove(dstFile)
+	defer func() {
+		_ = os.Remove(srcFile)
+		_ = os.Remove(dstFile)
+	}()
+
+	err := os.WriteFile(srcFile, testContent, 0o777)
+	assert.NoError(t, err)
+	err = CopyFile(srcFile, dstFile)
+	assert.NoError(t, err)
+	dstContent, err := os.ReadFile(dstFile)
+	assert.NoError(t, err)
+	assert.Equal(t, testContent, dstContent)
+}
+
 func TestAESGCM(t *testing.T) {
 	t.Parallel()
 
@@ -29,9 +55,4 @@ func TestAESGCM(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.Equal(t, plaintext, decrypted)
-
-	// at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring
-	legacy, err := com.AESGCMDecrypt(key, ciphertext)
-	assert.NoError(t, err)
-	assert.Equal(t, legacy, plaintext)
 }

modules/util/string.go

@@ -0,0 +1,88 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import "github.com/yuin/goldmark/util"
+
+func isSnakeCaseUpper(c byte) bool {
+	return 'A' <= c && c <= 'Z'
+}
+
+func isSnakeCaseLowerOrNumber(c byte) bool {
+	return 'a' <= c && c <= 'z' || '0' <= c && c <= '9'
+}
+
+// ToSnakeCase convert the input string to snake_case format.
+//
+// Some samples.
+//     "FirstName"  => "first_name"
+//     "HTTPServer" => "http_server"
+//     "NoHTTPS"    => "no_https"
+//     "GO_PATH"    => "go_path"
+//     "GO PATH"    => "go_path"      // space is converted to underscore.
+//     "GO-PATH"    => "go_path"      // hyphen is converted to underscore.
+//
+func ToSnakeCase(input string) string {
+	if len(input) == 0 {
+		return ""
+	}
+
+	var res []byte
+	if len(input) == 1 {
+		c := input[0]
+		if isSnakeCaseUpper(c) {
+			res = []byte{c + 'a' - 'A'}
+		} else if isSnakeCaseLowerOrNumber(c) {
+			res = []byte{c}
+		} else {
+			res = []byte{'_'}
+		}
+	} else {
+		res = make([]byte, 0, len(input)*4/3)
+		pos := 0
+		needSep := false
+		for pos < len(input) {
+			c := input[pos]
+			if c >= 0x80 {
+				res = append(res, c)
+				pos++
+				continue
+			}
+			isUpper := isSnakeCaseUpper(c)
+			if isUpper || isSnakeCaseLowerOrNumber(c) {
+				end := pos + 1
+				if isUpper {
+					// skip the following upper letters
+					for end < len(input) && isSnakeCaseUpper(input[end]) {
+						end++
+					}
+					if end-pos > 1 && end < len(input) && isSnakeCaseLowerOrNumber(input[end]) {
+						end--
+					}
+				}
+				// skip the following lower or number letters
+				for end < len(input) && (isSnakeCaseLowerOrNumber(input[end]) || input[end] >= 0x80) {
+					end++
+				}
+				if needSep {
+					res = append(res, '_')
+				}
+				res = append(res, input[pos:end]...)
+				pos = end
+				needSep = true
+			} else {
+				res = append(res, '_')
+				pos++
+				needSep = false
+			}
+		}
+		for i := 0; i < len(res); i++ {
+			if isSnakeCaseUpper(res[i]) {
+				res[i] += 'a' - 'A'
+			}
+		}
+	}
+	return util.BytesToReadOnlyString(res)
+}

modules/util/string_test.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestToSnakeCase(t *testing.T) {
+	cases := map[string]string{
+		// all old cases from the legacy package
+		"HTTPServer":         "http_server",
+		"_camelCase":         "_camel_case",
+		"NoHTTPS":            "no_https",
+		"Wi_thF":             "wi_th_f",
+		"_AnotherTES_TCaseP": "_another_tes_t_case_p",
+		"ALL":                "all",
+		"_HELLO_WORLD_":      "_hello_world_",
+		"HELLO_WORLD":        "hello_world",
+		"HELLO____WORLD":     "hello____world",
+		"TW":                 "tw",
+		"_C":                 "_c",
+
+		"  sentence case  ": "__sentence_case__",
+		" Mixed-hyphen case _and SENTENCE_case and UPPER-case": "_mixed_hyphen_case__and_sentence_case_and_upper_case",
+
+		// new cases
+		" ":        "_",
+		"A":        "a",
+		"A0":       "a0",
+		"a0":       "a0",
+		"Aa0":      "aa0",
+		"啊":        "啊",
+		"A啊":       "a啊",
+		"Aa啊b":     "aa啊b",
+		"A啊B":      "a啊_b",
+		"Aa啊B":     "aa啊_b",
+		"TheCase2": "the_case2",
+		"ObjIDs":   "obj_i_ds", // the strange database column name which already exists
+	}
+	for input, expected := range cases {
+		assert.Equal(t, expected, ToSnakeCase(input))
+	}
+}

services/webhook/webhook.go

@@ -6,6 +6,7 @@ package webhook
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	repo_model "code.gitea.io/gitea/models/repo"
@@ -106,7 +107,7 @@ func PrepareWebhook(w *webhook_model.Webhook, repo *repo_model.Repository, event
 		return err
 	}
 
-	go hookQueue.Add(repo.ID)
+	go hookQueue.Add(strconv.FormatInt(repo.ID, 10))
 	return nil
 }
 
@@ -187,7 +188,7 @@ func PrepareWebhooks(repo *repo_model.Repository, event webhook_model.HookEventT
 		return err
 	}
 
-	go hookQueue.Add(repo.ID)
+	go hookQueue.Add(strconv.FormatInt(repo.ID, 10))
 	return nil
 }
 
@@ -239,7 +240,7 @@ func ReplayHookTask(w *webhook_model.Webhook, uuid string) error {
 		return err
 	}
 
-	go hookQueue.Add(t.RepoID)
+	go hookQueue.Add(strconv.FormatInt(t.RepoID, 10))
 
 	return nil
 }