Sync branches into databases (#22743)

Related #14180
Related #25233 
Related #22639
Close #19786
Related #12763 

This PR will change all the branches retrieve method from reading git
data to read database to reduce git read operations.

- [x] Sync git branches information into database when push git data
- [x] Create a new table `Branch`, merge some columns of `DeletedBranch`
into `Branch` table and drop the table `DeletedBranch`.
- [x] Read `Branch` table when visit `code` -> `branch` page
- [x] Read `Branch` table when list branch names in `code` page dropdown
- [x] Read `Branch` table when list git ref compare page
- [x] Provide a button in admin page to manually sync all branches.
- [x] Sync branches if repository is not empty but database branches are
empty when visiting pages with branches list
- [x] Use `commit_time desc` as the default FindBranch order by to keep
consistent as before and deleted branches will be always at the end.

---------

Co-authored-by: Jason Song <i@wolfogre.com>

modules/context/repo.go

@@ -667,13 +667,38 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	}
 	ctx.Data["Tags"] = tags
 
-	brs, _, err := ctx.Repo.GitRepo.GetBranchNames(0, 0)
+	branchOpts := git_model.FindBranchOptions{
+		RepoID:          ctx.Repo.Repository.ID,
+		IsDeletedBranch: util.OptionalBoolFalse,
+		ListOptions: db.ListOptions{
+			ListAll: true,
+		},
+	}
+	branchesTotal, err := git_model.CountBranches(ctx, branchOpts)
+	if err != nil {
+		ctx.ServerError("CountBranches", err)
+		return
+	}
+
+	// non empty repo should have at least 1 branch, so this repository's branches haven't been synced yet
+	if branchesTotal == 0 { // fallback to do a sync immediately
+		branchesTotal, err = repo_module.SyncRepoBranches(ctx, ctx.Repo.Repository.ID, 0)
+		if err != nil {
+			ctx.ServerError("SyncRepoBranches", err)
+			return
+		}
+	}
+
+	// FIXME: use paganation and async loading
+	branchOpts.ExcludeBranchNames = []string{ctx.Repo.Repository.DefaultBranch}
+	brs, err := git_model.FindBranchNames(ctx, branchOpts)
 	if err != nil {
 		ctx.ServerError("GetBranches", err)
 		return
 	}
-	ctx.Data["Branches"] = brs
-	ctx.Data["BranchesCount"] = len(brs)
+	// always put default branch on the top
+	ctx.Data["Branches"] = append(branchOpts.ExcludeBranchNames, brs...)
+	ctx.Data["BranchesCount"] = branchesTotal
 
 	// If not branch selected, try default one.
 	// If default branch doesn't exist, fall back to some other branch.
@@ -897,9 +922,9 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 		if len(ctx.Params("*")) == 0 {
 			refName = ctx.Repo.Repository.DefaultBranch
 			if !ctx.Repo.GitRepo.IsBranchExist(refName) {
-				brs, _, err := ctx.Repo.GitRepo.GetBranchNames(0, 0)
+				brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 1)
 				if err == nil && len(brs) != 0 {
-					refName = brs[0]
+					refName = brs[0].Name
 				} else if len(brs) == 0 {
 					log.Error("No branches in non-empty repository %s", ctx.Repo.GitRepo.Path)
 					ctx.Repo.Repository.MarkAsBrokenEmpty()

modules/repository/branch.go

@@ -0,0 +1,135 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+// SyncRepoBranches synchronizes branch table with repository branches
+func SyncRepoBranches(ctx context.Context, repoID, doerID int64) (int64, error) {
+	repo, err := repo_model.GetRepositoryByID(ctx, repoID)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Debug("SyncRepoBranches: in Repo[%d:%s]", repo.ID, repo.FullName())
+
+	gitRepo, err := git.OpenRepository(ctx, repo.RepoPath())
+	if err != nil {
+		log.Error("OpenRepository[%s]: %w", repo.RepoPath(), err)
+		return 0, err
+	}
+	defer gitRepo.Close()
+
+	return SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doerID)
+}
+
+func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) (int64, error) {
+	allBranches := container.Set[string]{}
+	{
+		branches, _, err := gitRepo.GetBranchNames(0, 0)
+		if err != nil {
+			return 0, err
+		}
+		log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches)
+		for _, branch := range branches {
+			allBranches.Add(branch)
+		}
+	}
+
+	dbBranches := make(map[string]*git_model.Branch)
+	{
+		branches, err := git_model.FindBranches(ctx, git_model.FindBranchOptions{
+			ListOptions: db.ListOptions{
+				ListAll: true,
+			},
+			RepoID: repo.ID,
+		})
+		if err != nil {
+			return 0, err
+		}
+		for _, branch := range branches {
+			dbBranches[branch.Name] = branch
+		}
+	}
+
+	var toAdd []*git_model.Branch
+	var toUpdate []*git_model.Branch
+	var toRemove []int64
+	for branch := range allBranches {
+		dbb := dbBranches[branch]
+		commit, err := gitRepo.GetBranchCommit(branch)
+		if err != nil {
+			return 0, err
+		}
+		if dbb == nil {
+			toAdd = append(toAdd, &git_model.Branch{
+				RepoID:        repo.ID,
+				Name:          branch,
+				CommitID:      commit.ID.String(),
+				CommitMessage: commit.CommitMessage,
+				PusherID:      doerID,
+				CommitTime:    timeutil.TimeStamp(commit.Author.When.Unix()),
+			})
+		} else if commit.ID.String() != dbb.CommitID {
+			toUpdate = append(toUpdate, &git_model.Branch{
+				ID:            dbb.ID,
+				RepoID:        repo.ID,
+				Name:          branch,
+				CommitID:      commit.ID.String(),
+				CommitMessage: commit.CommitMessage,
+				PusherID:      doerID,
+				CommitTime:    timeutil.TimeStamp(commit.Author.When.Unix()),
+			})
+		}
+	}
+
+	for _, dbBranch := range dbBranches {
+		if !allBranches.Contains(dbBranch.Name) && !dbBranch.IsDeleted {
+			toRemove = append(toRemove, dbBranch.ID)
+		}
+	}
+
+	log.Trace("SyncRepoBranches[%s]: toAdd: %v, toUpdate: %v, toRemove: %v", repo.FullName(), toAdd, toUpdate, toRemove)
+
+	if len(toAdd) == 0 && len(toRemove) == 0 && len(toUpdate) == 0 {
+		return int64(len(allBranches)), nil
+	}
+
+	if err := db.WithTx(ctx, func(subCtx context.Context) error {
+		if len(toAdd) > 0 {
+			if err := git_model.AddBranches(subCtx, toAdd); err != nil {
+				return err
+			}
+		}
+
+		for _, b := range toUpdate {
+			if _, err := db.GetEngine(subCtx).ID(b.ID).
+				Cols("commit_id, commit_message, pusher_id, commit_time, is_deleted").
+				Update(b); err != nil {
+				return err
+			}
+		}
+
+		if len(toRemove) > 0 {
+			if err := git_model.DeleteBranches(subCtx, repo.ID, doerID, toRemove); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return 0, err
+	}
+	return int64(len(allBranches)), nil
+}

modules/repository/init.go

@@ -351,6 +351,12 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re
 		if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
 			return fmt.Errorf("setDefaultBranch: %w", err)
 		}
+
+		if !repo.IsEmpty {
+			if _, err := SyncRepoBranches(ctx, repo.ID, u.ID); err != nil {
+				return fmt.Errorf("SyncRepoBranches: %w", err)
+			}
+		}
 	}
 
 	if err = UpdateRepository(ctx, repo, false); err != nil {

modules/repository/repo.go

@@ -151,6 +151,10 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 			}
 		}
 
+		if _, err := SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil {
+			return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err)
+		}
+
 		if !opts.Releases {
 			// note: this will greatly improve release (tag) sync
 			// for pull-mirrors with many tags
@@ -169,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		}
 	}
 
-	ctx, committer, err := db.TxContext(db.DefaultContext)
+	ctx, committer, err := db.TxContext(ctx)
 	if err != nil {
 		return nil, err
 	}