Move generate from module to service (#29465)

(cherry picked from commit b5188cd55c535a588492fb4e153d646ec4f3232a)

services/repository/create.go

@@ -157,7 +157,7 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re
 		}
 
 		// Apply changes and commit.
-		if err = repo_module.InitRepoCommit(ctx, tmpDir, repo, u, opts.DefaultBranch); err != nil {
+		if err = initRepoCommit(ctx, tmpDir, repo, u, opts.DefaultBranch); err != nil {
 			return fmt.Errorf("initRepoCommit: %w", err)
 		}
 	}

services/repository/generate.go

@@ -0,0 +1,388 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/log"
+	repo_module "code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/gobwas/glob"
+	"github.com/huandu/xstrings"
+)
+
+type transformer struct {
+	Name      string
+	Transform func(string) string
+}
+
+type expansion struct {
+	Name         string
+	Value        string
+	Transformers []transformer
+}
+
+var defaultTransformers = []transformer{
+	{Name: "SNAKE", Transform: xstrings.ToSnakeCase},
+	{Name: "KEBAB", Transform: xstrings.ToKebabCase},
+	{Name: "CAMEL", Transform: func(str string) string {
+		return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str))
+	}},
+	{Name: "PASCAL", Transform: xstrings.ToCamelCase},
+	{Name: "LOWER", Transform: strings.ToLower},
+	{Name: "UPPER", Transform: strings.ToUpper},
+	{Name: "TITLE", Transform: util.ToTitleCase},
+}
+
+func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string {
+	expansions := []expansion{
+		{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers},
+		{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers},
+		{Name: "REPO_DESCRIPTION", Value: generateRepo.Description, Transformers: nil},
+		{Name: "TEMPLATE_DESCRIPTION", Value: templateRepo.Description, Transformers: nil},
+		{Name: "REPO_OWNER", Value: generateRepo.OwnerName, Transformers: defaultTransformers},
+		{Name: "TEMPLATE_OWNER", Value: templateRepo.OwnerName, Transformers: defaultTransformers},
+		{Name: "REPO_LINK", Value: generateRepo.Link(), Transformers: nil},
+		{Name: "TEMPLATE_LINK", Value: templateRepo.Link(), Transformers: nil},
+		{Name: "REPO_HTTPS_URL", Value: generateRepo.CloneLink().HTTPS, Transformers: nil},
+		{Name: "TEMPLATE_HTTPS_URL", Value: templateRepo.CloneLink().HTTPS, Transformers: nil},
+		{Name: "REPO_SSH_URL", Value: generateRepo.CloneLink().SSH, Transformers: nil},
+		{Name: "TEMPLATE_SSH_URL", Value: templateRepo.CloneLink().SSH, Transformers: nil},
+	}
+
+	expansionMap := make(map[string]string)
+	for _, e := range expansions {
+		expansionMap[e.Name] = e.Value
+		for _, tr := range e.Transformers {
+			expansionMap[fmt.Sprintf("%s_%s", e.Name, tr.Name)] = tr.Transform(e.Value)
+		}
+	}
+
+	return os.Expand(src, func(key string) string {
+		if expansion, ok := expansionMap[key]; ok {
+			if sanitizeFileName {
+				return fileNameSanitize(expansion)
+			}
+			return expansion
+		}
+		return key
+	})
+}
+
+// GiteaTemplate holds information about a .gitea/template file
+type GiteaTemplate struct {
+	Path    string
+	Content []byte
+
+	globs []glob.Glob
+}
+
+// Globs parses the .gitea/template globs or returns them if they were already parsed
+func (gt *GiteaTemplate) Globs() []glob.Glob {
+	if gt.globs != nil {
+		return gt.globs
+	}
+
+	gt.globs = make([]glob.Glob, 0)
+	scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		g, err := glob.Compile(line, '/')
+		if err != nil {
+			log.Info("Invalid glob expression '%s' (skipped): %v", line, err)
+			continue
+		}
+		gt.globs = append(gt.globs, g)
+	}
+	return gt.globs
+}
+
+func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
+	gtPath := filepath.Join(tmpDir, ".gitea", "template")
+	if _, err := os.Stat(gtPath); os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	content, err := os.ReadFile(gtPath)
+	if err != nil {
+		return nil, err
+	}
+
+	gt := &GiteaTemplate{
+		Path:    gtPath,
+		Content: content,
+	}
+
+	return gt, nil
+}
+
+func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error {
+	commitTimeStr := time.Now().Format(time.RFC3339)
+	authorSig := repo.Owner.NewGitSig()
+
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+authorSig.Name,
+		"GIT_AUTHOR_EMAIL="+authorSig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_NAME="+authorSig.Name,
+		"GIT_COMMITTER_EMAIL="+authorSig.Email,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+
+	// Clone to temporary path and do the init commit.
+	templateRepoPath := templateRepo.RepoPath()
+	if err := git.Clone(ctx, templateRepoPath, tmpDir, git.CloneRepoOptions{
+		Depth:  1,
+		Branch: templateRepo.DefaultBranch,
+	}); err != nil {
+		return fmt.Errorf("git clone: %w", err)
+	}
+
+	if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
+		return fmt.Errorf("remove git dir: %w", err)
+	}
+
+	// Variable expansion
+	gt, err := checkGiteaTemplate(tmpDir)
+	if err != nil {
+		return fmt.Errorf("checkGiteaTemplate: %w", err)
+	}
+
+	if gt != nil {
+		if err := util.Remove(gt.Path); err != nil {
+			return fmt.Errorf("remove .giteatemplate: %w", err)
+		}
+
+		// Avoid walking tree if there are no globs
+		if len(gt.Globs()) > 0 {
+			tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
+			if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
+				if walkErr != nil {
+					return walkErr
+				}
+
+				if d.IsDir() {
+					return nil
+				}
+
+				base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
+				for _, g := range gt.Globs() {
+					if g.Match(base) {
+						content, err := os.ReadFile(path)
+						if err != nil {
+							return err
+						}
+
+						if err := os.WriteFile(path,
+							[]byte(generateExpansion(string(content), templateRepo, generateRepo, false)),
+							0o644); err != nil {
+							return err
+						}
+
+						substPath := filepath.FromSlash(filepath.Join(tmpDirSlash,
+							generateExpansion(base, templateRepo, generateRepo, true)))
+
+						// Create parent subdirectories if needed or continue silently if it exists
+						if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil {
+							return err
+						}
+
+						// Substitute filename variables
+						if err := os.Rename(path, substPath); err != nil {
+							return err
+						}
+
+						break
+					}
+				}
+				return nil
+			}); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil {
+		return err
+	}
+
+	repoPath := repo.RepoPath()
+	if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repoPath).
+		SetDescription(fmt.Sprintf("generateRepoCommit (git remote add): %s to %s", templateRepoPath, tmpDir)).
+		RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil {
+		log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err)
+		return fmt.Errorf("git remote add: %w", err)
+	}
+
+	// set default branch based on whether it's specified in the newly generated repo or not
+	defaultBranch := repo.DefaultBranch
+	if strings.TrimSpace(defaultBranch) == "" {
+		defaultBranch = templateRepo.DefaultBranch
+	}
+
+	return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch)
+}
+
+func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) {
+	tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name)
+	if err != nil {
+		return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err)
+	}
+
+	defer func() {
+		if err := util.RemoveAll(tmpDir); err != nil {
+			log.Error("RemoveAll: %v", err)
+		}
+	}()
+
+	if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil {
+		return fmt.Errorf("generateRepoCommit: %w", err)
+	}
+
+	// re-fetch repo
+	if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil {
+		return fmt.Errorf("getRepositoryByID: %w", err)
+	}
+
+	// if there was no default branch supplied when generating the repo, use the default one from the template
+	if strings.TrimSpace(repo.DefaultBranch) == "" {
+		repo.DefaultBranch = templateRepo.DefaultBranch
+	}
+
+	gitRepo, err := gitrepo.OpenRepository(ctx, repo)
+	if err != nil {
+		return fmt.Errorf("openRepository: %w", err)
+	}
+	defer gitRepo.Close()
+	if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
+		return fmt.Errorf("setDefaultBranch: %w", err)
+	}
+	if err = UpdateRepository(ctx, repo, false); err != nil {
+		return fmt.Errorf("updateRepository: %w", err)
+	}
+
+	return nil
+}
+
+// GenerateGitContent generates git content from a template repository
+func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error {
+	if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil {
+		return err
+	}
+
+	if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil {
+		return fmt.Errorf("failed to update size for repository: %w", err)
+	}
+
+	if err := git_model.CopyLFS(ctx, generateRepo, templateRepo); err != nil {
+		return fmt.Errorf("failed to copy LFS: %w", err)
+	}
+	return nil
+}
+
+// GenerateRepoOptions contains the template units to generate
+type GenerateRepoOptions struct {
+	Name            string
+	DefaultBranch   string
+	Description     string
+	Private         bool
+	GitContent      bool
+	Topics          bool
+	GitHooks        bool
+	Webhooks        bool
+	Avatar          bool
+	IssueLabels     bool
+	ProtectedBranch bool
+}
+
+// IsValid checks whether at least one option is chosen for generation
+func (gro GenerateRepoOptions) IsValid() bool {
+	return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar ||
+		gro.IssueLabels || gro.ProtectedBranch // or other items as they are added
+}
+
+// generateRepository generates a repository from a template
+func generateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) {
+	generateRepo := &repo_model.Repository{
+		OwnerID:          owner.ID,
+		Owner:            owner,
+		OwnerName:        owner.Name,
+		Name:             opts.Name,
+		LowerName:        strings.ToLower(opts.Name),
+		Description:      opts.Description,
+		DefaultBranch:    opts.DefaultBranch,
+		IsPrivate:        opts.Private,
+		IsEmpty:          !opts.GitContent || templateRepo.IsEmpty,
+		IsFsckEnabled:    templateRepo.IsFsckEnabled,
+		TemplateID:       templateRepo.ID,
+		TrustModel:       templateRepo.TrustModel,
+		ObjectFormatName: templateRepo.ObjectFormatName,
+	}
+
+	if err = repo_module.CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil {
+		return nil, err
+	}
+
+	repoPath := generateRepo.RepoPath()
+	isExist, err := util.IsExist(repoPath)
+	if err != nil {
+		log.Error("Unable to check if %s exists. Error: %v", repoPath, err)
+		return nil, err
+	}
+	if isExist {
+		return nil, repo_model.ErrRepoFilesAlreadyExist{
+			Uname: generateRepo.OwnerName,
+			Name:  generateRepo.Name,
+		}
+	}
+
+	if err = repo_module.CheckInitRepository(ctx, owner.Name, generateRepo.Name, generateRepo.ObjectFormatName); err != nil {
+		return generateRepo, err
+	}
+
+	if err = repo_module.CheckDaemonExportOK(ctx, generateRepo); err != nil {
+		return generateRepo, fmt.Errorf("checkDaemonExportOK: %w", err)
+	}
+
+	if stdout, _, err := git.NewCommand(ctx, "update-server-info").
+		SetDescription(fmt.Sprintf("GenerateRepository(git update-server-info): %s", repoPath)).
+		RunStdString(&git.RunOpts{Dir: repoPath}); err != nil {
+		log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err)
+		return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %w", err)
+	}
+
+	return generateRepo, nil
+}
+
+var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`)
+
+// Sanitize user input to valid OS filenames
+//
+//		Based on https://github.com/sindresorhus/filename-reserved-regex
+//	 Adds ".." to prevent directory traversal
+func fileNameSanitize(s string) string {
+	return strings.TrimSpace(fileNameSanitizeRegexp.ReplaceAllString(s, "_"))
+}

services/repository/generate_test.go

@@ -0,0 +1,67 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var giteaTemplate = []byte(`
+# Header
+
+# All .go files
+**.go
+
+# All text files in /text/
+text/*.txt
+
+# All files in modules folders
+**/modules/*
+`)
+
+func TestGiteaTemplate(t *testing.T) {
+	gt := GiteaTemplate{Content: giteaTemplate}
+	assert.Len(t, gt.Globs(), 3)
+
+	tt := []struct {
+		Path  string
+		Match bool
+	}{
+		{Path: "main.go", Match: true},
+		{Path: "a/b/c/d/e.go", Match: true},
+		{Path: "main.txt", Match: false},
+		{Path: "a/b.txt", Match: false},
+		{Path: "text/a.txt", Match: true},
+		{Path: "text/b.txt", Match: true},
+		{Path: "text/c.json", Match: false},
+		{Path: "a/b/c/modules/README.md", Match: true},
+		{Path: "a/b/c/modules/d/README.md", Match: false},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.Path, func(t *testing.T) {
+			match := false
+			for _, g := range gt.Globs() {
+				if g.Match(tc.Path) {
+					match = true
+					break
+				}
+			}
+			assert.Equal(t, tc.Match, match)
+		})
+	}
+}
+
+func TestFileNameSanitize(t *testing.T) {
+	assert.Equal(t, "test_CON", fileNameSanitize("test_CON"))
+	assert.Equal(t, "test CON", fileNameSanitize("test CON "))
+	assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/.."))
+	assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git"))
+	assert.Equal(t, "_", fileNameSanitize("CON"))
+	assert.Equal(t, "_", fileNameSanitize("con"))
+	assert.Equal(t, "_", fileNameSanitize("\u0000"))
+	assert.Equal(t, "目标", fileNameSanitize("目标"))
+}

services/repository/init.go

@@ -0,0 +1,83 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	repo_module "code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/setting"
+	asymkey_service "code.gitea.io/gitea/services/asymkey"
+)
+
+// initRepoCommit temporarily changes with work directory.
+func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Repository, u *user_model.User, defaultBranch string) (err error) {
+	commitTimeStr := time.Now().Format(time.RFC3339)
+
+	sig := u.NewGitSig()
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+sig.Name,
+		"GIT_AUTHOR_EMAIL="+sig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+	committerName := sig.Name
+	committerEmail := sig.Email
+
+	if stdout, _, err := git.NewCommand(ctx, "add", "--all").
+		SetDescription(fmt.Sprintf("initRepoCommit (git add): %s", tmpPath)).
+		RunStdString(&git.RunOpts{Dir: tmpPath}); err != nil {
+		log.Error("git add --all failed: Stdout: %s\nError: %v", stdout, err)
+		return fmt.Errorf("git add --all: %w", err)
+	}
+
+	cmd := git.NewCommand(ctx, "commit", "--message=Initial commit").
+		AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)
+
+	sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u)
+	if sign {
+		cmd.AddOptionFormat("-S%s", keyID)
+
+		if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel {
+			// need to set the committer to the KeyID owner
+			committerName = signer.Name
+			committerEmail = signer.Email
+		}
+	} else {
+		cmd.AddArguments("--no-gpg-sign")
+	}
+
+	env = append(env,
+		"GIT_COMMITTER_NAME="+committerName,
+		"GIT_COMMITTER_EMAIL="+committerEmail,
+	)
+
+	if stdout, _, err := cmd.
+		SetDescription(fmt.Sprintf("initRepoCommit (git commit): %s", tmpPath)).
+		RunStdString(&git.RunOpts{Dir: tmpPath, Env: env}); err != nil {
+		log.Error("Failed to commit: %v: Stdout: %s\nError: %v", cmd.String(), stdout, err)
+		return fmt.Errorf("git commit: %w", err)
+	}
+
+	if len(defaultBranch) == 0 {
+		defaultBranch = setting.Repository.DefaultBranch
+	}
+
+	if stdout, _, err := git.NewCommand(ctx, "push", "origin").AddDynamicArguments("HEAD:" + defaultBranch).
+		SetDescription(fmt.Sprintf("initRepoCommit (git push): %s", tmpPath)).
+		RunStdString(&git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil {
+		log.Error("Failed to push back to HEAD: Stdout: %s\nError: %v", stdout, err)
+		return fmt.Errorf("git push: %w", err)
+	}
+
+	return nil
+}

services/repository/template.go

@@ -11,7 +11,6 @@ import (
 	issues_model "code.gitea.io/gitea/models/issues"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
-	repo_module "code.gitea.io/gitea/modules/repository"
 	notify_service "code.gitea.io/gitea/services/notify"
 )
 
@@ -63,7 +62,7 @@ func GenerateProtectedBranch(ctx context.Context, templateRepo, generateRepo *re
 }
 
 // GenerateRepository generates a repository from a template
-func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts repo_module.GenerateRepoOptions) (_ *repo_model.Repository, err error) {
+func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) {
 	if !doer.IsAdmin && !owner.CanCreateRepo() {
 		return nil, repo_model.ErrReachLimitOfRepo{
 			Limit: owner.MaxRepoCreation,
@@ -72,14 +71,14 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ
 
 	var generateRepo *repo_model.Repository
 	if err = db.WithTx(ctx, func(ctx context.Context) error {
-		generateRepo, err = repo_module.GenerateRepository(ctx, doer, owner, templateRepo, opts)
+		generateRepo, err = generateRepository(ctx, doer, owner, templateRepo, opts)
 		if err != nil {
 			return err
 		}
 
 		// Git Content
 		if opts.GitContent && !templateRepo.IsEmpty {
-			if err = repo_module.GenerateGitContent(ctx, templateRepo, generateRepo); err != nil {
+			if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil {
 				return err
 			}
 		}