Template Repositories (#8768)

* Start work on templates

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Continue work

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix IsTemplate vs IsGenerated

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix tabs vs spaces

* Tabs vs Spaces

* Add templates to API & start adding tests

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix integration tests

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Remove unused User

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Move template tests to existing repos

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Minor re-check updates and cleanup

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* make fmt

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Test cleanup

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix optionalbool

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* make fmt

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Test fixes and icon change

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Add new user and repo for tests

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix tests (finally)

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Update meta repo with env variables

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Move generation to create page

Combine with repo create template
Modify API search to prioritize owner for repo

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix tests and coverage

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix swagger and JS lint

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix API searching for own private repos

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Change wording

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix repo search test. User had a private repo that didn't show up

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Another search test fix

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Clarify git content

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Feedback updates

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Add topics WIP

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Finish adding topics

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Update locale

Signed-off-by: jolheiser <john.olheiser@gmail.com>

models/fixtures/repo_unit.yml

@@ -438,3 +438,17 @@
   type: 3
   config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}"
   created_unix: 946684810
+
+-
+  id: 64
+  repo_id: 44
+  type: 1
+  config: "{}"
+  created_unix: 946684810
+
+-
+  id: 65
+  repo_id: 45
+  type: 1
+  config: "{}"
+  created_unix: 946684810

models/fixtures/repository.yml

@@ -561,3 +561,29 @@
   num_issues: 0
   is_mirror: false
   status: 0
+
+-
+  id: 44
+  owner_id: 27
+  lower_name: template1
+  name: template1
+  is_private: false
+  is_template: true
+  num_stars: 0
+  num_forks: 0
+  num_issues: 0
+  is_mirror: false
+  status: 0
+
+-
+  id: 45
+  owner_id: 27
+  lower_name: template2
+  name: template2
+  is_private: false
+  is_template: true
+  num_stars: 0
+  num_forks: 0
+  num_issues: 0
+  is_mirror: false
+  status: 0

models/fixtures/user.yml

@@ -427,4 +427,19 @@
   num_repos: 1
   num_members: 0
   num_teams: 1
-  repo_admin_change_team_access: true
+  repo_admin_change_team_access: true
+
+-
+  id: 27
+  lower_name: user27
+  name: user27
+  full_name: User Twenty-Seven
+  email: user27@example.com
+  email_notifications_preference: enabled
+  passwd: 7d93daa0d1e6f2305cc8fa496847d61dc7320bb16262f9c55dd753480207234cdd96a93194e408341971742f4701772a025a # password
+  type: 0 # individual
+  salt: ZogKvWdyEx
+  is_admin: false
+  avatar: avatar27
+  avatar_email: user27@example.com
+  num_repos: 2

models/migrations/migrations.go

@@ -268,6 +268,8 @@ var migrations = []Migration{
 	NewMigration("add includes_all_repositories to teams", addTeamIncludesAllRepositories),
 	// v106 -> v107
 	NewMigration("add column `mode` to table watch", addModeColumnToWatch),
+	// v107 -> v108
+	NewMigration("Add template options to repository", addTemplateToRepo),
 }
 
 // Migrate database to current version

models/migrations/v107.go

@@ -0,0 +1,19 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"xorm.io/xorm"
+)
+
+func addTemplateToRepo(x *xorm.Engine) error {
+
+	type Repository struct {
+		IsTemplate bool  `xorm:"INDEX NOT NULL DEFAULT false"`
+		TemplateID int64 `xorm:"INDEX"`
+	}
+
+	return x.Sync2(new(Repository))
+}

models/repo.go

@@ -179,6 +179,9 @@ type Repository struct {
 	IsFork                          bool               `xorm:"INDEX NOT NULL DEFAULT false"`
 	ForkID                          int64              `xorm:"INDEX"`
 	BaseRepo                        *Repository        `xorm:"-"`
+	IsTemplate                      bool               `xorm:"INDEX NOT NULL DEFAULT false"`
+	TemplateID                      int64              `xorm:"INDEX"`
+	TemplateRepo                    *Repository        `xorm:"-"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
 	IndexerStatus                   *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@@ -351,6 +354,7 @@ func (repo *Repository) innerAPIFormat(e Engine, mode AccessMode, isParent bool)
 		FullName:                  repo.FullName(),
 		Description:               repo.Description,
 		Private:                   repo.IsPrivate,
+		Template:                  repo.IsTemplate,
 		Empty:                     repo.IsEmpty,
 		Archived:                  repo.IsArchived,
 		Size:                      int(repo.Size / 1024),
@@ -663,6 +667,27 @@ func (repo *Repository) getBaseRepo(e Engine) (err error) {
 	return err
 }
 
+// IsGenerated returns whether _this_ repository was generated from a template
+func (repo *Repository) IsGenerated() bool {
+	return repo.TemplateID != 0
+}
+
+// GetTemplateRepo populates repo.TemplateRepo for a generated repository and
+// returns an error on failure (NOTE: no error is returned for
+// non-generated repositories, and TemplateRepo will be left untouched)
+func (repo *Repository) GetTemplateRepo() (err error) {
+	return repo.getTemplateRepo(x)
+}
+
+func (repo *Repository) getTemplateRepo(e Engine) (err error) {
+	if !repo.IsGenerated() {
+		return nil
+	}
+
+	repo.TemplateRepo, err = getRepositoryByID(e, repo.TemplateID)
+	return err
+}
+
 func (repo *Repository) repoPath(e Engine) string {
 	return RepoPath(repo.mustOwnerName(e), repo.Name)
 }
@@ -1220,6 +1245,20 @@ type CreateRepoOptions struct {
 	Status      RepositoryStatus
 }
 
+// GenerateRepoOptions contains the template units to generate
+type GenerateRepoOptions struct {
+	Name        string
+	Description string
+	Private     bool
+	GitContent  bool
+	Topics      bool
+}
+
+// IsValid checks whether at least one option is chosen for generation
+func (gro GenerateRepoOptions) IsValid() bool {
+	return gro.GitContent || gro.Topics // or other items as they are added
+}
+
 func getRepoInitFile(tp, name string) ([]byte, error) {
 	cleanedName := strings.TrimLeft(path.Clean("/"+name), "/")
 	relPath := path.Join("options", tp, cleanedName)
@@ -1323,8 +1362,55 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
 	return nil
 }
 
-// InitRepository initializes README and .gitignore if needed.
-func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) {
+func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error {
+	commitTimeStr := time.Now().Format(time.RFC3339)
+	authorSig := repo.Owner.NewGitSig()
+
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+authorSig.Name,
+		"GIT_AUTHOR_EMAIL="+authorSig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_NAME="+authorSig.Name,
+		"GIT_COMMITTER_EMAIL="+authorSig.Email,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+
+	// Clone to temporary path and do the init commit.
+	templateRepoPath := templateRepo.repoPath(e)
+	_, stderr, err := process.GetManager().ExecDirEnv(
+		-1, "",
+		fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath),
+		env,
+		git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir,
+	)
+	if err != nil {
+		return fmt.Errorf("git clone: %v - %s", err, stderr)
+	}
+
+	if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
+		return fmt.Errorf("remove git dir: %v", err)
+	}
+
+	if err := git.InitRepository(tmpDir, false); err != nil {
+		return err
+	}
+
+	repoPath := repo.repoPath(e)
+	_, stderr, err = process.GetManager().ExecDirEnv(
+		-1, tmpDir,
+		fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
+		env,
+		git.GitExecutable, "remote", "add", "origin", repoPath,
+	)
+	if err != nil {
+		return fmt.Errorf("git remote add: %v - %s", err, stderr)
+	}
+
+	return initRepoCommit(tmpDir, repo.Owner)
+}
+
+func checkInitRepository(repoPath string) (err error) {
 	// Somehow the directory could exist.
 	if com.IsExist(repoPath) {
 		return fmt.Errorf("initRepository: path already exists: %s", repoPath)
@@ -1336,6 +1422,14 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C
 	} else if err = createDelegateHooks(repoPath); err != nil {
 		return fmt.Errorf("createDelegateHooks: %v", err)
 	}
+	return nil
+}
+
+// InitRepository initializes README and .gitignore if needed.
+func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) {
+	if err = checkInitRepository(repoPath); err != nil {
+		return err
+	}
 
 	tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
 
@@ -1376,6 +1470,37 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C
 	return nil
 }
 
+// generateRepository initializes repository from template
+func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
+	tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
+
+	if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
+		return fmt.Errorf("Failed to create dir %s: %v", tmpDir, err)
+	}
+
+	defer func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			log.Error("RemoveAll: %v", err)
+		}
+	}()
+
+	if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil {
+		return fmt.Errorf("generateRepoCommit: %v", err)
+	}
+
+	// re-fetch repo
+	if repo, err = getRepositoryByID(e, repo.ID); err != nil {
+		return fmt.Errorf("getRepositoryByID: %v", err)
+	}
+
+	repo.DefaultBranch = "master"
+	if err = updateRepository(e, repo, false); err != nil {
+		return fmt.Errorf("updateRepository: %v", err)
+	}
+
+	return nil
+}
+
 var (
 	reservedRepoNames    = []string{".", ".."}
 	reservedRepoPatterns = []string{"*.git", "*.wiki"}
@@ -2523,6 +2648,28 @@ func HasForkedRepo(ownerID, repoID int64) (*Repository, bool) {
 	return repo, has
 }
 
+// CopyLFS copies LFS data from one repo to another
+func CopyLFS(newRepo, oldRepo *Repository) error {
+	return copyLFS(x, newRepo, oldRepo)
+}
+
+func copyLFS(e Engine, newRepo, oldRepo *Repository) error {
+	var lfsObjects []*LFSMetaObject
+	if err := e.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
+		return err
+	}
+
+	for _, v := range lfsObjects {
+		v.ID = 0
+		v.RepositoryID = newRepo.ID
+		if _, err := e.Insert(v); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ForkRepository forks a repository
 func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (_ *Repository, err error) {
 	forkedRepo, err := oldRepo.GetUserFork(owner.ID)
@@ -2593,27 +2740,73 @@ func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (
 		log.Error("Failed to update size for repository: %v", err)
 	}
 
-	// Copy LFS meta objects in new session
-	sess2 := x.NewSession()
-	defer sess2.Close()
-	if err = sess2.Begin(); err != nil {
+	return repo, CopyLFS(repo, oldRepo)
+}
+
+// GenerateRepository generates a repository from a template
+func GenerateRepository(doer, owner *User, templateRepo *Repository, opts GenerateRepoOptions) (_ *Repository, err error) {
+	repo := &Repository{
+		OwnerID:       owner.ID,
+		Owner:         owner,
+		Name:          opts.Name,
+		LowerName:     strings.ToLower(opts.Name),
+		Description:   opts.Description,
+		IsPrivate:     opts.Private,
+		IsEmpty:       !opts.GitContent || templateRepo.IsEmpty,
+		IsFsckEnabled: templateRepo.IsFsckEnabled,
+		TemplateID:    templateRepo.ID,
+	}
+
+	createSess := x.NewSession()
+	defer createSess.Close()
+	if err = createSess.Begin(); err != nil {
+		return nil, err
+	}
+
+	if err = createRepository(createSess, doer, owner, repo); err != nil {
+		return nil, err
+	}
+
+	//Commit repo to get created repo ID
+	err = createSess.Commit()
+	if err != nil {
+		return nil, err
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err = sess.Begin(); err != nil {
 		return repo, err
 	}
 
-	var lfsObjects []*LFSMetaObject
-	if err = sess2.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
+	repoPath := RepoPath(owner.Name, repo.Name)
+	if err = checkInitRepository(repoPath); err != nil {
 		return repo, err
 	}
 
-	for _, v := range lfsObjects {
-		v.ID = 0
-		v.RepositoryID = repo.ID
-		if _, err = sess2.Insert(v); err != nil {
+	if opts.GitContent && !templateRepo.IsEmpty {
+		if err = generateRepository(sess, repo, templateRepo); err != nil {
 			return repo, err
 		}
+
+		if err = repo.updateSize(sess); err != nil {
+			return repo, fmt.Errorf("failed to update size for repository: %v", err)
+		}
+
+		if err = copyLFS(sess, repo, templateRepo); err != nil {
+			return repo, fmt.Errorf("failed to copy LFS: %v", err)
+		}
 	}
 
-	return repo, sess2.Commit()
+	if opts.Topics {
+		for _, topic := range templateRepo.Topics {
+			if _, err = addTopicByNameToRepo(sess, repo.ID, topic); err != nil {
+				return repo, err
+			}
+		}
+	}
+
+	return repo, sess.Commit()
 }
 
 // GetForks returns all the forks of the repository

models/repo_list.go

@@ -111,17 +111,18 @@ func (repos MirrorRepositoryList) LoadAttributes() error {
 
 // SearchRepoOptions holds the search options
 type SearchRepoOptions struct {
-	UserID      int64
-	UserIsAdmin bool
-	Keyword     string
-	OwnerID     int64
-	OrderBy     SearchOrderBy
-	Private     bool // Include private repositories in results
-	StarredByID int64
-	Page        int
-	IsProfile   bool
-	AllPublic   bool // Include also all public repositories
-	PageSize    int  // Can be smaller than or equal to setting.ExplorePagingNum
+	UserID          int64
+	UserIsAdmin     bool
+	Keyword         string
+	OwnerID         int64
+	PriorityOwnerID int64
+	OrderBy         SearchOrderBy
+	Private         bool // Include private repositories in results
+	StarredByID     int64
+	Page            int
+	IsProfile       bool
+	AllPublic       bool // Include also all public repositories
+	PageSize        int  // Can be smaller than or equal to setting.ExplorePagingNum
 	// None -> include collaborative AND non-collaborative
 	// True -> include just collaborative
 	// False -> incude just non-collaborative
@@ -130,6 +131,10 @@ type SearchRepoOptions struct {
 	// True -> include just forks
 	// False -> include just non-forks
 	Fork util.OptionalBool
+	// None -> include templates AND non-templates
+	// True -> include just templates
+	// False -> include just non-templates
+	Template util.OptionalBool
 	// None -> include mirrors AND non-mirrors
 	// True -> include just mirrors
 	// False -> include just non-mirrors
@@ -190,6 +195,10 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
 		cond = cond.And(accessCond)
 	}
 
+	if opts.Template != util.OptionalBoolNone {
+		cond = cond.And(builder.Eq{"is_template": opts.Template == util.OptionalBoolTrue})
+	}
+
 	// Restrict to starred repositories
 	if opts.StarredByID > 0 {
 		cond = cond.And(builder.In("id", builder.Select("repo_id").From("star").Where(builder.Eq{"uid": opts.StarredByID})))
@@ -266,6 +275,10 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
 		opts.OrderBy = SearchOrderByAlphabetically
 	}
 
+	if opts.PriorityOwnerID > 0 {
+		opts.OrderBy = SearchOrderBy(fmt.Sprintf("CASE WHEN owner_id = %d THEN 0 ELSE owner_id END, %s", opts.PriorityOwnerID, opts.OrderBy))
+	}
+
 	sess := x.NewSession()
 	defer sess.Close()
 
@@ -308,11 +321,15 @@ func accessibleRepositoryCondition(userID int64) builder.Cond {
 				builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))),
 		),
 		// 2. Be able to see all repositories that we have access to
-		builder.In("`repository`.id", builder.Select("repo_id").
-			From("`access`").
-			Where(builder.And(
-				builder.Eq{"user_id": userID},
-				builder.Gt{"mode": int(AccessModeNone)}))),
+		builder.Or(
+			builder.In("`repository`.id", builder.Select("repo_id").
+				From("`access`").
+				Where(builder.And(
+					builder.Eq{"user_id": userID},
+					builder.Gt{"mode": int(AccessModeNone)}))),
+			builder.In("`repository`.id", builder.Select("id").
+				From("`repository`").
+				Where(builder.Eq{"owner_id": userID}))),
 		// 3. Be able to see all repositories that we are in a team
 		builder.In("`repository`.id", builder.Select("`team_repo`.repo_id").
 			From("team_repo").

models/repo_list_test.go

@@ -174,10 +174,10 @@ func TestSearchRepository(t *testing.T) {
 			opts:  &SearchRepoOptions{Keyword: "big_test_", Page: 1, PageSize: 10, Private: true, AllPublic: true, Collaborate: util.OptionalBoolFalse},
 			count: 14},
 		{name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative",
-			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, AllPublic: true},
+			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, AllPublic: true, Template: util.OptionalBoolFalse},
 			count: 22},
 		{name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative",
-			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true},
+			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true, Template: util.OptionalBoolFalse},
 			count: 28},
 		{name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborativeByName",
 			opts:  &SearchRepoOptions{Keyword: "test", Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true},
@@ -186,8 +186,11 @@ func TestSearchRepository(t *testing.T) {
 			opts:  &SearchRepoOptions{Keyword: "test", Page: 1, PageSize: 10, OwnerID: 18, Private: true, AllPublic: true},
 			count: 13},
 		{name: "AllPublic/PublicRepositoriesOfOrganization",
-			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse},
+			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse, Template: util.OptionalBoolFalse},
 			count: 22},
+		{name: "AllTemplates",
+			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, Template: util.OptionalBoolTrue},
+			count: 2},
 	}
 
 	for _, testCase := range testCases {

models/user_test.go

@@ -153,7 +153,7 @@ func TestSearchUsers(t *testing.T) {
 	}
 
 	testUserSuccess(&SearchUserOptions{OrderBy: "id ASC", Page: 1},
-		[]int64{1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 24})
+		[]int64{1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 24, 27})
 
 	testUserSuccess(&SearchUserOptions{Page: 1, IsActive: util.OptionalBoolFalse},
 		[]int64{9})