Rework repository archive (#14723)

* Use storage to store archive files

* Fix backend lint

* Add archiver table on database

* Finish archive download

* Fix test

* Add database migrations

* Add status for archiver

* Fix lint

* Add queue

* Add doctor to check and delete old archives

* Improve archive queue

* Fix tests

* improve archive storage

* Delete repo archives

* Add missing fixture

* fix fixture

* Fix fixture

* Fix test

* Fix archiver cleaning

* Fix bug

* Add docs for repository archive storage

* remove repo-archive configuration

* Fix test

* Fix test

* Fix lint

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

modules/context/context.go

@@ -380,6 +380,21 @@ func (ctx *Context) ServeFile(file string, names ...string) {
 	http.ServeFile(ctx.Resp, ctx.Req, file)
 }
 
+// ServeStream serves file via io stream
+func (ctx *Context) ServeStream(rd io.Reader, name string) {
+	ctx.Resp.Header().Set("Content-Description", "File Transfer")
+	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+	ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
+	ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
+	ctx.Resp.Header().Set("Expires", "0")
+	ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
+	ctx.Resp.Header().Set("Pragma", "public")
+	_, err := io.Copy(ctx.Resp, rd)
+	if err != nil {
+		ctx.ServerError("Download file failed", err)
+	}
+}
+
 // Error returned an error to web browser
 func (ctx *Context) Error(status int, contents ...string) {
 	var v = http.StatusText(status)

modules/doctor/checkOldArchives.go

@@ -0,0 +1,59 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
+)
+
+func checkOldArchives(logger log.Logger, autofix bool) error {
+	numRepos := 0
+	numReposUpdated := 0
+	err := iterateRepositories(func(repo *models.Repository) error {
+		if repo.IsEmpty {
+			return nil
+		}
+
+		p := filepath.Join(repo.RepoPath(), "archives")
+		isDir, err := util.IsDir(p)
+		if err != nil {
+			log.Warn("check if %s is directory failed: %v", p, err)
+		}
+		if isDir {
+			numRepos++
+			if autofix {
+				if err := os.RemoveAll(p); err == nil {
+					numReposUpdated++
+				} else {
+					log.Warn("remove %s failed: %v", p, err)
+				}
+			}
+		}
+		return nil
+	})
+
+	if autofix {
+		logger.Info("%d / %d old archives in repository deleted", numReposUpdated, numRepos)
+	} else {
+		logger.Info("%d old archives in repository need to be deleted", numRepos)
+	}
+
+	return err
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check old archives",
+		Name:      "check-old-archives",
+		IsDefault: false,
+		Run:       checkOldArchives,
+		Priority:  7,
+	})
+}

modules/git/repo_archive.go

@@ -8,6 +8,7 @@ package git
 import (
 	"context"
 	"fmt"
+	"io"
 	"path/filepath"
 	"strings"
 )
@@ -33,32 +34,28 @@ func (a ArchiveType) String() string {
 	return "unknown"
 }
 
-// CreateArchiveOpts represents options for creating an archive
-type CreateArchiveOpts struct {
-	Format ArchiveType
-	Prefix bool
-}
-
 // CreateArchive create archive content to the target path
-func (c *Commit) CreateArchive(ctx context.Context, target string, opts CreateArchiveOpts) error {
-	if opts.Format.String() == "unknown" {
-		return fmt.Errorf("unknown format: %v", opts.Format)
+func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, target io.Writer, usePrefix bool, commitID string) error {
+	if format.String() == "unknown" {
+		return fmt.Errorf("unknown format: %v", format)
 	}
 
 	args := []string{
 		"archive",
 	}
-	if opts.Prefix {
-		args = append(args, "--prefix="+filepath.Base(strings.TrimSuffix(c.repo.Path, ".git"))+"/")
+	if usePrefix {
+		args = append(args, "--prefix="+filepath.Base(strings.TrimSuffix(repo.Path, ".git"))+"/")
 	}
 
 	args = append(args,
-		"--format="+opts.Format.String(),
-		"-o",
-		target,
-		c.ID.String(),
+		"--format="+format.String(),
+		commitID,
 	)
 
-	_, err := NewCommandContext(ctx, args...).RunInDir(c.repo.Path)
-	return err
+	var stderr strings.Builder
+	err := NewCommandContext(ctx, args...).RunInDirPipeline(repo.Path, target, &stderr)
+	if err != nil {
+		return ConcatenateError(err, stderr.String())
+	}
+	return nil
 }

modules/setting/repository.go

@@ -251,6 +251,10 @@ var (
 	}
 	RepoRootPath string
 	ScriptType   = "bash"
+
+	RepoArchive = struct {
+		Storage
+	}{}
 )
 
 func newRepository() {
@@ -328,4 +332,6 @@ func newRepository() {
 	if !filepath.IsAbs(Repository.Upload.TempPath) {
 		Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath)
 	}
+
+	RepoArchive.Storage = getStorage("repo-archive", "", nil)
 }

modules/setting/storage.go

@@ -43,6 +43,10 @@ func getStorage(name, typ string, targetSec *ini.Section) Storage {
 	sec.Key("MINIO_LOCATION").MustString("us-east-1")
 	sec.Key("MINIO_USE_SSL").MustBool(false)
 
+	if targetSec == nil {
+		targetSec, _ = Cfg.NewSection(name)
+	}
+
 	var storage Storage
 	storage.Section = targetSec
 	storage.Type = typ

modules/storage/storage.go

@@ -114,6 +114,9 @@ var (
 	Avatars ObjectStorage
 	// RepoAvatars represents repository avatars storage
 	RepoAvatars ObjectStorage
+
+	// RepoArchives represents repository archives storage
+	RepoArchives ObjectStorage
 )
 
 // Init init the stoarge
@@ -130,7 +133,11 @@ func Init() error {
 		return err
 	}
 
-	return initLFS()
+	if err := initLFS(); err != nil {
+		return err
+	}
+
+	return initRepoArchives()
 }
 
 // NewStorage takes a storage type and some config and returns an ObjectStorage or an error
@@ -169,3 +176,9 @@ func initRepoAvatars() (err error) {
 	RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage)
 	return
 }
+
+func initRepoArchives() (err error) {
+	log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
+	RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
+	return
+}