finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rework repository archive (#14723)

Browse source 
* Use storage to store archive files

* Fix backend lint

* Add archiver table on database

* Finish archive download

* Fix test

* Add database migrations

* Add status for archiver

* Fix lint

* Add queue

* Add doctor to check and delete old archives

* Improve archive queue

* Fix tests

* improve archive storage

* Delete repo archives

* Add missing fixture

* fix fixture

* Fix fixture

* Fix test

* Fix archiver cleaning

* Fix bug

* Add docs for repository archive storage

* remove repo-archive configuration

* Fix test

* Fix test

* Fix lint

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
Lunny Xiao 2021-06-24 05:12:38 +08:00 • committed by GitHub
parent c9c7afda1a
commit b223d36195
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 648 additions and 480 deletions
Download patch file Download diff file Expand all files Collapse all files

3
routers/api/v1/repo/file.go
View file

@ -18,6 +18,7 @@ import (
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/common"
"code.gitea.io/gitea/routers/web/repo"
)
// GetRawFile get a file by path on a repository
@ -126,7 +127,7 @@ func GetArchive(ctx *context.APIContext) {
ctx.Repo.GitRepo = gitRepo
defer gitRepo.Close()
common.Download(ctx.Context)
repo.Download(ctx.Context)
}
// GetEditorconfig get editor config of a repository

26
routers/common/repo.go
View file

@ -7,7 +7,6 @@ package common
import (
"fmt"
"io"
"net/http"
"path"
"path/filepath"
"strings"
@ -19,7 +18,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/services/archiver"
)
// ServeBlob download a git.Blob
@ -41,30 +39,6 @@ func ServeBlob(ctx *context.Context, blob *git.Blob) error {
return ServeData(ctx, ctx.Repo.TreePath, blob.Size(), dataRc)
}
// Download an archive of a repository
func Download(ctx *context.Context) {
uri := ctx.Params("*")
aReq := archiver.DeriveRequestFrom(ctx, uri)
if aReq == nil {
ctx.Error(http.StatusNotFound)
return
}
downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
complete := aReq.IsComplete()
if !complete {
aReq = archiver.ArchiveRepository(aReq)
complete = aReq.WaitForCompletion(ctx)
}
if complete {
ctx.ServeFile(aReq.GetArchivePath(), downloadName)
} else {
ctx.Error(http.StatusNotFound)
}
}
// ServeData download file from io.Reader
func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error {
buf := make([]byte, 1024)

4
routers/init.go
View file

@ -33,6 +33,7 @@ import (
"code.gitea.io/gitea/routers/common"
"code.gitea.io/gitea/routers/private"
web_routers "code.gitea.io/gitea/routers/web"
"code.gitea.io/gitea/services/archiver"
"code.gitea.io/gitea/services/auth"
"code.gitea.io/gitea/services/mailer"
mirror_service "code.gitea.io/gitea/services/mirror"
@ -63,6 +64,9 @@ func NewServices() {
mailer.NewContext()
_ = cache.NewContext()
notification.NewContext()
if err := archiver.Init(); err != nil {
log.Fatal("archiver init failed: %v", err)
}
}
// GlobalInit is for global configuration reload-able.

122
routers/web/repo/repo.go
View file

@ -15,8 +15,10 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/web"
archiver_service "code.gitea.io/gitea/services/archiver"
"code.gitea.io/gitea/services/forms"
@ -364,25 +366,123 @@ func RedirectDownload(ctx *context.Context) {
ctx.Error(http.StatusNotFound)
}
// InitiateDownload will enqueue an archival request, as needed. It may submit
// a request that's already in-progress, but the archiver service will just
// kind of drop it on the floor if this is the case.
func InitiateDownload(ctx *context.Context) {
// Download an archive of a repository
func Download(ctx *context.Context) {
uri := ctx.Params("*")
aReq := archiver_service.DeriveRequestFrom(ctx, uri)
aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri)
if err != nil {
ctx.ServerError("archiver_service.NewRequest", err)
return
}
if aReq == nil {
ctx.Error(http.StatusNotFound)
return
}
complete := aReq.IsComplete()
if !complete {
aReq = archiver_service.ArchiveRepository(aReq)
complete, _ = aReq.TimedWaitForCompletion(ctx, 2*time.Second)
archiver, err := models.GetRepoArchiver(models.DefaultDBContext(), aReq.RepoID, aReq.Type, aReq.CommitID)
if err != nil {
ctx.ServerError("models.GetRepoArchiver", err)
return
}
if archiver != nil && archiver.Status == models.RepoArchiverReady {
download(ctx, aReq.GetArchiveName(), archiver)
return
}
if err := archiver_service.StartArchive(aReq); err != nil {
ctx.ServerError("archiver_service.StartArchive", err)
return
}
var times int
var t = time.NewTicker(time.Second * 1)
defer t.Stop()
for {
select {
case <-graceful.GetManager().HammerContext().Done():
log.Warn("exit archive download because system stop")
return
case <-t.C:
if times > 20 {
ctx.ServerError("wait download timeout", nil)
return
}
times++
archiver, err = models.GetRepoArchiver(models.DefaultDBContext(), aReq.RepoID, aReq.Type, aReq.CommitID)
if err != nil {
ctx.ServerError("archiver_service.StartArchive", err)
return
}
if archiver != nil && archiver.Status == models.RepoArchiverReady {
download(ctx, aReq.GetArchiveName(), archiver)
return
}
}
}
}
func download(ctx *context.Context, archiveName string, archiver *models.RepoArchiver) {
downloadName := ctx.Repo.Repository.Name + "-" + archiveName
rPath, err := archiver.RelativePath()
if err != nil {
ctx.ServerError("archiver.RelativePath", err)
return
}
if setting.RepoArchive.ServeDirect {
//If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.URL(rPath, downloadName)
if u != nil && err == nil {
ctx.Redirect(u.String())
return
}
}
//If we have matched and access to release or issue
fr, err := storage.RepoArchives.Open(rPath)
if err != nil {
ctx.ServerError("Open", err)
return
}
defer fr.Close()
ctx.ServeStream(fr, downloadName)
}
// InitiateDownload will enqueue an archival request, as needed. It may submit
// a request that's already in-progress, but the archiver service will just
// kind of drop it on the floor if this is the case.
func InitiateDownload(ctx *context.Context) {
uri := ctx.Params("*")
aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri)
if err != nil {
ctx.ServerError("archiver_service.NewRequest", err)
return
}
if aReq == nil {
ctx.Error(http.StatusNotFound)
return
}
archiver, err := models.GetRepoArchiver(models.DefaultDBContext(), aReq.RepoID, aReq.Type, aReq.CommitID)
if err != nil {
ctx.ServerError("archiver_service.StartArchive", err)
return
}
if archiver == nil || archiver.Status != models.RepoArchiverReady {
if err := archiver_service.StartArchive(aReq); err != nil {
ctx.ServerError("archiver_service.StartArchive", err)
return
}
}
var completed bool
if archiver != nil && archiver.Status == models.RepoArchiverReady {
completed = true
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"complete": complete,
"complete": completed,
})
}

3
routers/web/web.go
View file

@ -22,7 +22,6 @@ import (
"code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/api/v1/misc"
"code.gitea.io/gitea/routers/common"
"code.gitea.io/gitea/routers/web/admin"
"code.gitea.io/gitea/routers/web/dev"
"code.gitea.io/gitea/routers/web/events"
@ -888,7 +887,7 @@ func RegisterRoutes(m *web.Route) {
}, context.RepoRef(), repo.MustBeNotEmpty, context.RequireRepoReaderOr(models.UnitTypeCode))
m.Group("/archive", func() {
m.Get("/*", common.Download)
m.Get("/*", repo.Download)
m.Post("/*", repo.InitiateDownload)
}, repo.MustBeNotEmpty, reqRepoCodeReader)