From 613e5387c5933849222207c5ebb9996297cd4046 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Tue, 2 Apr 2024 16:34:57 +0200 Subject: [PATCH 1/4] Count downloads for tag archives --- models/repo/archive_download_count.go | 87 +++++++++++++++++++ models/repo/archive_download_count_test.go | 65 ++++++++++++++ models/repo/archiver.go | 1 + models/repo/release.go | 70 ++++++++++----- modules/git/tag.go | 16 ++-- modules/structs/release.go | 7 +- modules/structs/repo_tag.go | 34 +++++--- routers/api/v1/repo/file.go | 2 +- routers/api/v1/repo/tag.go | 27 ++++++ routers/web/repo/release.go | 11 +++ routers/web/repo/repo.go | 20 ++++- services/convert/convert.go | 28 +++--- services/convert/release.go | 33 +++---- services/doctor/dbconsistency.go | 3 + services/release/release.go | 5 ++ services/repository/archiver/archiver.go | 27 ++++-- services/repository/archiver/archiver_test.go | 24 ++--- services/repository/delete.go | 1 + templates/repo/release/list.tmpl | 5 +- templates/swagger/v1_json.tmpl | 26 ++++++ tests/integration/api_releases_test.go | 36 ++++++++ tests/integration/api_repo_tags_test.go | 36 ++++++++ 22 files changed, 469 insertions(+), 95 deletions(-) create mode 100644 models/repo/archive_download_count.go create mode 100644 models/repo/archive_download_count_test.go diff --git a/models/repo/archive_download_count.go b/models/repo/archive_download_count.go new file mode 100644 index 000000000..6300307a3 --- /dev/null +++ b/models/repo/archive_download_count.go @@ -0,0 +1,87 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repo + +import ( + "context" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" +) + +// RepoArchiveDownloadCount counts all archive downloads for a tag +type RepoArchiveDownloadCount struct { //nolint:revive + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"index unique(s)"` + ReleaseID int64 `xorm:"index unique(s)"` + Type git.ArchiveType `xorm:"unique(s)"` + Count int64 +} + +func init() { + db.RegisterModel(new(RepoArchiveDownloadCount)) +} + +// CountArchiveDownload adds one download the the given archive +func CountArchiveDownload(ctx context.Context, repoID, releaseID int64, tp git.ArchiveType) error { + updateCount, err := db.GetEngine(ctx).Where("repo_id = ?", repoID).And("release_id = ?", releaseID).And("`type` = ?", tp).Incr("count").Update(new(RepoArchiveDownloadCount)) + if err != nil { + return err + } + + if updateCount != 0 { + // The count was updated, so we can exit + return nil + } + + // The archive does not esxists in the databse, so let's add it + newCounter := &RepoArchiveDownloadCount{ + RepoID: repoID, + ReleaseID: releaseID, + Type: tp, + Count: 1, + } + + _, err = db.GetEngine(ctx).Insert(newCounter) + return err +} + +// GetArchiveDownloadCount returns the download count of a tag +func GetArchiveDownloadCount(ctx context.Context, repoID, releaseID int64) (*api.TagArchiveDownloadCount, error) { + downloadCountList := make([]RepoArchiveDownloadCount, 0) + err := db.GetEngine(ctx).Where("repo_id = ?", repoID).And("release_id = ?", releaseID).Find(&downloadCountList) + if err != nil { + return nil, err + } + + tagCounter := new(api.TagArchiveDownloadCount) + + for _, singleCount := range downloadCountList { + switch singleCount.Type { + case git.ZIP: + tagCounter.Zip = singleCount.Count + case git.TARGZ: + tagCounter.TarGz = singleCount.Count + } + } + + return tagCounter, nil +} + +// GetDownloadCountForTagName returns the download count of a tag with the given name +func GetArchiveDownloadCountForTagName(ctx context.Context, repoID int64, tagName string) (*api.TagArchiveDownloadCount, error) { + release, err := GetRelease(ctx, repoID, tagName) + if err != nil { + return nil, err + } + + return GetArchiveDownloadCount(ctx, repoID, release.ID) +} + +// DeleteArchiveDownloadCountForRelease deletes the release from the repo_archive_download_count table +func DeleteArchiveDownloadCountForRelease(ctx context.Context, releaseID int64) error { + _, err := db.GetEngine(ctx).Delete(&RepoArchiveDownloadCount{ReleaseID: releaseID}) + return err +} diff --git a/models/repo/archive_download_count_test.go b/models/repo/archive_download_count_test.go new file mode 100644 index 000000000..53bdf9a1e --- /dev/null +++ b/models/repo/archive_download_count_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repo_test + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepoArchiveDownloadCount(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + release, err := repo_model.GetReleaseByID(db.DefaultContext, 1) + require.NoError(t, err) + + // We have no count, so it should return 0 + downloadCount, err := repo_model.GetArchiveDownloadCount(db.DefaultContext, release.RepoID, release.ID) + require.NoError(t, err) + assert.Equal(t, int64(0), downloadCount.Zip) + assert.Equal(t, int64(0), downloadCount.TarGz) + + // Set the TarGz counter to 1 + err = repo_model.CountArchiveDownload(db.DefaultContext, release.RepoID, release.ID, git.TARGZ) + require.NoError(t, err) + + downloadCount, err = repo_model.GetArchiveDownloadCountForTagName(db.DefaultContext, release.RepoID, release.TagName) + require.NoError(t, err) + assert.Equal(t, int64(0), downloadCount.Zip) + assert.Equal(t, int64(1), downloadCount.TarGz) + + // Set the TarGz counter to 2 + err = repo_model.CountArchiveDownload(db.DefaultContext, release.RepoID, release.ID, git.TARGZ) + require.NoError(t, err) + + downloadCount, err = repo_model.GetArchiveDownloadCountForTagName(db.DefaultContext, release.RepoID, release.TagName) + require.NoError(t, err) + assert.Equal(t, int64(0), downloadCount.Zip) + assert.Equal(t, int64(2), downloadCount.TarGz) + + // Set the Zip counter to 1 + err = repo_model.CountArchiveDownload(db.DefaultContext, release.RepoID, release.ID, git.ZIP) + require.NoError(t, err) + + downloadCount, err = repo_model.GetArchiveDownloadCountForTagName(db.DefaultContext, release.RepoID, release.TagName) + require.NoError(t, err) + assert.Equal(t, int64(1), downloadCount.Zip) + assert.Equal(t, int64(2), downloadCount.TarGz) + + // Delete the count + err = repo_model.DeleteArchiveDownloadCountForRelease(db.DefaultContext, release.ID) + require.NoError(t, err) + + downloadCount, err = repo_model.GetArchiveDownloadCountForTagName(db.DefaultContext, release.RepoID, release.TagName) + require.NoError(t, err) + assert.Equal(t, int64(0), downloadCount.Zip) + assert.Equal(t, int64(0), downloadCount.TarGz) +} diff --git a/models/repo/archiver.go b/models/repo/archiver.go index 14ffa1d89..3f05fcf75 100644 --- a/models/repo/archiver.go +++ b/models/repo/archiver.go @@ -35,6 +35,7 @@ type RepoArchiver struct { //revive:disable-line:exported Status ArchiverStatus CommitID string `xorm:"VARCHAR(64) unique(s)"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX NOT NULL created"` + ReleaseID int64 `xorm:"-"` } func init() { diff --git a/models/repo/release.go b/models/repo/release.go index a9f65f6c3..3168bdaae 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -65,28 +65,29 @@ func (err ErrReleaseNotExist) Unwrap() error { // Release represents a release of repository. type Release struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX UNIQUE(n)"` - Repo *Repository `xorm:"-"` - PublisherID int64 `xorm:"INDEX"` - Publisher *user_model.User `xorm:"-"` - TagName string `xorm:"INDEX UNIQUE(n)"` - OriginalAuthor string - OriginalAuthorID int64 `xorm:"index"` - LowerTagName string - Target string - TargetBehind string `xorm:"-"` // to handle non-existing or empty target - Title string - Sha1 string `xorm:"VARCHAR(64)"` - NumCommits int64 - NumCommitsBehind int64 `xorm:"-"` - Note string `xorm:"TEXT"` - RenderedNote template.HTML `xorm:"-"` - IsDraft bool `xorm:"NOT NULL DEFAULT false"` - IsPrerelease bool `xorm:"NOT NULL DEFAULT false"` - IsTag bool `xorm:"NOT NULL DEFAULT false"` // will be true only if the record is a tag and has no related releases - Attachments []*Attachment `xorm:"-"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX UNIQUE(n)"` + Repo *Repository `xorm:"-"` + PublisherID int64 `xorm:"INDEX"` + Publisher *user_model.User `xorm:"-"` + TagName string `xorm:"INDEX UNIQUE(n)"` + OriginalAuthor string + OriginalAuthorID int64 `xorm:"index"` + LowerTagName string + Target string + TargetBehind string `xorm:"-"` // to handle non-existing or empty target + Title string + Sha1 string `xorm:"VARCHAR(64)"` + NumCommits int64 + NumCommitsBehind int64 `xorm:"-"` + Note string `xorm:"TEXT"` + RenderedNote template.HTML `xorm:"-"` + IsDraft bool `xorm:"NOT NULL DEFAULT false"` + IsPrerelease bool `xorm:"NOT NULL DEFAULT false"` + IsTag bool `xorm:"NOT NULL DEFAULT false"` // will be true only if the record is a tag and has no related releases + Attachments []*Attachment `xorm:"-"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX"` + ArchiveDownloadCount *structs.TagArchiveDownloadCount `xorm:"-"` } func init() { @@ -112,9 +113,22 @@ func (r *Release) LoadAttributes(ctx context.Context) error { } } } + + err = r.LoadArchiveDownloadCount(ctx) + if err != nil { + return err + } + return GetReleaseAttachments(ctx, r) } +// LoadArchiveDownloadCount loads the download count for the source archives +func (r *Release) LoadArchiveDownloadCount(ctx context.Context) error { + var err error + r.ArchiveDownloadCount, err = GetArchiveDownloadCount(ctx, r.RepoID, r.ID) + return err +} + // APIURL the api url for a release. release must have attributes loaded func (r *Release) APIURL() string { return r.Repo.APIURL() + "/releases/" + strconv.FormatInt(r.ID, 10) @@ -447,6 +461,18 @@ func PushUpdateDeleteTagsContext(ctx context.Context, repo *Repository, tags []s lowerTags = append(lowerTags, strings.ToLower(tag)) } + for _, tag := range tags { + release, err := GetRelease(ctx, repo.ID, tag) + if err != nil { + return fmt.Errorf("GetRelease: %w", err) + } + + err = DeleteArchiveDownloadCountForRelease(ctx, release.ID) + if err != nil { + return fmt.Errorf("DeleteTagArchiveDownloadCount: %w", err) + } + } + if _, err := db.GetEngine(ctx). Where("repo_id = ? AND is_tag = ?", repo.ID, true). In("lower_tag_name", lowerTags). diff --git a/modules/git/tag.go b/modules/git/tag.go index 1fe4c16b5..04f50e8db 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" ) @@ -20,13 +21,14 @@ const ( // Tag represents a Git tag. type Tag struct { - Name string - ID ObjectID - Object ObjectID // The id of this commit object - Type string - Tagger *Signature - Message string - Signature *ObjectSignature + Name string + ID ObjectID + Object ObjectID // The id of this commit object + Type string + Tagger *Signature + Message string + Signature *ObjectSignature + ArchiveDownloadCount *api.TagArchiveDownloadCount } // Commit return the commit of the tag reference diff --git a/modules/structs/release.go b/modules/structs/release.go index c7378645c..02c68af18 100644 --- a/modules/structs/release.go +++ b/modules/structs/release.go @@ -24,9 +24,10 @@ type Release struct { // swagger:strfmt date-time CreatedAt time.Time `json:"created_at"` // swagger:strfmt date-time - PublishedAt time.Time `json:"published_at"` - Publisher *User `json:"author"` - Attachments []*Attachment `json:"assets"` + PublishedAt time.Time `json:"published_at"` + Publisher *User `json:"author"` + Attachments []*Attachment `json:"assets"` + ArchiveDownloadCount *TagArchiveDownloadCount `json:"archive_download_count"` } // CreateReleaseOption options when creating a release diff --git a/modules/structs/repo_tag.go b/modules/structs/repo_tag.go index 4a7d89528..961ca4e53 100644 --- a/modules/structs/repo_tag.go +++ b/modules/structs/repo_tag.go @@ -5,23 +5,25 @@ package structs // Tag represents a repository tag type Tag struct { - Name string `json:"name"` - Message string `json:"message"` - ID string `json:"id"` - Commit *CommitMeta `json:"commit"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` + Name string `json:"name"` + Message string `json:"message"` + ID string `json:"id"` + Commit *CommitMeta `json:"commit"` + ZipballURL string `json:"zipball_url"` + TarballURL string `json:"tarball_url"` + ArchiveDownloadCount *TagArchiveDownloadCount `json:"archive_download_count"` } // AnnotatedTag represents an annotated tag type AnnotatedTag struct { - Tag string `json:"tag"` - SHA string `json:"sha"` - URL string `json:"url"` - Message string `json:"message"` - Tagger *CommitUser `json:"tagger"` - Object *AnnotatedTagObject `json:"object"` - Verification *PayloadCommitVerification `json:"verification"` + Tag string `json:"tag"` + SHA string `json:"sha"` + URL string `json:"url"` + Message string `json:"message"` + Tagger *CommitUser `json:"tagger"` + Object *AnnotatedTagObject `json:"object"` + Verification *PayloadCommitVerification `json:"verification"` + ArchiveDownloadCount *TagArchiveDownloadCount `json:"archive_download_count"` } // AnnotatedTagObject contains meta information of the tag object @@ -38,3 +40,9 @@ type CreateTagOption struct { Message string `json:"message"` Target string `json:"target"` } + +// TagArchiveDownloadCount counts how many times a archive was downloaded +type TagArchiveDownloadCount struct { + Zip int64 `json:"zip"` + TarGz int64 `json:"tar_gz"` +} diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 712cba745..70eb3fbc2 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -302,7 +302,7 @@ func GetArchive(ctx *context.APIContext) { func archiveDownload(ctx *context.APIContext) { uri := ctx.Params("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + aReq, err := archiver_service.NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, "unknown archive format", err) diff --git a/routers/api/v1/repo/tag.go b/routers/api/v1/repo/tag.go index 84ec3dd91..b498f0e79 100644 --- a/routers/api/v1/repo/tag.go +++ b/routers/api/v1/repo/tag.go @@ -60,6 +60,12 @@ func ListTags(ctx *context.APIContext) { apiTags := make([]*api.Tag, len(tags)) for i := range tags { + tags[i].ArchiveDownloadCount, err = repo_model.GetArchiveDownloadCountForTagName(ctx, ctx.Repo.Repository.ID, tags[i].Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetTagArchiveDownloadCountForName", err) + return + } + apiTags[i] = convert.ToTag(ctx.Repo.Repository, tags[i]) } @@ -111,6 +117,13 @@ func GetAnnotatedTag(ctx *context.APIContext) { if err != nil { ctx.Error(http.StatusBadRequest, "GetAnnotatedTag", err) } + + tag.ArchiveDownloadCount, err = repo_model.GetArchiveDownloadCountForTagName(ctx, ctx.Repo.Repository.ID, tag.Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetTagArchiveDownloadCountForName", err) + return + } + ctx.JSON(http.StatusOK, convert.ToAnnotatedTag(ctx, ctx.Repo.Repository, tag, commit)) } } @@ -150,6 +163,13 @@ func GetTag(ctx *context.APIContext) { ctx.NotFound(tagName) return } + + tag.ArchiveDownloadCount, err = repo_model.GetArchiveDownloadCountForTagName(ctx, ctx.Repo.Repository.ID, tag.Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetTagArchiveDownloadCountForName", err) + return + } + ctx.JSON(http.StatusOK, convert.ToTag(ctx.Repo.Repository, tag)) } @@ -218,6 +238,13 @@ func CreateTag(ctx *context.APIContext) { ctx.InternalServerError(err) return } + + tag.ArchiveDownloadCount, err = repo_model.GetArchiveDownloadCountForTagName(ctx, ctx.Repo.Repository.ID, tag.Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetTagArchiveDownloadCountForName", err) + return + } + ctx.JSON(http.StatusCreated, convert.ToTag(ctx.Repo.Repository, tag)) } diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go index 54e9aed20..388fb6cb9 100644 --- a/routers/web/repo/release.go +++ b/routers/web/repo/release.go @@ -127,6 +127,11 @@ func getReleaseInfos(ctx *context.Context, opts *repo_model.FindReleasesOptions) return nil, err } + err = r.LoadArchiveDownloadCount(ctx) + if err != nil { + return nil, err + } + if !r.IsDraft { if err := calReleaseNumCommitsBehind(ctx.Repo, r, countCache); err != nil { return nil, err @@ -355,6 +360,12 @@ func SingleRelease(ctx *context.Context) { ctx.Data["Title"] = release.Title } + err = release.LoadArchiveDownloadCount(ctx) + if err != nil { + ctx.ServerError("LoadArchiveDownloadCount", err) + return + } + ctx.Data["Releases"] = releases ctx.HTML(http.StatusOK, tplReleasesList) } diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index b2458a561..a5f6c02c7 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -456,7 +456,7 @@ func RedirectDownload(ctx *context.Context) { // Download an archive of a repository func Download(ctx *context.Context) { uri := ctx.Params("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + aReq, err := archiver_service.NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, err.Error()) @@ -485,6 +485,14 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.RepoArchives.URL(rPath, downloadName) if u != nil && err == nil { + if archiver.ReleaseID != 0 { + err = repo_model.CountArchiveDownload(ctx, ctx.Repo.Repository.ID, archiver.ReleaseID, archiver.Type) + if err != nil { + ctx.ServerError("CountArchiveDownload", err) + return + } + } + ctx.Redirect(u.String()) return } @@ -498,6 +506,14 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep } defer fr.Close() + if archiver.ReleaseID != 0 { + err = repo_model.CountArchiveDownload(ctx, ctx.Repo.Repository.ID, archiver.ReleaseID, archiver.Type) + if err != nil { + ctx.ServerError("CountArchiveDownload", err) + return + } + } + ctx.ServeContent(fr, &context.ServeHeaderOptions{ Filename: downloadName, LastModified: archiver.CreatedUnix.AsLocalTime(), @@ -509,7 +525,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep // kind of drop it on the floor if this is the case. func InitiateDownload(ctx *context.Context) { uri := ctx.Params("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + aReq, err := archiver_service.NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) if err != nil { ctx.ServerError("archiver_service.NewRequest", err) return diff --git a/services/convert/convert.go b/services/convert/convert.go index dd2239458..5b06b9b1a 100644 --- a/services/convert/convert.go +++ b/services/convert/convert.go @@ -171,12 +171,13 @@ func ToBranchProtection(ctx context.Context, bp *git_model.ProtectedBranch) *api // ToTag convert a git.Tag to an api.Tag func ToTag(repo *repo_model.Repository, t *git.Tag) *api.Tag { return &api.Tag{ - Name: t.Name, - Message: strings.TrimSpace(t.Message), - ID: t.ID.String(), - Commit: ToCommitMeta(repo, t), - ZipballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".zip"), - TarballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".tar.gz"), + Name: t.Name, + Message: strings.TrimSpace(t.Message), + ID: t.ID.String(), + Commit: ToCommitMeta(repo, t), + ZipballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".zip"), + TarballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".tar.gz"), + ArchiveDownloadCount: t.ArchiveDownloadCount, } } @@ -349,13 +350,14 @@ func ToTeams(ctx context.Context, teams []*organization.Team, loadOrgs bool) ([] // ToAnnotatedTag convert git.Tag to api.AnnotatedTag func ToAnnotatedTag(ctx context.Context, repo *repo_model.Repository, t *git.Tag, c *git.Commit) *api.AnnotatedTag { return &api.AnnotatedTag{ - Tag: t.Name, - SHA: t.ID.String(), - Object: ToAnnotatedTagObject(repo, c), - Message: t.Message, - URL: util.URLJoin(repo.APIURL(), "git/tags", t.ID.String()), - Tagger: ToCommitUser(t.Tagger), - Verification: ToVerification(ctx, c), + Tag: t.Name, + SHA: t.ID.String(), + Object: ToAnnotatedTagObject(repo, c), + Message: t.Message, + URL: util.URLJoin(repo.APIURL(), "git/tags", t.ID.String()), + Tagger: ToCommitUser(t.Tagger), + Verification: ToVerification(ctx, c), + ArchiveDownloadCount: t.ArchiveDownloadCount, } } diff --git a/services/convert/release.go b/services/convert/release.go index bfff53e62..fb8bd4567 100644 --- a/services/convert/release.go +++ b/services/convert/release.go @@ -13,21 +13,22 @@ import ( // ToAPIRelease convert a repo_model.Release to api.Release func ToAPIRelease(ctx context.Context, repo *repo_model.Repository, r *repo_model.Release) *api.Release { return &api.Release{ - ID: r.ID, - TagName: r.TagName, - Target: r.Target, - Title: r.Title, - Note: r.Note, - URL: r.APIURL(), - HTMLURL: r.HTMLURL(), - TarURL: r.TarURL(), - ZipURL: r.ZipURL(), - UploadURL: r.APIUploadURL(), - IsDraft: r.IsDraft, - IsPrerelease: r.IsPrerelease, - CreatedAt: r.CreatedUnix.AsTime(), - PublishedAt: r.CreatedUnix.AsTime(), - Publisher: ToUser(ctx, r.Publisher, nil), - Attachments: ToAPIAttachments(repo, r.Attachments), + ID: r.ID, + TagName: r.TagName, + Target: r.Target, + Title: r.Title, + Note: r.Note, + URL: r.APIURL(), + HTMLURL: r.HTMLURL(), + TarURL: r.TarURL(), + ZipURL: r.ZipURL(), + UploadURL: r.APIUploadURL(), + IsDraft: r.IsDraft, + IsPrerelease: r.IsPrerelease, + CreatedAt: r.CreatedUnix.AsTime(), + PublishedAt: r.CreatedUnix.AsTime(), + Publisher: ToUser(ctx, r.Publisher, nil), + Attachments: ToAPIAttachments(repo, r.Attachments), + ArchiveDownloadCount: r.ArchiveDownloadCount, } } diff --git a/services/doctor/dbconsistency.go b/services/doctor/dbconsistency.go index 0903ecc2a..d036f75b2 100644 --- a/services/doctor/dbconsistency.go +++ b/services/doctor/dbconsistency.go @@ -227,6 +227,9 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er // find redirects without existing user. genericOrphanCheck("Orphaned Redirects without existing redirect user", "user_redirect", "user", "user_redirect.redirect_user_id=`user`.id"), + // find archive download count without existing release + genericOrphanCheck("Archive download count without existing Release", + "repo_archive_download_count", "release", "repo_archive_download_count.release_id=release.id"), ) for _, c := range consistencyChecks { diff --git a/services/release/release.go b/services/release/release.go index ba5fd1dd9..b387ccb5c 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -318,6 +318,11 @@ func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *re } } + err = repo_model.DeleteArchiveDownloadCountForRelease(ctx, rel.ID) + if err != nil { + return err + } + if stdout, _, err := git.NewCommand(ctx, "tag", "-d").AddDashesAndList(rel.TagName). SetDescription(fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID)). RunStdString(&git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index 01c58f0ce..c74712b4b 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -30,10 +30,11 @@ import ( // This is entirely opaque to external entities, though, and mostly used as a // handle elsewhere. type ArchiveRequest struct { - RepoID int64 - refName string - Type git.ArchiveType - CommitID string + RepoID int64 + refName string + Type git.ArchiveType + CommitID string + ReleaseID int64 } // ErrUnknownArchiveFormat request archive format is not supported @@ -70,7 +71,7 @@ func (e RepoRefNotFoundError) Is(err error) bool { // NewRequest creates an archival request, based on the URI. The // resulting ArchiveRequest is suitable for being passed to ArchiveRepository() // if it's determined that the request still needs to be satisfied. -func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest, error) { +func NewRequest(ctx context.Context, repoID int64, repo *git.Repository, uri string) (*ArchiveRequest, error) { r := &ArchiveRequest{ RepoID: repoID, } @@ -99,6 +100,17 @@ func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest } r.CommitID = commitID.String() + + release, err := repo_model.GetRelease(ctx, repoID, r.refName) + if err != nil { + if !repo_model.IsErrReleaseNotExist(err) { + return nil, err + } + } + if release != nil { + r.ReleaseID = release.ID + } + return r, nil } @@ -120,6 +132,10 @@ func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver return nil, fmt.Errorf("models.GetRepoArchiver: %w", err) } + if archiver != nil { + archiver.ReleaseID = aReq.ReleaseID + } + if archiver != nil && archiver.Status == repo_model.ArchiverReady { // Archive already generated, we're done. return archiver, nil @@ -145,6 +161,7 @@ func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver return nil, fmt.Errorf("repo_model.GetRepoArchiver: %w", err) } if archiver != nil && archiver.Status == repo_model.ArchiverReady { + archiver.ReleaseID = aReq.ReleaseID return archiver, nil } } diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go index ec6e9dfac..dbd4d9b3c 100644 --- a/services/repository/archiver/archiver_test.go +++ b/services/repository/archiver/archiver_test.go @@ -31,47 +31,47 @@ func TestArchive_Basic(t *testing.T) { contexttest.LoadGitRepo(t, ctx) defer ctx.Repo.GitRepo.Close() - bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + bogusReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) // Check a series of bogus requests. // Step 1, valid commit with a bad extension. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".dilbert") + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".dilbert") assert.Error(t, err) assert.Nil(t, bogusReq) // Step 2, missing commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip") + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip") assert.Error(t, err) assert.Nil(t, bogusReq) // Step 3, doesn't look like branch/tag/commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip") + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip") assert.Error(t, err) assert.Nil(t, bogusReq) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) // Now two valid requests, firstCommit with valid extensions. - zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, zipReq) - tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz") + tgzReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz") assert.NoError(t, err) assert.NotNil(t, tgzReq) - secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".zip") + secondReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".zip") assert.NoError(t, err) assert.NotNil(t, secondReq) @@ -91,7 +91,7 @@ func TestArchive_Basic(t *testing.T) { // Sleep two seconds to make sure the queue doesn't change. time.Sleep(2 * time.Second) - zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq2, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) // This zipReq should match what's sitting in the queue, as we haven't // let it release yet. From the consumer's point of view, this looks like @@ -106,12 +106,12 @@ func TestArchive_Basic(t *testing.T) { // Now we'll submit a request and TimedWaitForCompletion twice, before and // after we release it. We should trigger both the timeout and non-timeout // cases. - timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz") + timedReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz") assert.NoError(t, err) assert.NotNil(t, timedReq) ArchiveRepository(db.DefaultContext, timedReq) - zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq2, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) // Now, we're guaranteed to have released the original zipReq from the queue. // Ensure that we don't get handed back the released entry somehow, but they diff --git a/services/repository/delete.go b/services/repository/delete.go index 08d6800ee..4ff42e627 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -162,6 +162,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID &actions_model.ActionScheduleSpec{RepoID: repoID}, &actions_model.ActionSchedule{RepoID: repoID}, &actions_model.ActionArtifact{RepoID: repoID}, + &repo_model.RepoArchiveDownloadCount{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } diff --git a/templates/repo/release/list.tmpl b/templates/repo/release/list.tmpl index f1e6ffda9..dddb4ee45 100644 --- a/templates/repo/release/list.tmpl +++ b/templates/repo/release/list.tmpl @@ -71,13 +71,16 @@ {{if and (not $.DisableDownloadSourceArchives) (not $release.IsDraft) ($.Permission.CanRead $.UnitTypeCode)}}
  • {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (ZIP) + + {{svg "octicon-info"}} + {{svg "octicon-info"}}
  • {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (TAR.GZ) - + {{svg "octicon-info"}}
    • diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index bcf370b3f..7857d72d4 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -17606,6 +17606,9 @@ "description": "AnnotatedTag represents an annotated tag", "type": "object", "properties": { + "archive_download_count": { + "$ref": "#/definitions/TagArchiveDownloadCount" + }, "message": { "type": "string", "x-go-name": "Message" @@ -22755,6 +22758,9 @@ "description": "Release represents a repository release", "type": "object", "properties": { + "archive_download_count": { + "$ref": "#/definitions/TagArchiveDownloadCount" + }, "assets": { "type": "array", "items": { @@ -23330,6 +23336,9 @@ "description": "Tag represents a repository tag", "type": "object", "properties": { + "archive_download_count": { + "$ref": "#/definitions/TagArchiveDownloadCount" + }, "commit": { "$ref": "#/definitions/CommitMeta" }, @@ -23356,6 +23365,23 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "TagArchiveDownloadCount": { + "description": "TagArchiveDownloadCount counts how many times a archive was downloaded", + "type": "object", + "properties": { + "tar_gz": { + "type": "integer", + "format": "int64", + "x-go-name": "TarGz" + }, + "zip": { + "type": "integer", + "format": "int64", + "x-go-name": "Zip" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "Team": { "description": "Team represents a team in an organization", "type": "object", diff --git a/tests/integration/api_releases_test.go b/tests/integration/api_releases_test.go index 49aa4c4e1..5dc51044e 100644 --- a/tests/integration/api_releases_test.go +++ b/tests/integration/api_releases_test.go @@ -319,3 +319,39 @@ func TestAPIUploadAssetRelease(t *testing.T) { assert.EqualValues(t, 104, attachment.Size) }) } + +func TestAPIGetReleaseArchiveDownloadCount(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + session := loginUser(t, owner.LowerName) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + + name := "ReleaseDownloadCount" + + createNewReleaseUsingAPI(t, session, token, owner, repo, name, "", name, "test") + + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/releases/tags/%s", owner.Name, repo.Name, name) + + req := NewRequest(t, "GET", urlStr) + resp := MakeRequest(t, req, http.StatusOK) + + var release *api.Release + DecodeJSON(t, resp, &release) + + // Check if everything defaults to 0 + assert.Equal(t, int64(0), release.ArchiveDownloadCount.TarGz) + assert.Equal(t, int64(0), release.ArchiveDownloadCount.Zip) + + // Download the tarball to increase the count + MakeRequest(t, NewRequest(t, "GET", release.TarURL), http.StatusOK) + + // Check if the count has increased + resp = MakeRequest(t, req, http.StatusOK) + + DecodeJSON(t, resp, &release) + + assert.Equal(t, int64(1), release.ArchiveDownloadCount.TarGz) + assert.Equal(t, int64(0), release.ArchiveDownloadCount.Zip) +} diff --git a/tests/integration/api_repo_tags_test.go b/tests/integration/api_repo_tags_test.go index c6eeb404c..10a82e11a 100644 --- a/tests/integration/api_repo_tags_test.go +++ b/tests/integration/api_repo_tags_test.go @@ -85,3 +85,39 @@ func createNewTagUsingAPI(t *testing.T, session *TestSession, token, ownerName, DecodeJSON(t, resp, &respObj) return &respObj } + +func TestAPIGetTagArchiveDownloadCount(t *testing.T) { + defer tests.PrepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + // Login as User2. + session := loginUser(t, user.Name) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + + repoName := "repo1" + tagName := "TagDownloadCount" + + createNewTagUsingAPI(t, session, token, user.Name, repoName, tagName, "", "") + + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/tags/%s?token=%s", user.Name, repoName, tagName, token) + + req := NewRequest(t, "GET", urlStr) + resp := MakeRequest(t, req, http.StatusOK) + + var tagInfo *api.Tag + DecodeJSON(t, resp, &tagInfo) + + // Check if everything defaults to 0 + assert.Equal(t, int64(0), tagInfo.ArchiveDownloadCount.TarGz) + assert.Equal(t, int64(0), tagInfo.ArchiveDownloadCount.Zip) + + // Download the tarball to increase the count + MakeRequest(t, NewRequest(t, "GET", tagInfo.TarballURL), http.StatusOK) + + // Check if the count has increased + resp = MakeRequest(t, req, http.StatusOK) + + DecodeJSON(t, resp, &tagInfo) + + assert.Equal(t, int64(1), tagInfo.ArchiveDownloadCount.TarGz) + assert.Equal(t, int64(0), tagInfo.ArchiveDownloadCount.Zip) +} From a610e93cc9fc53e079cc0e643522728001d7ec41 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Thu, 4 Apr 2024 10:40:25 +0200 Subject: [PATCH 2/4] Add fix and migration --- models/forgejo_migrations/migrate.go | 2 ++ models/repo/archive_download_count.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/models/forgejo_migrations/migrate.go b/models/forgejo_migrations/migrate.go index 7c295247c..c1a654cbd 100644 --- a/models/forgejo_migrations/migrate.go +++ b/models/forgejo_migrations/migrate.go @@ -58,6 +58,8 @@ var migrations = []*Migration{ NewMigration("Add the `apply_to_admins` column to the `protected_branch` table", forgejo_v1_22.AddApplyToAdminsSetting), // v9 -> v10 NewMigration("Add pronouns to user", forgejo_v1_22.AddPronounsToUser), + // v11 -> v12 + NewMigration("Add repo_archive_download_count table", forgejo_v1_22.AddRepoArchiveDownloadCount), } // GetCurrentDBVersion returns the current Forgejo database version. diff --git a/models/repo/archive_download_count.go b/models/repo/archive_download_count.go index 6300307a3..9c1382e0a 100644 --- a/models/repo/archive_download_count.go +++ b/models/repo/archive_download_count.go @@ -74,6 +74,9 @@ func GetArchiveDownloadCount(ctx context.Context, repoID, releaseID int64) (*api func GetArchiveDownloadCountForTagName(ctx context.Context, repoID int64, tagName string) (*api.TagArchiveDownloadCount, error) { release, err := GetRelease(ctx, repoID, tagName) if err != nil { + if IsErrReleaseNotExist(err) { + return new(api.TagArchiveDownloadCount), nil + } return nil, err } From ffbe89ec2e01e76d62eb8865a17bb0276fa56790 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Fri, 5 Apr 2024 10:27:38 +0200 Subject: [PATCH 3/4] Update Frontend --- templates/repo/release/list.tmpl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/templates/repo/release/list.tmpl b/templates/repo/release/list.tmpl index dddb4ee45..2d38cacfc 100644 --- a/templates/repo/release/list.tmpl +++ b/templates/repo/release/list.tmpl @@ -70,17 +70,20 @@ {{$hasReleaseAttachment := gt (len $release.Attachments) 0}} {{if and (not $.DisableDownloadSourceArchives) (not $release.IsDraft) ($.Permission.CanRead $.UnitTypeCode)}}
  • - {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (ZIP) - - {{svg "octicon-info"}} - + {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (ZIP) +
    + {{ctx.Locale.TrN .Release.ArchiveDownloadCount.Zip "repo.release.download_count_one" "repo.release.download_count_few" (ctx.Locale.PrettyNumber .Release.ArchiveDownloadCount.Zip)}} +
    {{svg "octicon-info"}}
  • - {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (TAR.GZ) - + {{svg "octicon-file-zip" 16 "tw-mr-1"}}{{ctx.Locale.Tr "repo.release.source_code"}} (TAR.GZ) +
    + {{ctx.Locale.TrN .Release.ArchiveDownloadCount.TarGz "repo.release.download_count_one" "repo.release.download_count_few" (ctx.Locale.PrettyNumber .Release.ArchiveDownloadCount.TarGz)}} +
    + {{svg "octicon-info"}}
    • From 94d4b241b3ce036b5ab9ac9843273343e0bf59e6 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Mon, 8 Apr 2024 12:55:05 +0200 Subject: [PATCH 4/4] Resolve conflicts --- models/forgejo_migrations/v1_22/v11.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 models/forgejo_migrations/v1_22/v11.go diff --git a/models/forgejo_migrations/v1_22/v11.go b/models/forgejo_migrations/v1_22/v11.go new file mode 100644 index 000000000..682252470 --- /dev/null +++ b/models/forgejo_migrations/v1_22/v11.go @@ -0,0 +1,18 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_22 //nolint + +import "xorm.io/xorm" + +func AddRepoArchiveDownloadCount(x *xorm.Engine) error { + type RepoArchiveDownloadCount struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"index unique(s)"` + ReleaseID int64 `xorm:"index unique(s)"` + Type int `xorm:"unique(s)"` + Count int64 + } + + return x.Sync(&RepoArchiveDownloadCount{}) +}