Add Package Registry (#16510)
* Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger <thomas@webhippie.de> * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Thomas Boerger <thomas@webhippie.de>
This commit is contained in:
KN4CK3R
GitHub
parent
2bce1ea986
commit
1d332342db
197 changed files with 18563 additions and 55 deletions
|
|
@ -41,6 +41,11 @@ func isAttachmentDownload(req *http.Request) bool {
|
|||
return strings.HasPrefix(req.URL.Path, "/attachments/") && req.Method == "GET"
|
||||
}
|
||||
|
||||
// isContainerPath checks if the request targets the container endpoint
|
||||
func isContainerPath(req *http.Request) bool {
|
||||
return strings.HasPrefix(req.URL.Path, "/v2/")
|
||||
}
|
||||
|
||||
var (
|
||||
gitRawReleasePathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/(?:(?:git-(?:(?:upload)|(?:receive))-pack$)|(?:info/refs$)|(?:HEAD$)|(?:objects/)|(?:raw/)|(?:releases/download/))`)
|
||||
lfsPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/info/lfs/`)
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ func (b *Basic) Name() string {
|
|||
// Returns nil if header is empty or validation fails.
|
||||
func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) *user_model.User {
|
||||
// Basic authentication should only fire on API, Download or on Git or LFSPaths
|
||||
if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) {
|
||||
if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import (
|
|||
"code.gitea.io/gitea/services/auth"
|
||||
"code.gitea.io/gitea/services/migrations"
|
||||
mirror_service "code.gitea.io/gitea/services/mirror"
|
||||
packages_service "code.gitea.io/gitea/services/packages"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
archiver_service "code.gitea.io/gitea/services/repository/archiver"
|
||||
)
|
||||
|
|
@ -139,6 +140,20 @@ func registerCleanupHookTaskTable() {
|
|||
})
|
||||
}
|
||||
|
||||
func registerCleanupPackages() {
|
||||
RegisterTaskFatal("cleanup_packages", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
OlderThan: 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
realConfig := config.(*OlderThanConfig)
|
||||
return packages_service.Cleanup(ctx, realConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func initBasicTasks() {
|
||||
registerUpdateMirrorTask()
|
||||
registerRepoHealthCheck()
|
||||
|
|
@ -150,4 +165,7 @@ func initBasicTasks() {
|
|||
registerUpdateMigrationPosterID()
|
||||
}
|
||||
registerCleanupHookTaskTable()
|
||||
if setting.Packages.Enabled {
|
||||
registerCleanupPackages()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -239,6 +239,7 @@ type WebhookForm struct {
|
|||
PullRequestReview bool
|
||||
PullRequestSync bool
|
||||
Repository bool
|
||||
Package bool
|
||||
Active bool
|
||||
BranchFilter string `binding:"GlobPattern"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -430,3 +430,15 @@ func (f *WebauthnDeleteForm) Validate(req *http.Request, errs binding.Errors) bi
|
|||
ctx := context.GetContext(req)
|
||||
return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
|
||||
}
|
||||
|
||||
// PackageSettingForm form for package settings
|
||||
type PackageSettingForm struct {
|
||||
Action string
|
||||
RepoID int64 `form:"repo_id"`
|
||||
}
|
||||
|
||||
// Validate validates the fields
|
||||
func (f *PackageSettingForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {
|
||||
ctx := context.GetContext(req)
|
||||
return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
|
|
@ -32,6 +33,13 @@ func DeleteOrganization(org *organization.Organization) error {
|
|||
return models.ErrUserOwnRepos{UID: org.ID}
|
||||
}
|
||||
|
||||
// Check ownership of packages.
|
||||
if ownsPackages, err := packages_model.HasOwnerPackages(ctx, org.ID); err != nil {
|
||||
return fmt.Errorf("HasOwnerPackages: %v", err)
|
||||
} else if ownsPackages {
|
||||
return models.ErrUserOwnPackages{UID: org.ID}
|
||||
}
|
||||
|
||||
if err := organization.DeleteOrganization(ctx, org); err != nil {
|
||||
return fmt.Errorf("DeleteOrganization: %v", err)
|
||||
}
|
||||
|
|
|
|||
66
services/packages/auth.go
Normal file
66
services/packages/auth.go
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
)
|
||||
|
||||
type packageClaims struct {
|
||||
jwt.RegisteredClaims
|
||||
UserID int64
|
||||
}
|
||||
|
||||
func CreateAuthorizationToken(u *user_model.User) (string, error) {
|
||||
now := time.Now()
|
||||
|
||||
claims := packageClaims{
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(now.Add(24 * time.Hour)),
|
||||
NotBefore: jwt.NewNumericDate(now),
|
||||
},
|
||||
UserID: u.ID,
|
||||
}
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
|
||||
tokenString, err := token.SignedString([]byte(setting.SecretKey))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tokenString, nil
|
||||
}
|
||||
|
||||
func ParseAuthorizationToken(req *http.Request) (int64, error) {
|
||||
parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return 0, fmt.Errorf("no token")
|
||||
}
|
||||
|
||||
token, err := jwt.ParseWithClaims(parts[1], &packageClaims{}, func(t *jwt.Token) (interface{}, error) {
|
||||
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
|
||||
}
|
||||
return []byte(setting.SecretKey), nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
c, ok := token.Claims.(*packageClaims)
|
||||
if !token.Valid || !ok {
|
||||
return 0, fmt.Errorf("invalid token claim")
|
||||
}
|
||||
|
||||
return c.UserID, nil
|
||||
}
|
||||
136
services/packages/container/blob_uploader.go
Normal file
136
services/packages/container/blob_uploader.go
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
var (
|
||||
// errWriteAfterRead occurs if Write is called after a read operation
|
||||
errWriteAfterRead = errors.New("write is unsupported after a read operation")
|
||||
// errOffsetMissmatch occurs if the file offset is different than the model
|
||||
errOffsetMissmatch = errors.New("offset mismatch between file and model")
|
||||
)
|
||||
|
||||
// BlobUploader handles chunked blob uploads
|
||||
type BlobUploader struct {
|
||||
*packages_model.PackageBlobUpload
|
||||
*packages_module.MultiHasher
|
||||
file *os.File
|
||||
reading bool
|
||||
}
|
||||
|
||||
func buildFilePath(id string) string {
|
||||
return filepath.Join(setting.Packages.ChunkedUploadPath, path.Clean("/" + strings.ReplaceAll(id, "\\", "/"))[1:])
|
||||
}
|
||||
|
||||
// NewBlobUploader creates a new blob uploader for the given id
|
||||
func NewBlobUploader(ctx context.Context, id string) (*BlobUploader, error) {
|
||||
model, err := packages_model.GetBlobUploadByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hash := packages_module.NewMultiHasher()
|
||||
if len(model.HashStateBytes) != 0 {
|
||||
if err := hash.UnmarshalBinary(model.HashStateBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(buildFilePath(model.ID), os.O_RDWR|os.O_CREATE, 0o666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BlobUploader{
|
||||
model,
|
||||
hash,
|
||||
f,
|
||||
false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close implements io.Closer
|
||||
func (u *BlobUploader) Close() error {
|
||||
return u.file.Close()
|
||||
}
|
||||
|
||||
// Append appends a chunk of data and updates the model
|
||||
func (u *BlobUploader) Append(ctx context.Context, r io.Reader) error {
|
||||
if u.reading {
|
||||
return errWriteAfterRead
|
||||
}
|
||||
|
||||
offset, err := u.file.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if offset != u.BytesReceived {
|
||||
return errOffsetMissmatch
|
||||
}
|
||||
|
||||
n, err := io.Copy(io.MultiWriter(u.file, u.MultiHasher), r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// fast path if nothing was written
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
u.BytesReceived += n
|
||||
|
||||
u.HashStateBytes, err = u.MultiHasher.MarshalBinary()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return packages_model.UpdateBlobUpload(ctx, u.PackageBlobUpload)
|
||||
}
|
||||
|
||||
func (u *BlobUploader) Size() int64 {
|
||||
return u.BytesReceived
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (u *BlobUploader) Read(p []byte) (int, error) {
|
||||
if !u.reading {
|
||||
_, err := u.file.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
u.reading = true
|
||||
}
|
||||
|
||||
return u.file.Read(p)
|
||||
}
|
||||
|
||||
// Remove deletes the data and the model of a blob upload
|
||||
func RemoveBlobUploadByID(ctx context.Context, id string) error {
|
||||
if err := packages_model.DeleteBlobUploadByID(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := os.Remove(buildFilePath(id))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
75
services/packages/container/cleanup.go
Normal file
75
services/packages/container/cleanup.go
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
container_model "code.gitea.io/gitea/models/packages/container"
|
||||
)
|
||||
|
||||
// Cleanup removes expired container data
|
||||
func Cleanup(ctx context.Context, olderThan time.Duration) error {
|
||||
if err := cleanupExpiredBlobUploads(ctx, olderThan); err != nil {
|
||||
return err
|
||||
}
|
||||
return cleanupExpiredUploadedBlobs(ctx, olderThan)
|
||||
}
|
||||
|
||||
// cleanupExpiredBlobUploads removes expired blob uploads
|
||||
func cleanupExpiredBlobUploads(ctx context.Context, olderThan time.Duration) error {
|
||||
pbus, err := packages_model.FindExpiredBlobUploads(ctx, olderThan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, pbu := range pbus {
|
||||
if err := RemoveBlobUploadByID(ctx, pbu.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupExpiredUploadedBlobs removes expired uploaded blobs not referenced by a manifest
|
||||
func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) error {
|
||||
pfs, err := container_model.SearchExpiredUploadedBlobs(ctx, olderThan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versions := make(map[int64]struct{})
|
||||
for _, pf := range pfs {
|
||||
versions[pf.VersionID] = struct{}{}
|
||||
|
||||
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for versionID := range versions {
|
||||
has, err := packages_model.HasVersionFileReferences(ctx, versionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, versionID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := packages_model.DeleteVersionByID(ctx, versionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
458
services/packages/packages.go
Normal file
458
services/packages/packages.go
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/notification"
|
||||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
container_service "code.gitea.io/gitea/services/packages/container"
|
||||
)
|
||||
|
||||
// PackageInfo describes a package
|
||||
type PackageInfo struct {
|
||||
Owner *user_model.User
|
||||
PackageType packages_model.Type
|
||||
Name string
|
||||
Version string
|
||||
}
|
||||
|
||||
// PackageCreationInfo describes a package to create
|
||||
type PackageCreationInfo struct {
|
||||
PackageInfo
|
||||
SemverCompatible bool
|
||||
Creator *user_model.User
|
||||
Metadata interface{}
|
||||
Properties map[string]string
|
||||
}
|
||||
|
||||
// PackageFileInfo describes a package file
|
||||
type PackageFileInfo struct {
|
||||
Filename string
|
||||
CompositeKey string
|
||||
}
|
||||
|
||||
// PackageFileCreationInfo describes a package file to create
|
||||
type PackageFileCreationInfo struct {
|
||||
PackageFileInfo
|
||||
Data packages_module.HashedSizeReader
|
||||
IsLead bool
|
||||
Properties map[string]string
|
||||
OverwriteExisting bool
|
||||
}
|
||||
|
||||
// CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned
|
||||
func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
|
||||
return createPackageAndAddFile(pvci, pfci, false)
|
||||
}
|
||||
|
||||
// CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already
|
||||
func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
|
||||
return createPackageAndAddFile(pvci, pfci, true)
|
||||
}
|
||||
|
||||
func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
|
||||
ctx, committer, err := db.TxContext()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer committer.Close()
|
||||
|
||||
pv, created, err := createPackageAndVersion(ctx, pvci, allowDuplicate)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
|
||||
removeBlob := false
|
||||
defer func() {
|
||||
if blobCreated && removeBlob {
|
||||
contentStore := packages_module.NewContentStore()
|
||||
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
|
||||
log.Error("Error deleting package blob from content store: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
removeBlob = true
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := committer.Commit(); err != nil {
|
||||
removeBlob = true
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if created {
|
||||
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
notification.NotifyPackageCreate(pvci.Creator, pd)
|
||||
}
|
||||
|
||||
return pv, pf, nil
|
||||
}
|
||||
|
||||
func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, bool, error) {
|
||||
log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate)
|
||||
|
||||
p := &packages_model.Package{
|
||||
OwnerID: pvci.Owner.ID,
|
||||
Type: pvci.PackageType,
|
||||
Name: pvci.Name,
|
||||
LowerName: strings.ToLower(pvci.Name),
|
||||
SemverCompatible: pvci.SemverCompatible,
|
||||
}
|
||||
var err error
|
||||
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
|
||||
if err != packages_model.ErrDuplicatePackage {
|
||||
log.Error("Error inserting package: %v", err)
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
|
||||
metadataJSON, err := json.Marshal(pvci.Metadata)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
created := true
|
||||
pv := &packages_model.PackageVersion{
|
||||
PackageID: p.ID,
|
||||
CreatorID: pvci.Creator.ID,
|
||||
Version: pvci.Version,
|
||||
LowerVersion: strings.ToLower(pvci.Version),
|
||||
MetadataJSON: string(metadataJSON),
|
||||
}
|
||||
if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
|
||||
if err == packages_model.ErrDuplicatePackageVersion {
|
||||
created = false
|
||||
}
|
||||
if err != packages_model.ErrDuplicatePackageVersion || !allowDuplicate {
|
||||
log.Error("Error inserting package: %v", err)
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
|
||||
if created {
|
||||
for name, value := range pvci.Properties {
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
|
||||
log.Error("Error setting package version property: %v", err)
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pv, created, nil
|
||||
}
|
||||
|
||||
// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
|
||||
func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
|
||||
ctx, committer, err := db.TxContext()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer committer.Close()
|
||||
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
|
||||
removeBlob := false
|
||||
defer func() {
|
||||
if removeBlob {
|
||||
contentStore := packages_module.NewContentStore()
|
||||
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
|
||||
log.Error("Error deleting package blob from content store: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
removeBlob = blobCreated
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := committer.Commit(); err != nil {
|
||||
removeBlob = blobCreated
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return pv, pf, nil
|
||||
}
|
||||
|
||||
// NewPackageBlob creates a package blob instance
|
||||
func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.PackageBlob {
|
||||
hashMD5, hashSHA1, hashSHA256, hashSHA512 := hsr.Sums()
|
||||
|
||||
return &packages_model.PackageBlob{
|
||||
Size: hsr.Size(),
|
||||
HashMD5: fmt.Sprintf("%x", hashMD5),
|
||||
HashSHA1: fmt.Sprintf("%x", hashSHA1),
|
||||
HashSHA256: fmt.Sprintf("%x", hashSHA256),
|
||||
HashSHA512: fmt.Sprintf("%x", hashSHA512),
|
||||
}
|
||||
}
|
||||
|
||||
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
|
||||
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
|
||||
|
||||
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
|
||||
if err != nil {
|
||||
log.Error("Error inserting package blob: %v", err)
|
||||
return nil, nil, false, err
|
||||
}
|
||||
if !exists {
|
||||
contentStore := packages_module.NewContentStore()
|
||||
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil {
|
||||
log.Error("Error saving package blob in content store: %v", err)
|
||||
return nil, nil, false, err
|
||||
}
|
||||
}
|
||||
|
||||
if pfci.OverwriteExisting {
|
||||
pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey)
|
||||
if err != nil && err != packages_model.ErrPackageFileNotExist {
|
||||
return nil, pb, !exists, err
|
||||
}
|
||||
if pf != nil {
|
||||
// Short circuit if blob is the same
|
||||
if pf.BlobID == pb.ID {
|
||||
return pf, pb, !exists, nil
|
||||
}
|
||||
|
||||
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
|
||||
return nil, pb, !exists, err
|
||||
}
|
||||
if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
|
||||
return nil, pb, !exists, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pf := &packages_model.PackageFile{
|
||||
VersionID: pv.ID,
|
||||
BlobID: pb.ID,
|
||||
Name: pfci.Filename,
|
||||
LowerName: strings.ToLower(pfci.Filename),
|
||||
CompositeKey: pfci.CompositeKey,
|
||||
IsLead: pfci.IsLead,
|
||||
}
|
||||
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
|
||||
if err != packages_model.ErrDuplicatePackageFile {
|
||||
log.Error("Error inserting package file: %v", err)
|
||||
}
|
||||
return nil, pb, !exists, err
|
||||
}
|
||||
|
||||
for name, value := range pfci.Properties {
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
|
||||
log.Error("Error setting package file property: %v", err)
|
||||
return pf, pb, !exists, err
|
||||
}
|
||||
}
|
||||
|
||||
return pf, pb, !exists, nil
|
||||
}
|
||||
|
||||
// RemovePackageVersionByNameAndVersion deletes a package version and all associated files
|
||||
func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error {
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return RemovePackageVersion(doer, pv)
|
||||
}
|
||||
|
||||
// RemovePackageVersion deletes the package version and all associated files
|
||||
func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersion) error {
|
||||
ctx, committer, err := db.TxContext()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer committer.Close()
|
||||
|
||||
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Trace("Deleting package: %v", pv.ID)
|
||||
|
||||
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := committer.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
notification.NotifyPackageDelete(doer, pd)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePackageVersionAndReferences deletes the package version and its properties and files
|
||||
func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
|
||||
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, pf := range pfs {
|
||||
if err := DeletePackageFile(ctx, pf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return packages_model.DeleteVersionByID(ctx, pv.ID)
|
||||
}
|
||||
|
||||
// DeletePackageFile deletes the package file and its properties
|
||||
func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) error {
|
||||
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
return packages_model.DeleteFileByID(ctx, pf.ID)
|
||||
}
|
||||
|
||||
// Cleanup removes old unreferenced package blobs
|
||||
func Cleanup(unused context.Context, olderThan time.Duration) error {
|
||||
ctx, committer, err := db.TxContext()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer committer.Close()
|
||||
|
||||
if err := container_service.Cleanup(ctx, olderThan); err != nil {
|
||||
log.Error("hier")
|
||||
return err
|
||||
}
|
||||
|
||||
if err := packages_model.DeletePackagesIfUnreferenced(ctx); err != nil {
|
||||
log.Error("hier2")
|
||||
return err
|
||||
}
|
||||
|
||||
pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
|
||||
if err != nil {
|
||||
log.Error("hier3")
|
||||
return err
|
||||
}
|
||||
|
||||
for _, pb := range pbs {
|
||||
if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
|
||||
log.Error("hier4")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := committer.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contentStore := packages_module.NewContentStore()
|
||||
for _, pb := range pbs {
|
||||
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
|
||||
log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
|
||||
func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
|
||||
log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)
|
||||
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
|
||||
if err != nil {
|
||||
if err == packages_model.ErrPackageNotExist {
|
||||
return nil, nil, err
|
||||
}
|
||||
log.Error("Error getting package: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return GetFileStreamByPackageVersion(ctx, pv, pfi)
|
||||
}
|
||||
|
||||
// GetFileStreamByPackageVersionAndFileID returns the content of the specific package file
|
||||
func GetFileStreamByPackageVersionAndFileID(ctx context.Context, owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) {
|
||||
log.Trace("Getting package file stream: %v, %v, %v", owner.ID, versionID, fileID)
|
||||
|
||||
pv, err := packages_model.GetVersionByID(ctx, versionID)
|
||||
if err != nil {
|
||||
if err == packages_model.ErrPackageVersionNotExist {
|
||||
return nil, nil, packages_model.ErrPackageNotExist
|
||||
}
|
||||
log.Error("Error getting package version: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
|
||||
if err != nil {
|
||||
log.Error("Error getting package: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if p.OwnerID != owner.ID {
|
||||
return nil, nil, packages_model.ErrPackageNotExist
|
||||
}
|
||||
|
||||
pf, err := packages_model.GetFileForVersionByID(ctx, versionID, fileID)
|
||||
if err != nil {
|
||||
log.Error("Error getting file: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return GetPackageFileStream(ctx, pv, pf)
|
||||
}
|
||||
|
||||
// GetFileStreamByPackageVersion returns the content of the specific package file
|
||||
func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
|
||||
pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, pfi.Filename, pfi.CompositeKey)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return GetPackageFileStream(ctx, pv, pf)
|
||||
}
|
||||
|
||||
// GetPackageFileStream returns the content of the specific package file
|
||||
func GetPackageFileStream(ctx context.Context, pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) {
|
||||
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256))
|
||||
if err == nil {
|
||||
if pf.IsLead {
|
||||
if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil {
|
||||
log.Error("Error incrementing download counter: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return s, pf, err
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
|
|
@ -43,8 +44,11 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod
|
|||
notification.NotifyDeleteRepository(doer, repo)
|
||||
}
|
||||
|
||||
err := models.DeleteRepository(doer, repo.OwnerID, repo.ID)
|
||||
return err
|
||||
if err := models.DeleteRepository(doer, repo.OwnerID, repo.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return packages_model.UnlinkRepositoryFromAllPackages(ctx, repo.ID)
|
||||
}
|
||||
|
||||
// PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
asymkey_model "code.gitea.io/gitea/models/asymkey"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/avatar"
|
||||
|
|
@ -58,6 +59,13 @@ func DeleteUser(u *user_model.User) error {
|
|||
return models.ErrUserHasOrgs{UID: u.ID}
|
||||
}
|
||||
|
||||
// Check ownership of packages.
|
||||
if ownsPackages, err := packages_model.HasOwnerPackages(ctx, u.ID); err != nil {
|
||||
return fmt.Errorf("HasOwnerPackages: %v", err)
|
||||
} else if ownsPackages {
|
||||
return models.ErrUserOwnPackages{UID: u.ID}
|
||||
}
|
||||
|
||||
if err := models.DeleteUser(ctx, u); err != nil {
|
||||
return fmt.Errorf("DeleteUser: %v", err)
|
||||
}
|
||||
|
|
@ -111,7 +119,7 @@ func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) error {
|
|||
}
|
||||
if err := DeleteUser(u); err != nil {
|
||||
// Ignore users that were set inactive by admin.
|
||||
if models.IsErrUserOwnRepos(err) || models.IsErrUserHasOrgs(err) {
|
||||
if models.IsErrUserOwnRepos(err) || models.IsErrUserHasOrgs(err) || models.IsErrUserOwnPackages(err) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue