finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add Debian package registry (#24426)

Browse source 
Co-authored-by: @awkwardbunny

This PR adds a Debian package registry.
You can follow [this
tutorial](https://www.baeldung.com/linux/create-debian-package) to build
a *.deb package for testing.
Source packages are not supported at the moment and I did not find
documentation of the architecture "all" and how these packages should be
treated.


![grafik](https://user-images.githubusercontent.com/1666336/218126879-eb80a866-775c-4c8e-8529-5797203a64e6.png)

Part of #20751.

Revised copy of #22854.

---------

Co-authored-by: Brian Hong <brian@hongs.me>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
KN4CK3R 2023-05-02 18:31:35 +02:00 • committed by GitHub
parent 1f52560ca4
commit bf999e4069
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 2008 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

2
models/migrations/migrations.go
View file

@ -489,6 +489,8 @@ var migrations = []Migration{
NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
// v255 -> v256
NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
// v256 -> v257
NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
}
// GetCurrentDBVersion returns the current db version

23
models/migrations/v1_20/v256.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_20 //nolint
import (
"xorm.io/xorm"
)
func AddIsInternalColumnToPackage(x *xorm.Engine) error {
type Package struct {
ID int64 `xorm:"pk autoincr"`
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
RepoID int64 `xorm:"INDEX"`
Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
Name string `xorm:"NOT NULL"`
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
SemverCompatible bool `xorm:"NOT NULL DEFAULT false"`
IsInternal bool `xorm:"NOT NULL DEFAULT false"`
}
return x.Sync(new(Package))
}

11
models/packages/container/search.go
View file

@ -101,16 +101,7 @@ func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit
return nil, err
}
pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs))
for _, pf := range pfs {
pfd, err := packages.GetPackageFileDescriptor(ctx, pf)
if err != nil {
return nil, err
}
pfds = append(pfds, pfd)
}
return pfds, nil
return packages.GetPackageFileDescriptors(ctx, pfs)
}
// GetManifestVersions gets all package versions representing the matching manifest

131
models/packages/debian/search.go Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package debian
import (
"context"
"strconv"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/packages"
debian_module "code.gitea.io/gitea/modules/packages/debian"
"xorm.io/builder"
)
type PackageSearchOptions struct {
OwnerID int64
Distribution string
Component string
Architecture string
}
// SearchLatestPackages gets the latest packages matching the search options
func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) {
var cond builder.Cond = builder.Eq{
"package_file.is_lead": true,
"package.type": packages.TypeDebian,
"package.owner_id": opts.OwnerID,
"package.is_internal": false,
"package_version.is_internal": false,
}
props := make(map[string]string)
if opts.Distribution != "" {
props[debian_module.PropertyDistribution] = opts.Distribution
}
if opts.Component != "" {
props[debian_module.PropertyComponent] = opts.Component
}
if opts.Architecture != "" {
props[debian_module.PropertyArchitecture] = opts.Architecture
}
if len(props) > 0 {
var propsCond builder.Cond = builder.Eq{
"package_property.ref_type": packages.PropertyTypeFile,
}
propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id"))
propsCondBlock := builder.NewCond()
for name, value := range props {
propsCondBlock = propsCondBlock.Or(builder.Eq{
"package_property.name": name,
"package_property.value": value,
})
}
propsCond = propsCond.And(propsCondBlock)
cond = cond.And(builder.Eq{
strconv.Itoa(len(props)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"),
})
}
cond = cond.
And(builder.Expr("pv2.id IS NULL"))
joinCond := builder.
Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
And(builder.Eq{"pv2.is_internal": false})
pfs := make([]*packages.PackageFile, 0, 10)
err := db.GetEngine(ctx).
Table("package_file").
Select("package_file.*").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("LEFT", "package_version pv2", joinCond).
Join("INNER", "package", "package.id = package_version.package_id").
Where(cond).
Desc("package_version.created_unix").
Find(&pfs)
if err != nil {
return nil, err
}
return packages.GetPackageFileDescriptors(ctx, pfs)
}
// GetDistributions gets all available distributions
func GetDistributions(ctx context.Context, ownerID int64) ([]string, error) {
return getDistinctPropertyValues(ctx, ownerID, "", debian_module.PropertyDistribution)
}
// GetComponents gets all available components for the given distribution
func GetComponents(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyComponent)
}
// GetArchitectures gets all available architectures for the given distribution
func GetArchitectures(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyArchitecture)
}
func getDistinctPropertyValues(ctx context.Context, ownerID int64, distribution, propName string) ([]string, error) {
var cond builder.Cond = builder.Eq{
"package_property.ref_type": packages.PropertyTypeFile,
"package_property.name": propName,
"package.type": packages.TypeDebian,
"package.owner_id": ownerID,
}
if distribution != "" {
innerCond := builder.
Expr("pp.ref_id = package_property.ref_id").
And(builder.Eq{
"pp.ref_type": packages.PropertyTypeFile,
"pp.name": debian_module.PropertyDistribution,
"pp.value": distribution,
})
cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond)))
}
values := make([]string, 0, 5)
return values, db.GetEngine(ctx).
Table("package_property").
Distinct("package_property.value").
Join("INNER", "package_file", "package_file.id = package_property.ref_id").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("INNER", "package", "package.id = package_version.package_id").
Where(cond).
Find(&values)
}

26
models/packages/descriptor.go
View file

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/modules/packages/conan"
"code.gitea.io/gitea/modules/packages/conda"
"code.gitea.io/gitea/modules/packages/container"
"code.gitea.io/gitea/modules/packages/debian"
"code.gitea.io/gitea/modules/packages/helm"
"code.gitea.io/gitea/modules/packages/maven"
"code.gitea.io/gitea/modules/packages/npm"
@ -127,13 +128,9 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
return nil, err
}
pfds := make([]*PackageFileDescriptor, 0, len(pfs))
for _, pf := range pfs {
pfd, err := GetPackageFileDescriptor(ctx, pf)
if err != nil {
return nil, err
}
pfds = append(pfds, pfd)
pfds, err := GetPackageFileDescriptors(ctx, pfs)
if err != nil {
return nil, err
}
var metadata interface{}
@ -150,6 +147,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
metadata = &conda.VersionMetadata{}
case TypeContainer:
metadata = &container.Metadata{}
case TypeDebian:
metadata = &debian.Metadata{}
case TypeGeneric:
// generic packages have no metadata
case TypeHelm:
@ -210,6 +209,19 @@ func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFil
}, nil
}
// GetPackageFileDescriptors gets the package file descriptors for the package files
func GetPackageFileDescriptors(ctx context.Context, pfs []*PackageFile) ([]*PackageFileDescriptor, error) {
pfds := make([]*PackageFileDescriptor, 0, len(pfs))
for _, pf := range pfs {
pfd, err := GetPackageFileDescriptor(ctx, pf)
if err != nil {
return nil, err
}
pfds = append(pfds, pfd)
}
return pfds, nil
}
// GetPackageDescriptors gets the package descriptions for the versions
func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) {
pds := make([]*PackageDescriptor, 0, len(pvs))

19
models/packages/package.go
View file

@ -36,6 +36,7 @@ const (
TypeConan Type = "conan"
TypeConda Type = "conda"
TypeContainer Type = "container"
TypeDebian Type = "debian"
TypeGeneric Type = "generic"
TypeHelm Type = "helm"
TypeMaven Type = "maven"
@ -55,6 +56,7 @@ var TypeList = []Type{
TypeConan,
TypeConda,
TypeContainer,
TypeDebian,
TypeGeneric,
TypeHelm,
TypeMaven,
@ -82,6 +84,8 @@ func (pt Type) Name() string {
return "Conda"
case TypeContainer:
return "Container"
case TypeDebian:
return "Debian"
case TypeGeneric:
return "Generic"
case TypeHelm:
@ -121,6 +125,8 @@ func (pt Type) SVGName() string {
return "gitea-conda"
case TypeContainer:
return "octicon-container"
case TypeDebian:
return "gitea-debian"
case TypeGeneric:
return "octicon-package"
case TypeHelm:
@ -154,6 +160,7 @@ type Package struct {
Name string `xorm:"NOT NULL"`
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
SemverCompatible bool `xorm:"NOT NULL DEFAULT false"`
IsInternal bool `xorm:"NOT NULL DEFAULT false"`
}
// TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned
@ -214,9 +221,10 @@ func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) {
// GetPackageByName gets a package by name
func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name string) (*Package, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package.lower_name": strings.ToLower(name),
"package.owner_id": ownerID,
"package.type": packageType,
"package.lower_name": strings.ToLower(name),
"package.is_internal": false,
}
p := &Package{}
@ -236,8 +244,9 @@ func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name
// GetPackagesByType gets all packages of a specific type
func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]*Package, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package.owner_id": ownerID,
"package.type": packageType,
"package.is_internal": false,
}
ps := make([]*Package, 0, 10)

36
models/packages/package_file.go
View file

@ -117,13 +117,15 @@ func DeleteFileByID(ctx context.Context, fileID int64) error {
// PackageFileSearchOptions are options for SearchXXX methods
type PackageFileSearchOptions struct {
OwnerID int64
PackageType string
VersionID int64
Query string
CompositeKey string
Properties map[string]string
OlderThan time.Duration
OwnerID int64
PackageType string
VersionID int64
Query string
CompositeKey string
Properties map[string]string
OlderThan time.Duration
HashAlgorithm string
Hash string
db.Paginator
}
@ -182,6 +184,26 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond {
cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-opts.OlderThan).Unix()})
}
if opts.Hash != "" {
var field string
switch strings.ToLower(opts.HashAlgorithm) {
case "md5":
field = "package_blob.hash_md5"
case "sha1":
field = "package_blob.hash_sha1"
case "sha256":
field = "package_blob.hash_sha256"
case "sha512":
fallthrough
default: // default to SHA512 if not specified or unknown
field = "package_blob.hash_sha512"
}
innerCond := builder.
Expr("package_blob.id = package_file.blob_id").
And(builder.Eq{field: opts.Hash})
cond = cond.And(builder.Exists(builder.Select("package_blob.id").From("package_blob").Where(innerCond)))
}
return cond
}

6
models/packages/package_version.go
View file

@ -173,7 +173,7 @@ const (
)
// PackageSearchOptions are options for SearchXXX methods
// Besides IsInternal are all fields optional and are not used if they have their default value (nil, "", 0)
// All fields optional and are not used if they have their default value (nil, "", 0)
type PackageSearchOptions struct {
OwnerID int64
RepoID int64
@ -192,7 +192,9 @@ type PackageSearchOptions struct {
func (opts *PackageSearchOptions) toConds() builder.Cond {
cond := builder.NewCond()
if !opts.IsInternal.IsNone() {
cond = builder.Eq{"package_version.is_internal": opts.IsInternal.IsTrue()}
cond = builder.Eq{
"package_version.is_internal": opts.IsInternal.IsTrue(),
}
}
if opts.OwnerID != 0 {

5
models/user/setting.go
View file

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/cache"
setting_module "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"xorm.io/builder"
)
@ -42,6 +43,10 @@ func (err ErrUserSettingIsNotExist) Error() string {
return fmt.Sprintf("Setting[%s] is not exist", err.Key)
}
func (err ErrUserSettingIsNotExist) Unwrap() error {
return util.ErrNotExist
}
// IsErrUserSettingIsNotExist return true if err is ErrSettingIsNotExist
func IsErrUserSettingIsNotExist(err error) bool {
_, ok := err.(ErrUserSettingIsNotExist)