Add Conda package registry (#22262)
This PR adds a [Conda](https://conda.io/) package registry.
This commit is contained in:
KN4CK3R
GitHub
parent
5882e179a9
commit
6ba9ff7b48
24 changed files with 1244 additions and 3 deletions
243
modules/packages/conda/metadata.go
Normal file
243
modules/packages/conda/metadata.go
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package conda
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/bzip2"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument}
|
||||
ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument}
|
||||
ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument}
|
||||
)
|
||||
|
||||
const (
|
||||
PropertyName = "conda.name"
|
||||
PropertyChannel = "conda.channel"
|
||||
PropertySubdir = "conda.subdir"
|
||||
PropertyMetadata = "conda.metdata"
|
||||
)
|
||||
|
||||
// Package represents a Conda package
|
||||
type Package struct {
|
||||
Name string
|
||||
Version string
|
||||
Subdir string
|
||||
VersionMetadata *VersionMetadata
|
||||
FileMetadata *FileMetadata
|
||||
}
|
||||
|
||||
// VersionMetadata represents the metadata of a Conda package
|
||||
type VersionMetadata struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
RepositoryURL string `json:"repository_url,omitempty"`
|
||||
DocumentationURL string `json:"documentation_url,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
LicenseFamily string `json:"license_family,omitempty"`
|
||||
}
|
||||
|
||||
// FileMetadata represents the metadata of a Conda package file
|
||||
type FileMetadata struct {
|
||||
IsCondaPackage bool `json:"is_conda"`
|
||||
Architecture string `json:"architecture,omitempty"`
|
||||
NoArch string `json:"noarch,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
BuildNumber int64 `json:"build_number,omitempty"`
|
||||
Dependencies []string `json:"dependencies,omitempty"`
|
||||
Platform string `json:"platform,omitempty"`
|
||||
Timestamp int64 `json:"timestamp,omitempty"`
|
||||
}
|
||||
|
||||
type index struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Architecture string `json:"arch"`
|
||||
NoArch string `json:"noarch"`
|
||||
Build string `json:"build"`
|
||||
BuildNumber int64 `json:"build_number"`
|
||||
Dependencies []string `json:"depends"`
|
||||
License string `json:"license"`
|
||||
LicenseFamily string `json:"license_family"`
|
||||
Platform string `json:"platform"`
|
||||
Subdir string `json:"subdir"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
type about struct {
|
||||
Description string `json:"description"`
|
||||
Summary string `json:"summary"`
|
||||
ProjectURL string `json:"home"`
|
||||
RepositoryURL string `json:"dev_url"`
|
||||
DocumentationURL string `json:"doc_url"`
|
||||
}
|
||||
|
||||
type ReaderAndReaderAt interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
// ParsePackageBZ2 parses the Conda package file compressed with bzip2
|
||||
func ParsePackageBZ2(r io.Reader) (*Package, error) {
|
||||
gzr := bzip2.NewReader(r)
|
||||
|
||||
return parsePackageTar(gzr)
|
||||
}
|
||||
|
||||
// ParsePackageConda parses the Conda package file compressed with zip and zstd
|
||||
func ParsePackageConda(r io.ReaderAt, size int64) (*Package, error) {
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range zr.File {
|
||||
if strings.HasPrefix(file.Name, "info-") && strings.HasSuffix(file.Name, ".tar.zst") {
|
||||
f, err := zr.Open(file.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
dec, err := zstd.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer dec.Close()
|
||||
|
||||
p, err := parsePackageTar(dec)
|
||||
if p != nil {
|
||||
p.FileMetadata.IsCondaPackage = true
|
||||
}
|
||||
return p, err
|
||||
}
|
||||
}
|
||||
|
||||
return nil, ErrInvalidStructure
|
||||
}
|
||||
|
||||
func parsePackageTar(r io.Reader) (*Package, error) {
|
||||
var i *index
|
||||
var a *about
|
||||
|
||||
tr := tar.NewReader(r)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if hdr.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
|
||||
if hdr.Name == "info/index.json" {
|
||||
if err := json.NewDecoder(tr).Decode(&i); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !checkName(i.Name) {
|
||||
return nil, ErrInvalidName
|
||||
}
|
||||
|
||||
if !checkVersion(i.Version) {
|
||||
return nil, ErrInvalidVersion
|
||||
}
|
||||
|
||||
if a != nil {
|
||||
break // stop loop if both files were found
|
||||
}
|
||||
} else if hdr.Name == "info/about.json" {
|
||||
if err := json.NewDecoder(tr).Decode(&a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(a.ProjectURL) {
|
||||
a.ProjectURL = ""
|
||||
}
|
||||
if !validation.IsValidURL(a.RepositoryURL) {
|
||||
a.RepositoryURL = ""
|
||||
}
|
||||
if !validation.IsValidURL(a.DocumentationURL) {
|
||||
a.DocumentationURL = ""
|
||||
}
|
||||
|
||||
if i != nil {
|
||||
break // stop loop if both files were found
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if i == nil {
|
||||
return nil, ErrInvalidStructure
|
||||
}
|
||||
if a == nil {
|
||||
a = &about{}
|
||||
}
|
||||
|
||||
return &Package{
|
||||
Name: i.Name,
|
||||
Version: i.Version,
|
||||
Subdir: i.Subdir,
|
||||
VersionMetadata: &VersionMetadata{
|
||||
License: i.License,
|
||||
LicenseFamily: i.LicenseFamily,
|
||||
Description: a.Description,
|
||||
Summary: a.Summary,
|
||||
ProjectURL: a.ProjectURL,
|
||||
RepositoryURL: a.RepositoryURL,
|
||||
DocumentationURL: a.DocumentationURL,
|
||||
},
|
||||
FileMetadata: &FileMetadata{
|
||||
Architecture: i.Architecture,
|
||||
NoArch: i.NoArch,
|
||||
Build: i.Build,
|
||||
BuildNumber: i.BuildNumber,
|
||||
Dependencies: i.Dependencies,
|
||||
Platform: i.Platform,
|
||||
Timestamp: i.Timestamp,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1393
|
||||
func checkName(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
if name != strings.ToLower(name) {
|
||||
return false
|
||||
}
|
||||
return !checkBadCharacters(name, "!")
|
||||
}
|
||||
|
||||
// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1403
|
||||
func checkVersion(version string) bool {
|
||||
if version == "" {
|
||||
return false
|
||||
}
|
||||
return !checkBadCharacters(version, "-")
|
||||
}
|
||||
|
||||
func checkBadCharacters(s, additional string) bool {
|
||||
if strings.ContainsAny(s, "=@#$%^&*:;\"'\\|<>?/ ") {
|
||||
return true
|
||||
}
|
||||
return strings.ContainsAny(s, additional)
|
||||
}
|
||||
150
modules/packages/conda/metadata_test.go
Normal file
150
modules/packages/conda/metadata_test.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package conda
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/dsnet/compress/bzip2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
packageName = "gitea"
|
||||
packageVersion = "1.0.1"
|
||||
description = "Package Description"
|
||||
projectURL = "https://gitea.io"
|
||||
repositoryURL = "https://gitea.io/gitea/gitea"
|
||||
documentationURL = "https://docs.gitea.io"
|
||||
)
|
||||
|
||||
func TestParsePackage(t *testing.T) {
|
||||
createArchive := func(files map[string][]byte) *bytes.Buffer {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
for filename, content := range files {
|
||||
hdr := &tar.Header{
|
||||
Name: filename,
|
||||
Mode: 0o600,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
tw.WriteHeader(hdr)
|
||||
tw.Write(content)
|
||||
}
|
||||
tw.Close()
|
||||
return &buf
|
||||
}
|
||||
|
||||
t.Run("MissingIndexFile", func(t *testing.T) {
|
||||
buf := createArchive(map[string][]byte{"dummy.txt": {}})
|
||||
|
||||
p, err := parsePackageTar(buf)
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidStructure)
|
||||
})
|
||||
|
||||
t.Run("MissingAboutFile", func(t *testing.T) {
|
||||
buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"1.0"}`)})
|
||||
|
||||
p, err := parsePackageTar(buf)
|
||||
assert.NotNil(t, p)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "name", p.Name)
|
||||
assert.Equal(t, "1.0", p.Version)
|
||||
assert.Empty(t, p.VersionMetadata.ProjectURL)
|
||||
})
|
||||
|
||||
t.Run("InvalidName", func(t *testing.T) {
|
||||
for _, name := range []string{"", "name!", "nAMe"} {
|
||||
buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"` + name + `","version":"1.0"}`)})
|
||||
|
||||
p, err := parsePackageTar(buf)
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidName)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("InvalidVersion", func(t *testing.T) {
|
||||
for _, version := range []string{"", "1.0-2"} {
|
||||
buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"` + version + `"}`)})
|
||||
|
||||
p, err := parsePackageTar(buf)
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidVersion)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
buf := createArchive(map[string][]byte{
|
||||
"info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `","subdir":"linux-64"}`),
|
||||
"info/about.json": []byte(`{"description":"` + description + `","dev_url":"` + repositoryURL + `","doc_url":"` + documentationURL + `","home":"` + projectURL + `"}`),
|
||||
})
|
||||
|
||||
p, err := parsePackageTar(buf)
|
||||
assert.NotNil(t, p)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, packageName, p.Name)
|
||||
assert.Equal(t, packageVersion, p.Version)
|
||||
assert.Equal(t, "linux-64", p.Subdir)
|
||||
assert.Equal(t, description, p.VersionMetadata.Description)
|
||||
assert.Equal(t, projectURL, p.VersionMetadata.ProjectURL)
|
||||
assert.Equal(t, repositoryURL, p.VersionMetadata.RepositoryURL)
|
||||
assert.Equal(t, documentationURL, p.VersionMetadata.DocumentationURL)
|
||||
})
|
||||
|
||||
t.Run(".tar.bz2", func(t *testing.T) {
|
||||
tarArchive := createArchive(map[string][]byte{
|
||||
"info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`),
|
||||
})
|
||||
|
||||
var buf bytes.Buffer
|
||||
bw, _ := bzip2.NewWriter(&buf, nil)
|
||||
io.Copy(bw, tarArchive)
|
||||
bw.Close()
|
||||
|
||||
br := bytes.NewReader(buf.Bytes())
|
||||
|
||||
p, err := ParsePackageBZ2(br)
|
||||
assert.NotNil(t, p)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, packageName, p.Name)
|
||||
assert.Equal(t, packageVersion, p.Version)
|
||||
assert.False(t, p.FileMetadata.IsCondaPackage)
|
||||
})
|
||||
|
||||
t.Run(".conda", func(t *testing.T) {
|
||||
tarArchive := createArchive(map[string][]byte{
|
||||
"info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`),
|
||||
})
|
||||
|
||||
var infoBuf bytes.Buffer
|
||||
zsw, _ := zstd.NewWriter(&infoBuf)
|
||||
io.Copy(zsw, tarArchive)
|
||||
zsw.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zpw := zip.NewWriter(&buf)
|
||||
w, _ := zpw.Create("info-x.tar.zst")
|
||||
w.Write(infoBuf.Bytes())
|
||||
zpw.Close()
|
||||
|
||||
br := bytes.NewReader(buf.Bytes())
|
||||
|
||||
p, err := ParsePackageConda(br, int64(br.Len()))
|
||||
assert.NotNil(t, p)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, packageName, p.Name)
|
||||
assert.Equal(t, packageVersion, p.Version)
|
||||
assert.True(t, p.FileMetadata.IsCondaPackage)
|
||||
})
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ var (
|
|||
LimitTotalOwnerSize int64
|
||||
LimitSizeComposer int64
|
||||
LimitSizeConan int64
|
||||
LimitSizeConda int64
|
||||
LimitSizeContainer int64
|
||||
LimitSizeGeneric int64
|
||||
LimitSizeHelm int64
|
||||
|
|
@ -66,6 +67,7 @@ func newPackages() {
|
|||
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
|
||||
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
|
||||
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
|
||||
Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA")
|
||||
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
|
||||
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
|
||||
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue