Add Package Registry (#16510)
* Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger <thomas@webhippie.de> * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Thomas Boerger <thomas@webhippie.de>
This commit is contained in:
KN4CK3R
GitHub
parent
2bce1ea986
commit
1d332342db
197 changed files with 18563 additions and 55 deletions
|
|
@ -70,6 +70,7 @@ type Context struct {
|
|||
ContextUser *user_model.User
|
||||
Repo *Repository
|
||||
Org *Organization
|
||||
Package *Package
|
||||
}
|
||||
|
||||
// TrHTMLEscapeArgs runs Tr but pre-escapes all arguments with html.EscapeString.
|
||||
|
|
@ -331,6 +332,18 @@ func (ctx *Context) RespHeader() http.Header {
|
|||
return ctx.Resp.Header()
|
||||
}
|
||||
|
||||
// SetServeHeaders sets necessary content serve headers
|
||||
func (ctx *Context) SetServeHeaders(filename string) {
|
||||
ctx.Resp.Header().Set("Content-Description", "File Transfer")
|
||||
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
|
||||
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename)
|
||||
ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
|
||||
ctx.Resp.Header().Set("Expires", "0")
|
||||
ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
|
||||
ctx.Resp.Header().Set("Pragma", "public")
|
||||
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
|
||||
}
|
||||
|
||||
// ServeContent serves content to http request
|
||||
func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) {
|
||||
modTime := time.Now()
|
||||
|
|
@ -340,14 +353,7 @@ func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interfa
|
|||
modTime = v
|
||||
}
|
||||
}
|
||||
ctx.Resp.Header().Set("Content-Description", "File Transfer")
|
||||
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
|
||||
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
|
||||
ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
|
||||
ctx.Resp.Header().Set("Expires", "0")
|
||||
ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
|
||||
ctx.Resp.Header().Set("Pragma", "public")
|
||||
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
|
||||
ctx.SetServeHeaders(name)
|
||||
http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r)
|
||||
}
|
||||
|
||||
|
|
@ -359,31 +365,41 @@ func (ctx *Context) ServeFile(file string, names ...string) {
|
|||
} else {
|
||||
name = path.Base(file)
|
||||
}
|
||||
ctx.Resp.Header().Set("Content-Description", "File Transfer")
|
||||
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
|
||||
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
|
||||
ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
|
||||
ctx.Resp.Header().Set("Expires", "0")
|
||||
ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
|
||||
ctx.Resp.Header().Set("Pragma", "public")
|
||||
ctx.SetServeHeaders(name)
|
||||
http.ServeFile(ctx.Resp, ctx.Req, file)
|
||||
}
|
||||
|
||||
// ServeStream serves file via io stream
|
||||
func (ctx *Context) ServeStream(rd io.Reader, name string) {
|
||||
ctx.Resp.Header().Set("Content-Description", "File Transfer")
|
||||
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
|
||||
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
|
||||
ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
|
||||
ctx.Resp.Header().Set("Expires", "0")
|
||||
ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
|
||||
ctx.Resp.Header().Set("Pragma", "public")
|
||||
ctx.SetServeHeaders(name)
|
||||
_, err := io.Copy(ctx.Resp, rd)
|
||||
if err != nil {
|
||||
ctx.ServerError("Download file failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// UploadStream returns the request body or the first form file
|
||||
// Only form files need to get closed.
|
||||
func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) {
|
||||
contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type"))
|
||||
if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") {
|
||||
if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if ctx.Req.MultipartForm.File == nil {
|
||||
return nil, false, http.ErrMissingFile
|
||||
}
|
||||
for _, files := range ctx.Req.MultipartForm.File {
|
||||
if len(files) > 0 {
|
||||
r, err := files[0].Open()
|
||||
return r, true, err
|
||||
}
|
||||
}
|
||||
return nil, false, http.ErrMissingFile
|
||||
}
|
||||
return ctx.Req.Body, false, nil
|
||||
}
|
||||
|
||||
// Error returned an error to web browser
|
||||
func (ctx *Context) Error(status int, contents ...string) {
|
||||
v := http.StatusText(status)
|
||||
|
|
|
|||
109
modules/context/package.go
Normal file
109
modules/context/package.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package context
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
"code.gitea.io/gitea/models/perm"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
)
|
||||
|
||||
// Package contains owner, access mode and optional the package descriptor
|
||||
type Package struct {
|
||||
Owner *user_model.User
|
||||
AccessMode perm.AccessMode
|
||||
Descriptor *packages_model.PackageDescriptor
|
||||
}
|
||||
|
||||
// PackageAssignment returns a middleware to handle Context.Package assignment
|
||||
func PackageAssignment() func(ctx *Context) {
|
||||
return func(ctx *Context) {
|
||||
packageAssignment(ctx, func(status int, title string, obj interface{}) {
|
||||
err, ok := obj.(error)
|
||||
if !ok {
|
||||
err = fmt.Errorf("%s", obj)
|
||||
}
|
||||
if status == http.StatusNotFound {
|
||||
ctx.NotFound(title, err)
|
||||
} else {
|
||||
ctx.ServerError(title, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// PackageAssignmentAPI returns a middleware to handle Context.Package assignment
|
||||
func PackageAssignmentAPI() func(ctx *APIContext) {
|
||||
return func(ctx *APIContext) {
|
||||
packageAssignment(ctx.Context, ctx.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
|
||||
ctx.Package = &Package{
|
||||
Owner: ctx.ContextUser,
|
||||
}
|
||||
|
||||
if ctx.Doer != nil && ctx.Doer.ID == ctx.ContextUser.ID {
|
||||
ctx.Package.AccessMode = perm.AccessModeOwner
|
||||
} else {
|
||||
if ctx.Package.Owner.IsOrganization() {
|
||||
if organization.HasOrgOrUserVisible(ctx, ctx.Package.Owner, ctx.Doer) {
|
||||
ctx.Package.AccessMode = perm.AccessModeRead
|
||||
if ctx.Doer != nil {
|
||||
var err error
|
||||
ctx.Package.AccessMode, err = organization.OrgFromUser(ctx.Package.Owner).GetOrgUserMaxAuthorizeLevel(ctx.Doer.ID)
|
||||
if err != nil {
|
||||
errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.Package.AccessMode = perm.AccessModeRead
|
||||
}
|
||||
}
|
||||
|
||||
packageType := ctx.Params("type")
|
||||
name := ctx.Params("name")
|
||||
version := ctx.Params("version")
|
||||
if packageType != "" && name != "" && version != "" {
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.Type(packageType), name, version)
|
||||
if err != nil {
|
||||
if err == packages_model.ErrPackageNotExist {
|
||||
errCb(http.StatusNotFound, "GetVersionByNameAndVersion", err)
|
||||
} else {
|
||||
errCb(http.StatusInternalServerError, "GetVersionByNameAndVersion", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Package.Descriptor, err = packages_model.GetPackageDescriptor(ctx, pv)
|
||||
if err != nil {
|
||||
errCb(http.StatusInternalServerError, "GetPackageDescriptor", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PackageContexter initializes a package context for a request.
|
||||
func PackageContexter() func(next http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
||||
ctx := Context{
|
||||
Resp: NewResponse(resp),
|
||||
Data: map[string]interface{}{},
|
||||
}
|
||||
|
||||
ctx.Req = WithContext(req, &ctx)
|
||||
|
||||
next.ServeHTTP(ctx.Resp, ctx.Req)
|
||||
})
|
||||
}
|
||||
}
|
||||
43
modules/convert/package.go
Normal file
43
modules/convert/package.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package convert
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/models/packages"
|
||||
"code.gitea.io/gitea/models/perm"
|
||||
api "code.gitea.io/gitea/modules/structs"
|
||||
)
|
||||
|
||||
// ToPackage convert a packages.PackageDescriptor to api.Package
|
||||
func ToPackage(pd *packages.PackageDescriptor) *api.Package {
|
||||
var repo *api.Repository
|
||||
if pd.Repository != nil {
|
||||
repo = ToRepo(pd.Repository, perm.AccessModeNone)
|
||||
}
|
||||
|
||||
return &api.Package{
|
||||
ID: pd.Version.ID,
|
||||
Owner: ToUser(pd.Owner, nil),
|
||||
Repository: repo,
|
||||
Creator: ToUser(pd.Creator, nil),
|
||||
Type: string(pd.Package.Type),
|
||||
Name: pd.Package.Name,
|
||||
Version: pd.Version.Version,
|
||||
CreatedAt: pd.Version.CreatedUnix.AsTime(),
|
||||
}
|
||||
}
|
||||
|
||||
// ToPackageFile converts packages.PackageFileDescriptor to api.PackageFile
|
||||
func ToPackageFile(pfd *packages.PackageFileDescriptor) *api.PackageFile {
|
||||
return &api.PackageFile{
|
||||
ID: pfd.File.ID,
|
||||
Size: pfd.Blob.Size,
|
||||
Name: pfd.File.Name,
|
||||
HashMD5: pfd.Blob.HashMD5,
|
||||
HashSHA1: pfd.Blob.HashSHA1,
|
||||
HashSHA256: pfd.Blob.HashSHA256,
|
||||
HashSHA512: pfd.Blob.HashSHA512,
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ package base
|
|||
|
||||
import (
|
||||
"code.gitea.io/gitea/models"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/repository"
|
||||
|
|
@ -54,4 +55,6 @@ type Notifier interface {
|
|||
NotifySyncCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string)
|
||||
NotifySyncDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string)
|
||||
NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository)
|
||||
NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor)
|
||||
NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ package base
|
|||
|
||||
import (
|
||||
"code.gitea.io/gitea/models"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/repository"
|
||||
|
|
@ -173,3 +174,11 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model
|
|||
// NotifyRepoPendingTransfer places a place holder function
|
||||
func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository) {
|
||||
}
|
||||
|
||||
// NotifyPackageCreate places a place holder function
|
||||
func (*NullNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
}
|
||||
|
||||
// NotifyPackageDelete places a place holder function
|
||||
func (*NullNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ package notification
|
|||
|
||||
import (
|
||||
"code.gitea.io/gitea/models"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/notification/action"
|
||||
|
|
@ -306,3 +307,17 @@ func NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model
|
|||
notifier.NotifyRepoPendingTransfer(doer, newOwner, repo)
|
||||
}
|
||||
}
|
||||
|
||||
// NotifyPackageCreate notifies creation of a package to notifiers
|
||||
func NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
for _, notifier := range notifiers {
|
||||
notifier.NotifyPackageCreate(doer, pd)
|
||||
}
|
||||
}
|
||||
|
||||
// NotifyPackageDelete notifies deletion of a package to notifiers
|
||||
func NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
for _, notifier := range notifiers {
|
||||
notifier.NotifyPackageDelete(doer, pd)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
"code.gitea.io/gitea/models/perm"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unit"
|
||||
|
|
@ -855,3 +856,33 @@ func (m *webhookNotifier) NotifySyncCreateRef(pusher *user_model.User, repo *rep
|
|||
func (m *webhookNotifier) NotifySyncDeleteRef(pusher *user_model.User, repo *repo_model.Repository, refType, refFullName string) {
|
||||
m.NotifyDeleteRef(pusher, repo, refType, refFullName)
|
||||
}
|
||||
|
||||
func (m *webhookNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
notifyPackage(doer, pd, api.HookPackageCreated)
|
||||
}
|
||||
|
||||
func (m *webhookNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
|
||||
notifyPackage(doer, pd, api.HookPackageDeleted)
|
||||
}
|
||||
|
||||
func notifyPackage(sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) {
|
||||
if pd.Repository == nil {
|
||||
// TODO https://github.com/go-gitea/gitea/pull/17940
|
||||
return
|
||||
}
|
||||
|
||||
org := pd.Owner
|
||||
if !org.IsOrganization() {
|
||||
org = nil
|
||||
}
|
||||
|
||||
if err := webhook_services.PrepareWebhooks(pd.Repository, webhook.HookEventPackage, &api.PackagePayload{
|
||||
Action: action,
|
||||
Repository: convert.ToRepo(pd.Repository, perm.AccessModeNone),
|
||||
Package: convert.ToPackage(pd),
|
||||
Organization: convert.ToUser(org, nil),
|
||||
Sender: convert.ToUser(sender, nil),
|
||||
}); err != nil {
|
||||
log.Error("PrepareWebhooks: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
147
modules/packages/composer/metadata.go
Normal file
147
modules/packages/composer/metadata.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package composer
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
// TypeProperty is the name of the property for Composer package types
|
||||
const TypeProperty = "composer.type"
|
||||
|
||||
var (
|
||||
// ErrMissingComposerFile indicates a missing composer.json file
|
||||
ErrMissingComposerFile = errors.New("composer.json file is missing")
|
||||
// ErrInvalidName indicates an invalid package name
|
||||
ErrInvalidName = errors.New("package name is invalid")
|
||||
// ErrInvalidVersion indicates an invalid package version
|
||||
ErrInvalidVersion = errors.New("package version is invalid")
|
||||
)
|
||||
|
||||
// Package represents a Composer package
|
||||
type Package struct {
|
||||
Name string
|
||||
Version string
|
||||
Type string
|
||||
Metadata *Metadata
|
||||
}
|
||||
|
||||
// Metadata represents the metadata of a Composer package
|
||||
type Metadata struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
Homepage string `json:"homepage,omitempty"`
|
||||
License Licenses `json:"license,omitempty"`
|
||||
Authors []Author `json:"authors,omitempty"`
|
||||
Autoload map[string]interface{} `json:"autoload,omitempty"`
|
||||
AutoloadDev map[string]interface{} `json:"autoload-dev,omitempty"`
|
||||
Extra map[string]interface{} `json:"extra,omitempty"`
|
||||
Require map[string]string `json:"require,omitempty"`
|
||||
RequireDev map[string]string `json:"require-dev,omitempty"`
|
||||
Suggest map[string]string `json:"suggest,omitempty"`
|
||||
Provide map[string]string `json:"provide,omitempty"`
|
||||
}
|
||||
|
||||
// Licenses represents the licenses of a Composer package
|
||||
type Licenses []string
|
||||
|
||||
// UnmarshalJSON reads from a string or array
|
||||
func (l *Licenses) UnmarshalJSON(data []byte) error {
|
||||
switch data[0] {
|
||||
case '"':
|
||||
var value string
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
return err
|
||||
}
|
||||
*l = Licenses{value}
|
||||
case '[':
|
||||
values := make([]string, 0, 5)
|
||||
if err := json.Unmarshal(data, &values); err != nil {
|
||||
return err
|
||||
}
|
||||
*l = Licenses(values)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Author represents an author
|
||||
type Author struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Email string `json:"email,omitempty"`
|
||||
Homepage string `json:"homepage,omitempty"`
|
||||
}
|
||||
|
||||
var nameMatch = regexp.MustCompile(`\A[a-z0-9]([_\.-]?[a-z0-9]+)*/[a-z0-9](([_\.]?|-{0,2})[a-z0-9]+)*\z`)
|
||||
|
||||
// ParsePackage parses the metadata of a Composer package file
|
||||
func ParsePackage(r io.ReaderAt, size int64) (*Package, error) {
|
||||
archive, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range archive.File {
|
||||
if strings.Count(file.Name, "/") > 1 {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(file.Name), "composer.json") {
|
||||
f, err := archive.Open(file.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return ParseComposerFile(f)
|
||||
}
|
||||
}
|
||||
return nil, ErrMissingComposerFile
|
||||
}
|
||||
|
||||
// ParseComposerFile parses a composer.json file to retrieve the metadata of a Composer package
|
||||
func ParseComposerFile(r io.Reader) (*Package, error) {
|
||||
var cj struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Type string `json:"type"`
|
||||
Metadata
|
||||
}
|
||||
if err := json.NewDecoder(r).Decode(&cj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !nameMatch.MatchString(cj.Name) {
|
||||
return nil, ErrInvalidName
|
||||
}
|
||||
|
||||
if cj.Version != "" {
|
||||
if _, err := version.NewSemver(cj.Version); err != nil {
|
||||
return nil, ErrInvalidVersion
|
||||
}
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(cj.Homepage) {
|
||||
cj.Homepage = ""
|
||||
}
|
||||
|
||||
if cj.Type == "" {
|
||||
cj.Type = "library"
|
||||
}
|
||||
|
||||
return &Package{
|
||||
Name: cj.Name,
|
||||
Version: cj.Version,
|
||||
Type: cj.Type,
|
||||
Metadata: &cj.Metadata,
|
||||
}, nil
|
||||
}
|
||||
130
modules/packages/composer/metadata_test.go
Normal file
130
modules/packages/composer/metadata_test.go
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package composer
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
name = "gitea/composer-package"
|
||||
description = "Package Description"
|
||||
packageType = "composer-plugin"
|
||||
author = "Gitea Authors"
|
||||
email = "no.reply@gitea.io"
|
||||
homepage = "https://gitea.io"
|
||||
license = "MIT"
|
||||
)
|
||||
|
||||
const composerContent = `{
|
||||
"name": "` + name + `",
|
||||
"description": "` + description + `",
|
||||
"type": "` + packageType + `",
|
||||
"license": "` + license + `",
|
||||
"authors": [
|
||||
{
|
||||
"name": "` + author + `",
|
||||
"email": "` + email + `"
|
||||
}
|
||||
],
|
||||
"homepage": "` + homepage + `",
|
||||
"autoload": {
|
||||
"psr-4": {"Gitea\\ComposerPackage\\": "src/"}
|
||||
},
|
||||
"require": {
|
||||
"php": ">=7.2 || ^8.0"
|
||||
}
|
||||
}`
|
||||
|
||||
func TestLicenseUnmarshal(t *testing.T) {
|
||||
var l Licenses
|
||||
assert.NoError(t, json.NewDecoder(strings.NewReader(`["MIT"]`)).Decode(&l))
|
||||
assert.Len(t, l, 1)
|
||||
assert.Equal(t, "MIT", l[0])
|
||||
assert.NoError(t, json.NewDecoder(strings.NewReader(`"MIT"`)).Decode(&l))
|
||||
assert.Len(t, l, 1)
|
||||
assert.Equal(t, "MIT", l[0])
|
||||
}
|
||||
|
||||
func TestParsePackage(t *testing.T) {
|
||||
createArchive := func(name, content string) []byte {
|
||||
var buf bytes.Buffer
|
||||
archive := zip.NewWriter(&buf)
|
||||
w, _ := archive.Create(name)
|
||||
w.Write([]byte(content))
|
||||
archive.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
t.Run("MissingComposerFile", func(t *testing.T) {
|
||||
data := createArchive("dummy.txt", "")
|
||||
|
||||
cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, cp)
|
||||
assert.ErrorIs(t, err, ErrMissingComposerFile)
|
||||
})
|
||||
|
||||
t.Run("MissingComposerFileInRoot", func(t *testing.T) {
|
||||
data := createArchive("sub/sub/composer.json", "")
|
||||
|
||||
cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, cp)
|
||||
assert.ErrorIs(t, err, ErrMissingComposerFile)
|
||||
})
|
||||
|
||||
t.Run("InvalidComposerFile", func(t *testing.T) {
|
||||
data := createArchive("composer.json", "")
|
||||
|
||||
cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, cp)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
data := createArchive("composer.json", composerContent)
|
||||
|
||||
cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, cp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseComposerFile(t *testing.T) {
|
||||
t.Run("InvalidPackageName", func(t *testing.T) {
|
||||
cp, err := ParseComposerFile(strings.NewReader(`{}`))
|
||||
assert.Nil(t, cp)
|
||||
assert.ErrorIs(t, err, ErrInvalidName)
|
||||
})
|
||||
|
||||
t.Run("InvalidPackageVersion", func(t *testing.T) {
|
||||
cp, err := ParseComposerFile(strings.NewReader(`{"name": "gitea/composer-package", "version": "1.a.3"}`))
|
||||
assert.Nil(t, cp)
|
||||
assert.ErrorIs(t, err, ErrInvalidVersion)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
cp, err := ParseComposerFile(strings.NewReader(composerContent))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, cp)
|
||||
|
||||
assert.Equal(t, name, cp.Name)
|
||||
assert.Empty(t, cp.Version)
|
||||
assert.Equal(t, description, cp.Metadata.Description)
|
||||
assert.Len(t, cp.Metadata.Authors, 1)
|
||||
assert.Equal(t, author, cp.Metadata.Authors[0].Name)
|
||||
assert.Equal(t, email, cp.Metadata.Authors[0].Email)
|
||||
assert.Equal(t, homepage, cp.Metadata.Homepage)
|
||||
assert.Equal(t, packageType, cp.Type)
|
||||
assert.Len(t, cp.Metadata.License, 1)
|
||||
assert.Equal(t, license, cp.Metadata.License[0])
|
||||
})
|
||||
}
|
||||
68
modules/packages/conan/conanfile_parser.go
Normal file
68
modules/packages/conan/conanfile_parser.go
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
patternAuthor = compilePattern("author")
|
||||
patternHomepage = compilePattern("homepage")
|
||||
patternURL = compilePattern("url")
|
||||
patternLicense = compilePattern("license")
|
||||
patternDescription = compilePattern("description")
|
||||
patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`)
|
||||
patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`)
|
||||
)
|
||||
|
||||
func compilePattern(name string) *regexp.Regexp {
|
||||
return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`)
|
||||
}
|
||||
|
||||
func ParseConanfile(r io.Reader) (*Metadata, error) {
|
||||
buf, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metadata := &Metadata{}
|
||||
|
||||
m := patternAuthor.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
metadata.Author = string(m[1])
|
||||
}
|
||||
m = patternHomepage.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
metadata.ProjectURL = string(m[1])
|
||||
}
|
||||
m = patternURL.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
metadata.RepositoryURL = string(m[1])
|
||||
}
|
||||
m = patternLicense.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "")
|
||||
}
|
||||
m = patternDescription.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
metadata.Description = string(m[1])
|
||||
}
|
||||
m = patternTopics.FindSubmatch(buf)
|
||||
if len(m) > 1 && len(m[1]) > 0 {
|
||||
m2 := patternTopicList.FindAllSubmatch(m[1], -1)
|
||||
if len(m2) > 0 {
|
||||
metadata.Keywords = make([]string, 0, len(m2))
|
||||
for _, g := range m2 {
|
||||
if len(g) > 1 {
|
||||
metadata.Keywords = append(metadata.Keywords, string(g[1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
51
modules/packages/conan/conanfile_parser_test.go
Normal file
51
modules/packages/conan/conanfile_parser_test.go
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
name = "ConanPackage"
|
||||
version = "1.2"
|
||||
license = "MIT"
|
||||
author = "Gitea <info@gitea.io>"
|
||||
homepage = "https://gitea.io/"
|
||||
url = "https://gitea.com/"
|
||||
description = "Description of ConanPackage"
|
||||
topic1 = "gitea"
|
||||
topic2 = "conan"
|
||||
contentConanfile = `from conans import ConanFile, CMake, tools
|
||||
|
||||
class ConanPackageConan(ConanFile):
|
||||
name = "` + name + `"
|
||||
version = "` + version + `"
|
||||
license = "` + license + `"
|
||||
author = "` + author + `"
|
||||
homepage = "` + homepage + `"
|
||||
url = "` + url + `"
|
||||
description = "` + description + `"
|
||||
topics = ("` + topic1 + `", "` + topic2 + `")
|
||||
settings = "os", "compiler", "build_type", "arch"
|
||||
options = {"shared": [True, False], "fPIC": [True, False]}
|
||||
default_options = {"shared": False, "fPIC": True}
|
||||
generators = "cmake"
|
||||
`
|
||||
)
|
||||
|
||||
func TestParseConanfile(t *testing.T) {
|
||||
metadata, err := ParseConanfile(strings.NewReader(contentConanfile))
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, license, metadata.License)
|
||||
assert.Equal(t, author, metadata.Author)
|
||||
assert.Equal(t, homepage, metadata.ProjectURL)
|
||||
assert.Equal(t, url, metadata.RepositoryURL)
|
||||
assert.Equal(t, description, metadata.Description)
|
||||
assert.Equal(t, []string{topic1, topic2}, metadata.Keywords)
|
||||
}
|
||||
123
modules/packages/conan/conaninfo_parser.go
Normal file
123
modules/packages/conan/conaninfo_parser.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Conaninfo represents infos of a Conan package
|
||||
type Conaninfo struct {
|
||||
Settings map[string]string `json:"settings"`
|
||||
FullSettings map[string]string `json:"full_settings"`
|
||||
Requires []string `json:"requires"`
|
||||
FullRequires []string `json:"full_requires"`
|
||||
Options map[string]string `json:"options"`
|
||||
FullOptions map[string]string `json:"full_options"`
|
||||
RecipeHash string `json:"recipe_hash"`
|
||||
Environment map[string][]string `json:"environment"`
|
||||
}
|
||||
|
||||
func ParseConaninfo(r io.Reader) (*Conaninfo, error) {
|
||||
sections, err := readSections(io.LimitReader(r, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := &Conaninfo{}
|
||||
for section, lines := range sections {
|
||||
if len(lines) == 0 {
|
||||
continue
|
||||
}
|
||||
switch section {
|
||||
case "settings":
|
||||
info.Settings = toMap(lines)
|
||||
case "full_settings":
|
||||
info.FullSettings = toMap(lines)
|
||||
case "options":
|
||||
info.Options = toMap(lines)
|
||||
case "full_options":
|
||||
info.FullOptions = toMap(lines)
|
||||
case "requires":
|
||||
info.Requires = lines
|
||||
case "full_requires":
|
||||
info.FullRequires = lines
|
||||
case "recipe_hash":
|
||||
info.RecipeHash = lines[0]
|
||||
case "env":
|
||||
info.Environment = toMapArray(lines)
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func readSections(r io.Reader) (map[string][]string, error) {
|
||||
sections := make(map[string][]string)
|
||||
|
||||
section := ""
|
||||
lines := make([]string, 0, 5)
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
|
||||
if section != "" {
|
||||
sections[section] = lines
|
||||
}
|
||||
section = line[1 : len(line)-1]
|
||||
lines = make([]string, 0, 5)
|
||||
continue
|
||||
}
|
||||
if section != "" {
|
||||
if line != "" {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if line != "" {
|
||||
return nil, errors.New("Invalid conaninfo.txt")
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if section != "" {
|
||||
sections[section] = lines
|
||||
}
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
func toMap(lines []string) map[string]string {
|
||||
result := make(map[string]string)
|
||||
for _, line := range lines {
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
|
||||
continue
|
||||
}
|
||||
result[parts[0]] = parts[1]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toMapArray(lines []string) map[string][]string {
|
||||
result := make(map[string][]string)
|
||||
for _, line := range lines {
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
|
||||
continue
|
||||
}
|
||||
var items []string
|
||||
if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") {
|
||||
items = strings.Split(parts[1], ",")
|
||||
} else {
|
||||
items = []string{parts[1]}
|
||||
}
|
||||
result[parts[0]] = items
|
||||
}
|
||||
return result
|
||||
}
|
||||
85
modules/packages/conan/conaninfo_parser_test.go
Normal file
85
modules/packages/conan/conaninfo_parser_test.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
settingsKey = "arch"
|
||||
settingsValue = "x84_64"
|
||||
optionsKey = "shared"
|
||||
optionsValue = "False"
|
||||
requires = "fmt/7.1.3"
|
||||
hash = "74714915a51073acb548ca1ce29afbac"
|
||||
envKey = "CC"
|
||||
envValue = "gcc-10"
|
||||
|
||||
contentConaninfo = `[settings]
|
||||
` + settingsKey + `=` + settingsValue + `
|
||||
|
||||
[requires]
|
||||
` + requires + `
|
||||
|
||||
[options]
|
||||
` + optionsKey + `=` + optionsValue + `
|
||||
|
||||
[full_settings]
|
||||
` + settingsKey + `=` + settingsValue + `
|
||||
|
||||
[full_requires]
|
||||
` + requires + `
|
||||
|
||||
[full_options]
|
||||
` + optionsKey + `=` + optionsValue + `
|
||||
|
||||
[recipe_hash]
|
||||
` + hash + `
|
||||
|
||||
[env]
|
||||
` + envKey + `=` + envValue + `
|
||||
|
||||
`
|
||||
)
|
||||
|
||||
func TestParseConaninfo(t *testing.T) {
|
||||
info, err := ParseConaninfo(strings.NewReader(contentConaninfo))
|
||||
assert.NotNil(t, info)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{
|
||||
settingsKey: settingsValue,
|
||||
},
|
||||
info.Settings,
|
||||
)
|
||||
assert.Equal(t, info.Settings, info.FullSettings)
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{
|
||||
optionsKey: optionsValue,
|
||||
},
|
||||
info.Options,
|
||||
)
|
||||
assert.Equal(t, info.Options, info.FullOptions)
|
||||
assert.Equal(
|
||||
t,
|
||||
[]string{requires},
|
||||
info.Requires,
|
||||
)
|
||||
assert.Equal(t, info.Requires, info.FullRequires)
|
||||
assert.Equal(t, hash, info.RecipeHash)
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string][]string{
|
||||
envKey: {envValue},
|
||||
},
|
||||
info.Environment,
|
||||
)
|
||||
}
|
||||
24
modules/packages/conan/metadata.go
Normal file
24
modules/packages/conan/metadata.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
const (
|
||||
PropertyRecipeUser = "conan.recipe.user"
|
||||
PropertyRecipeChannel = "conan.recipe.channel"
|
||||
PropertyRecipeRevision = "conan.recipe.revision"
|
||||
PropertyPackageReference = "conan.package.reference"
|
||||
PropertyPackageRevision = "conan.package.revision"
|
||||
PropertyPackageInfo = "conan.package.info"
|
||||
)
|
||||
|
||||
// Metadata represents the metadata of a Conan package
|
||||
type Metadata struct {
|
||||
Author string `json:"author,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
RepositoryURL string `json:"repository_url,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
}
|
||||
155
modules/packages/conan/reference.go
Normal file
155
modules/packages/conan/reference.go
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
|
||||
goversion "github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
const (
|
||||
// taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py
|
||||
minChars = 2
|
||||
maxChars = 51
|
||||
|
||||
// DefaultRevision if no revision is specified
|
||||
DefaultRevision = "0"
|
||||
)
|
||||
|
||||
var (
|
||||
namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1))
|
||||
revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars))
|
||||
|
||||
ErrValidation = errors.New("Could not validate one or more reference fields")
|
||||
)
|
||||
|
||||
// RecipeReference represents a recipe <Name>/<Version>@<User>/<Channel>#<Revision>
|
||||
type RecipeReference struct {
|
||||
Name string
|
||||
Version string
|
||||
User string
|
||||
Channel string
|
||||
Revision string
|
||||
}
|
||||
|
||||
func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) {
|
||||
log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision)
|
||||
|
||||
if user == "_" {
|
||||
user = ""
|
||||
}
|
||||
if channel == "_" {
|
||||
channel = ""
|
||||
}
|
||||
|
||||
if (user != "" && channel == "") || (user == "" && channel != "") {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
|
||||
if !namePattern.MatchString(name) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if _, err := goversion.NewSemver(version); err != nil {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if user != "" && !namePattern.MatchString(user) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if channel != "" && !namePattern.MatchString(channel) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if revision != "" && !revisionPattern.MatchString(revision) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
|
||||
return &RecipeReference{name, version, user, channel, revision}, nil
|
||||
}
|
||||
|
||||
func (r *RecipeReference) RevisionOrDefault() string {
|
||||
if r.Revision == "" {
|
||||
return DefaultRevision
|
||||
}
|
||||
return r.Revision
|
||||
}
|
||||
|
||||
func (r *RecipeReference) String() string {
|
||||
rev := ""
|
||||
if r.Revision != "" {
|
||||
rev = "#" + r.Revision
|
||||
}
|
||||
if r.User == "" || r.Channel == "" {
|
||||
return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev)
|
||||
}
|
||||
return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev)
|
||||
}
|
||||
|
||||
func (r *RecipeReference) LinkName() string {
|
||||
user := r.User
|
||||
if user == "" {
|
||||
user = "_"
|
||||
}
|
||||
channel := r.Channel
|
||||
if channel == "" {
|
||||
channel = "_"
|
||||
}
|
||||
return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault())
|
||||
}
|
||||
|
||||
func (r *RecipeReference) WithRevision(revision string) *RecipeReference {
|
||||
return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision}
|
||||
}
|
||||
|
||||
// AsKey builds the additional key for the package file
|
||||
func (r *RecipeReference) AsKey() string {
|
||||
return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault())
|
||||
}
|
||||
|
||||
// PackageReference represents a package of a recipe <Name>/<Version>@<User>/<Channel>#<Revision> <Reference>#<Revision>
|
||||
type PackageReference struct {
|
||||
Recipe *RecipeReference
|
||||
Reference string
|
||||
Revision string
|
||||
}
|
||||
|
||||
func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) {
|
||||
log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision)
|
||||
|
||||
if recipe == nil {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if reference == "" || !revisionPattern.MatchString(reference) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
if revision != "" && !revisionPattern.MatchString(revision) {
|
||||
return nil, ErrValidation
|
||||
}
|
||||
|
||||
return &PackageReference{recipe, reference, revision}, nil
|
||||
}
|
||||
|
||||
func (r *PackageReference) RevisionOrDefault() string {
|
||||
if r.Revision == "" {
|
||||
return DefaultRevision
|
||||
}
|
||||
return r.Revision
|
||||
}
|
||||
|
||||
func (r *PackageReference) LinkName() string {
|
||||
return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault())
|
||||
}
|
||||
|
||||
func (r *PackageReference) WithRevision(revision string) *PackageReference {
|
||||
return &PackageReference{r.Recipe, r.Reference, revision}
|
||||
}
|
||||
|
||||
// AsKey builds the additional key for the package file
|
||||
func (r *PackageReference) AsKey() string {
|
||||
return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault())
|
||||
}
|
||||
147
modules/packages/conan/reference_test.go
Normal file
147
modules/packages/conan/reference_test.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package conan
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewRecipeReference(t *testing.T) {
|
||||
cases := []struct {
|
||||
Name string
|
||||
Version string
|
||||
User string
|
||||
Channel string
|
||||
Revision string
|
||||
IsValid bool
|
||||
}{
|
||||
{"", "", "", "", "", false},
|
||||
{"name", "", "", "", "", false},
|
||||
{"", "1.0", "", "", "", false},
|
||||
{"", "", "user", "", "", false},
|
||||
{"", "", "", "channel", "", false},
|
||||
{"", "", "", "", "0", false},
|
||||
{"name", "1.0", "", "", "", true},
|
||||
{"name", "1.0", "user", "", "", false},
|
||||
{"name", "1.0", "", "channel", "", false},
|
||||
{"name", "1.0", "user", "channel", "", true},
|
||||
{"name", "1.0", "_", "", "", true},
|
||||
{"name", "1.0", "", "_", "", true},
|
||||
{"name", "1.0", "_", "_", "", true},
|
||||
{"name", "1.0", "_", "_", "0", true},
|
||||
{"name", "1.0", "", "", "0", true},
|
||||
{"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision)
|
||||
if c.IsValid {
|
||||
assert.NoError(t, err, "case %d, should be invalid", i)
|
||||
assert.NotNil(t, rref, "case %d, should not be nil", i)
|
||||
} else {
|
||||
assert.Error(t, err, "case %d, should be valid", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecipeReferenceRevisionOrDefault(t *testing.T) {
|
||||
rref, err := NewRecipeReference("name", "1.0", "", "", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "", "", "Az09")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Az09", rref.RevisionOrDefault())
|
||||
}
|
||||
|
||||
func TestRecipeReferenceString(t *testing.T) {
|
||||
rref, err := NewRecipeReference("name", "1.0", "", "", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0", rref.String())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0@user/channel", rref.String())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0@user/channel#Az09", rref.String())
|
||||
}
|
||||
|
||||
func TestRecipeReferenceLinkName(t *testing.T) {
|
||||
rref, err := NewRecipeReference("name", "1.0", "", "", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0/_/_/0", rref.LinkName())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName())
|
||||
|
||||
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName())
|
||||
}
|
||||
|
||||
func TestNewPackageReference(t *testing.T) {
|
||||
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
|
||||
|
||||
cases := []struct {
|
||||
Recipe *RecipeReference
|
||||
Reference string
|
||||
Revision string
|
||||
IsValid bool
|
||||
}{
|
||||
{nil, "", "", false},
|
||||
{rref, "", "", false},
|
||||
{nil, "aZ09", "", false},
|
||||
{rref, "aZ09", "", true},
|
||||
{rref, "", "Az09", false},
|
||||
{rref, "aZ09", "Az09", true},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision)
|
||||
if c.IsValid {
|
||||
assert.NoError(t, err, "case %d, should be invalid", i)
|
||||
assert.NotNil(t, pref, "case %d, should not be nil", i)
|
||||
} else {
|
||||
assert.Error(t, err, "case %d, should be valid", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageReferenceRevisionOrDefault(t *testing.T) {
|
||||
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
|
||||
|
||||
pref, err := NewPackageReference(rref, "ref", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
|
||||
|
||||
pref, err = NewPackageReference(rref, "ref", DefaultRevision)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
|
||||
|
||||
pref, err = NewPackageReference(rref, "ref", "Az09")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Az09", pref.RevisionOrDefault())
|
||||
}
|
||||
|
||||
func TestPackageReferenceLinkName(t *testing.T) {
|
||||
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
|
||||
|
||||
pref, err := NewPackageReference(rref, "ref", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "ref/0", pref.LinkName())
|
||||
|
||||
pref, err = NewPackageReference(rref, "ref", "Az09")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "ref/Az09", pref.LinkName())
|
||||
}
|
||||
56
modules/packages/container/helm/helm.go
Normal file
56
modules/packages/container/helm/helm.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package helm
|
||||
|
||||
// https://github.com/helm/helm/blob/main/pkg/chart/
|
||||
|
||||
const ConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
|
||||
|
||||
// Maintainer describes a Chart maintainer.
|
||||
type Maintainer struct {
|
||||
// Name is a user name or organization name
|
||||
Name string `json:"name,omitempty"`
|
||||
// Email is an optional email address to contact the named maintainer
|
||||
Email string `json:"email,omitempty"`
|
||||
// URL is an optional URL to an address for the named maintainer
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
// Metadata for a Chart file. This models the structure of a Chart.yaml file.
|
||||
type Metadata struct {
|
||||
// The name of the chart. Required.
|
||||
Name string `json:"name,omitempty"`
|
||||
// The URL to a relevant project page, git repo, or contact person
|
||||
Home string `json:"home,omitempty"`
|
||||
// Source is the URL to the source code of this chart
|
||||
Sources []string `json:"sources,omitempty"`
|
||||
// A SemVer 2 conformant version string of the chart. Required.
|
||||
Version string `json:"version,omitempty"`
|
||||
// A one-sentence description of the chart
|
||||
Description string `json:"description,omitempty"`
|
||||
// A list of string keywords
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
// A list of name and URL/email address combinations for the maintainer(s)
|
||||
Maintainers []*Maintainer `json:"maintainers,omitempty"`
|
||||
// The URL to an icon file.
|
||||
Icon string `json:"icon,omitempty"`
|
||||
// The API Version of this chart. Required.
|
||||
APIVersion string `json:"apiVersion,omitempty"`
|
||||
// The condition to check to enable chart
|
||||
Condition string `json:"condition,omitempty"`
|
||||
// The tags to check to enable chart
|
||||
Tags string `json:"tags,omitempty"`
|
||||
// The version of the application enclosed inside of this chart.
|
||||
AppVersion string `json:"appVersion,omitempty"`
|
||||
// Whether or not this chart is deprecated
|
||||
Deprecated bool `json:"deprecated,omitempty"`
|
||||
// Annotations are additional mappings uninterpreted by Helm,
|
||||
// made available for inspection by other applications.
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
// KubeVersion is a SemVer constraint specifying the version of Kubernetes required.
|
||||
KubeVersion string `json:"kubeVersion,omitempty"`
|
||||
// Specifies the chart type: application or library
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
157
modules/packages/container/metadata.go
Normal file
157
modules/packages/container/metadata.go
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/packages/container/helm"
|
||||
"code.gitea.io/gitea/modules/packages/container/oci"
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
)
|
||||
|
||||
const (
|
||||
PropertyDigest = "container.digest"
|
||||
PropertyMediaType = "container.mediatype"
|
||||
PropertyManifestTagged = "container.manifest.tagged"
|
||||
PropertyManifestReference = "container.manifest.reference"
|
||||
|
||||
DefaultPlatform = "linux/amd64"
|
||||
|
||||
labelLicenses = "org.opencontainers.image.licenses"
|
||||
labelURL = "org.opencontainers.image.url"
|
||||
labelSource = "org.opencontainers.image.source"
|
||||
labelDocumentation = "org.opencontainers.image.documentation"
|
||||
labelDescription = "org.opencontainers.image.description"
|
||||
labelAuthors = "org.opencontainers.image.authors"
|
||||
)
|
||||
|
||||
type ImageType string
|
||||
|
||||
const (
|
||||
TypeOCI ImageType = "oci"
|
||||
TypeHelm ImageType = "helm"
|
||||
)
|
||||
|
||||
// Name gets the name of the image type
|
||||
func (it ImageType) Name() string {
|
||||
switch it {
|
||||
case TypeHelm:
|
||||
return "Helm Chart"
|
||||
default:
|
||||
return "OCI / Docker"
|
||||
}
|
||||
}
|
||||
|
||||
// Metadata represents the metadata of a Container package
|
||||
type Metadata struct {
|
||||
Type ImageType `json:"type"`
|
||||
IsTagged bool `json:"is_tagged"`
|
||||
Platform string `json:"platform,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Authors []string `json:"authors,omitempty"`
|
||||
Licenses string `json:"license,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
RepositoryURL string `json:"repository_url,omitempty"`
|
||||
DocumentationURL string `json:"documentation_url,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
ImageLayers []string `json:"layer_creation,omitempty"`
|
||||
MultiArch map[string]string `json:"multiarch,omitempty"`
|
||||
}
|
||||
|
||||
// ParseImageConfig parses the metadata of an image config
|
||||
func ParseImageConfig(mediaType oci.MediaType, r io.Reader) (*Metadata, error) {
|
||||
if strings.EqualFold(string(mediaType), helm.ConfigMediaType) {
|
||||
return parseHelmConfig(r)
|
||||
}
|
||||
|
||||
// fallback to OCI Image Config
|
||||
return parseOCIImageConfig(r)
|
||||
}
|
||||
|
||||
func parseOCIImageConfig(r io.Reader) (*Metadata, error) {
|
||||
var image oci.Image
|
||||
if err := json.NewDecoder(r).Decode(&image); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
platform := DefaultPlatform
|
||||
if image.OS != "" && image.Architecture != "" {
|
||||
platform = fmt.Sprintf("%s/%s", image.OS, image.Architecture)
|
||||
if image.Variant != "" {
|
||||
platform = fmt.Sprintf("%s/%s", platform, image.Variant)
|
||||
}
|
||||
}
|
||||
|
||||
imageLayers := make([]string, 0, len(image.History))
|
||||
for _, history := range image.History {
|
||||
cmd := history.CreatedBy
|
||||
if i := strings.Index(cmd, "#(nop) "); i != -1 {
|
||||
cmd = strings.TrimSpace(cmd[i+7:])
|
||||
}
|
||||
imageLayers = append(imageLayers, cmd)
|
||||
}
|
||||
|
||||
metadata := &Metadata{
|
||||
Type: TypeOCI,
|
||||
Platform: platform,
|
||||
Licenses: image.Config.Labels[labelLicenses],
|
||||
ProjectURL: image.Config.Labels[labelURL],
|
||||
RepositoryURL: image.Config.Labels[labelSource],
|
||||
DocumentationURL: image.Config.Labels[labelDocumentation],
|
||||
Description: image.Config.Labels[labelDescription],
|
||||
Labels: image.Config.Labels,
|
||||
ImageLayers: imageLayers,
|
||||
}
|
||||
|
||||
if authors, ok := image.Config.Labels[labelAuthors]; ok {
|
||||
metadata.Authors = []string{authors}
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(metadata.ProjectURL) {
|
||||
metadata.ProjectURL = ""
|
||||
}
|
||||
if !validation.IsValidURL(metadata.RepositoryURL) {
|
||||
metadata.RepositoryURL = ""
|
||||
}
|
||||
if !validation.IsValidURL(metadata.DocumentationURL) {
|
||||
metadata.DocumentationURL = ""
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func parseHelmConfig(r io.Reader) (*Metadata, error) {
|
||||
var config helm.Metadata
|
||||
if err := json.NewDecoder(r).Decode(&config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metadata := &Metadata{
|
||||
Type: TypeHelm,
|
||||
Description: config.Description,
|
||||
ProjectURL: config.Home,
|
||||
}
|
||||
|
||||
if len(config.Maintainers) > 0 {
|
||||
authors := make([]string, 0, len(config.Maintainers))
|
||||
for _, maintainer := range config.Maintainers {
|
||||
authors = append(authors, maintainer.Name)
|
||||
}
|
||||
metadata.Authors = authors
|
||||
}
|
||||
|
||||
if len(config.Sources) > 0 && validation.IsValidURL(config.Sources[0]) {
|
||||
metadata.RepositoryURL = config.Sources[0]
|
||||
}
|
||||
if !validation.IsValidURL(metadata.ProjectURL) {
|
||||
metadata.ProjectURL = ""
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
62
modules/packages/container/metadata_test.go
Normal file
62
modules/packages/container/metadata_test.go
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/packages/container/helm"
|
||||
"code.gitea.io/gitea/modules/packages/container/oci"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseImageConfig(t *testing.T) {
|
||||
description := "Image Description"
|
||||
author := "Gitea"
|
||||
license := "MIT"
|
||||
projectURL := "https://gitea.io"
|
||||
repositoryURL := "https://gitea.com/gitea"
|
||||
documentationURL := "https://docs.gitea.io"
|
||||
|
||||
configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}`
|
||||
|
||||
metadata, err := ParseImageConfig(oci.MediaType(oci.MediaTypeImageManifest), strings.NewReader(configOCI))
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, TypeOCI, metadata.Type)
|
||||
assert.Equal(t, description, metadata.Description)
|
||||
assert.ElementsMatch(t, []string{author}, metadata.Authors)
|
||||
assert.Equal(t, license, metadata.Licenses)
|
||||
assert.Equal(t, projectURL, metadata.ProjectURL)
|
||||
assert.Equal(t, repositoryURL, metadata.RepositoryURL)
|
||||
assert.Equal(t, documentationURL, metadata.DocumentationURL)
|
||||
assert.Equal(t, []string{"do it 1", "do it 2"}, metadata.ImageLayers)
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{
|
||||
labelAuthors: author,
|
||||
labelLicenses: license,
|
||||
labelURL: projectURL,
|
||||
labelSource: repositoryURL,
|
||||
labelDocumentation: documentationURL,
|
||||
labelDescription: description,
|
||||
},
|
||||
metadata.Labels,
|
||||
)
|
||||
assert.Empty(t, metadata.MultiArch)
|
||||
|
||||
configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}`
|
||||
|
||||
metadata, err = ParseImageConfig(oci.MediaType(helm.ConfigMediaType), strings.NewReader(configHelm))
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, TypeHelm, metadata.Type)
|
||||
assert.Equal(t, description, metadata.Description)
|
||||
assert.ElementsMatch(t, []string{author}, metadata.Authors)
|
||||
assert.Equal(t, projectURL, metadata.ProjectURL)
|
||||
assert.Equal(t, repositoryURL, metadata.RepositoryURL)
|
||||
}
|
||||
27
modules/packages/container/oci/digest.go
Normal file
27
modules/packages/container/oci/digest.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var digestPattern = regexp.MustCompile(`\Asha256:[a-f0-9]{64}\z`)
|
||||
|
||||
type Digest string
|
||||
|
||||
// Validate checks if the digest has a valid SHA256 signature
|
||||
func (d Digest) Validate() bool {
|
||||
return digestPattern.MatchString(string(d))
|
||||
}
|
||||
|
||||
func (d Digest) Hash() string {
|
||||
p := strings.SplitN(string(d), ":", 2)
|
||||
if len(p) != 2 {
|
||||
return ""
|
||||
}
|
||||
return p[1]
|
||||
}
|
||||
36
modules/packages/container/oci/mediatype.go
Normal file
36
modules/packages/container/oci/mediatype.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json"
|
||||
MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json"
|
||||
MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
)
|
||||
|
||||
type MediaType string
|
||||
|
||||
// IsValid tests if the media type is in the OCI or Docker namespace
|
||||
func (m MediaType) IsValid() bool {
|
||||
s := string(m)
|
||||
return strings.HasPrefix(s, "application/vnd.docker.") || strings.HasPrefix(s, "application/vnd.oci.")
|
||||
}
|
||||
|
||||
// IsImageManifest tests if the media type is an image manifest
|
||||
func (m MediaType) IsImageManifest() bool {
|
||||
s := string(m)
|
||||
return strings.EqualFold(s, MediaTypeDockerManifest) || strings.EqualFold(s, MediaTypeImageManifest)
|
||||
}
|
||||
|
||||
// IsImageIndex tests if the media type is an image index
|
||||
func (m MediaType) IsImageIndex() bool {
|
||||
s := string(m)
|
||||
return strings.EqualFold(s, MediaTypeDockerManifestList) || strings.EqualFold(s, MediaTypeImageIndex)
|
||||
}
|
||||
191
modules/packages/container/oci/oci.go
Normal file
191
modules/packages/container/oci/oci.go
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// https://github.com/opencontainers/image-spec/tree/main/specs-go/v1
|
||||
|
||||
// ImageConfig defines the execution parameters which should be used as a base when running a container using an image.
|
||||
type ImageConfig struct {
|
||||
// User defines the username or UID which the process in the container should run as.
|
||||
User string `json:"User,omitempty"`
|
||||
|
||||
// ExposedPorts a set of ports to expose from a container running this image.
|
||||
ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"`
|
||||
|
||||
// Env is a list of environment variables to be used in a container.
|
||||
Env []string `json:"Env,omitempty"`
|
||||
|
||||
// Entrypoint defines a list of arguments to use as the command to execute when the container starts.
|
||||
Entrypoint []string `json:"Entrypoint,omitempty"`
|
||||
|
||||
// Cmd defines the default arguments to the entrypoint of the container.
|
||||
Cmd []string `json:"Cmd,omitempty"`
|
||||
|
||||
// Volumes is a set of directories describing where the process is likely write data specific to a container instance.
|
||||
Volumes map[string]struct{} `json:"Volumes,omitempty"`
|
||||
|
||||
// WorkingDir sets the current working directory of the entrypoint process in the container.
|
||||
WorkingDir string `json:"WorkingDir,omitempty"`
|
||||
|
||||
// Labels contains arbitrary metadata for the container.
|
||||
Labels map[string]string `json:"Labels,omitempty"`
|
||||
|
||||
// StopSignal contains the system call signal that will be sent to the container to exit.
|
||||
StopSignal string `json:"StopSignal,omitempty"`
|
||||
}
|
||||
|
||||
// RootFS describes a layer content addresses
|
||||
type RootFS struct {
|
||||
// Type is the type of the rootfs.
|
||||
Type string `json:"type"`
|
||||
|
||||
// DiffIDs is an array of layer content hashes, in order from bottom-most to top-most.
|
||||
DiffIDs []string `json:"diff_ids"`
|
||||
}
|
||||
|
||||
// History describes the history of a layer.
|
||||
type History struct {
|
||||
// Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6.
|
||||
Created *time.Time `json:"created,omitempty"`
|
||||
|
||||
// CreatedBy is the command which created the layer.
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
|
||||
// Author is the author of the build point.
|
||||
Author string `json:"author,omitempty"`
|
||||
|
||||
// Comment is a custom message set when creating the layer.
|
||||
Comment string `json:"comment,omitempty"`
|
||||
|
||||
// EmptyLayer is used to mark if the history item created a filesystem diff.
|
||||
EmptyLayer bool `json:"empty_layer,omitempty"`
|
||||
}
|
||||
|
||||
// Image is the JSON structure which describes some basic information about the image.
|
||||
// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON.
|
||||
type Image struct {
|
||||
// Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6.
|
||||
Created *time.Time `json:"created,omitempty"`
|
||||
|
||||
// Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image.
|
||||
Author string `json:"author,omitempty"`
|
||||
|
||||
// Architecture is the CPU architecture which the binaries in this image are built to run on.
|
||||
Architecture string `json:"architecture"`
|
||||
|
||||
// Variant is the variant of the specified CPU architecture which image binaries are intended to run on.
|
||||
Variant string `json:"variant,omitempty"`
|
||||
|
||||
// OS is the name of the operating system which the image is built to run on.
|
||||
OS string `json:"os"`
|
||||
|
||||
// OSVersion is an optional field specifying the operating system
|
||||
// version, for example on Windows `10.0.14393.1066`.
|
||||
OSVersion string `json:"os.version,omitempty"`
|
||||
|
||||
// OSFeatures is an optional field specifying an array of strings,
|
||||
// each listing a required OS feature (for example on Windows `win32k`).
|
||||
OSFeatures []string `json:"os.features,omitempty"`
|
||||
|
||||
// Config defines the execution parameters which should be used as a base when running a container using the image.
|
||||
Config ImageConfig `json:"config,omitempty"`
|
||||
|
||||
// RootFS references the layer content addresses used by the image.
|
||||
RootFS RootFS `json:"rootfs"`
|
||||
|
||||
// History describes the history of each layer.
|
||||
History []History `json:"history,omitempty"`
|
||||
}
|
||||
|
||||
// Descriptor describes the disposition of targeted content.
|
||||
// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype
|
||||
// when marshalled to JSON.
|
||||
type Descriptor struct {
|
||||
// MediaType is the media type of the object this schema refers to.
|
||||
MediaType MediaType `json:"mediaType,omitempty"`
|
||||
|
||||
// Digest is the digest of the targeted content.
|
||||
Digest Digest `json:"digest"`
|
||||
|
||||
// Size specifies the size in bytes of the blob.
|
||||
Size int64 `json:"size"`
|
||||
|
||||
// URLs specifies a list of URLs from which this object MAY be downloaded
|
||||
URLs []string `json:"urls,omitempty"`
|
||||
|
||||
// Annotations contains arbitrary metadata relating to the targeted content.
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
|
||||
// Data is an embedding of the targeted content. This is encoded as a base64
|
||||
// string when marshalled to JSON (automatically, by encoding/json). If
|
||||
// present, Data can be used directly to avoid fetching the targeted content.
|
||||
Data []byte `json:"data,omitempty"`
|
||||
|
||||
// Platform describes the platform which the image in the manifest runs on.
|
||||
//
|
||||
// This should only be used when referring to a manifest.
|
||||
Platform *Platform `json:"platform,omitempty"`
|
||||
}
|
||||
|
||||
// Platform describes the platform which the image in the manifest runs on.
|
||||
type Platform struct {
|
||||
// Architecture field specifies the CPU architecture, for example
|
||||
// `amd64` or `ppc64`.
|
||||
Architecture string `json:"architecture"`
|
||||
|
||||
// OS specifies the operating system, for example `linux` or `windows`.
|
||||
OS string `json:"os"`
|
||||
|
||||
// OSVersion is an optional field specifying the operating system
|
||||
// version, for example on Windows `10.0.14393.1066`.
|
||||
OSVersion string `json:"os.version,omitempty"`
|
||||
|
||||
// OSFeatures is an optional field specifying an array of strings,
|
||||
// each listing a required OS feature (for example on Windows `win32k`).
|
||||
OSFeatures []string `json:"os.features,omitempty"`
|
||||
|
||||
// Variant is an optional field specifying a variant of the CPU, for
|
||||
// example `v7` to specify ARMv7 when architecture is `arm`.
|
||||
Variant string `json:"variant,omitempty"`
|
||||
}
|
||||
|
||||
type SchemaMediaBase struct {
|
||||
// SchemaVersion is the image manifest schema that this image follows
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
|
||||
// MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json`
|
||||
MediaType MediaType `json:"mediaType,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON.
|
||||
type Manifest struct {
|
||||
SchemaMediaBase
|
||||
|
||||
// Config references a configuration object for a container, by digest.
|
||||
// The referenced configuration object is a JSON blob that the runtime uses to set up the container.
|
||||
Config Descriptor `json:"config"`
|
||||
|
||||
// Layers is an indexed list of layers referenced by the manifest.
|
||||
Layers []Descriptor `json:"layers"`
|
||||
|
||||
// Annotations contains arbitrary metadata for the image manifest.
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
}
|
||||
|
||||
// Index references manifests for various platforms.
|
||||
// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON.
|
||||
type Index struct {
|
||||
SchemaMediaBase
|
||||
|
||||
// Manifests references platform specific manifests.
|
||||
Manifests []Descriptor `json:"manifests"`
|
||||
|
||||
// Annotations contains arbitrary metadata for the image index.
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
}
|
||||
17
modules/packages/container/oci/reference.go
Normal file
17
modules/packages/container/oci/reference.go
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
)
|
||||
|
||||
var referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`)
|
||||
|
||||
type Reference string
|
||||
|
||||
func (r Reference) Validate() bool {
|
||||
return referencePattern.MatchString(string(r))
|
||||
}
|
||||
47
modules/packages/content_store.go
Normal file
47
modules/packages/content_store.go
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
)
|
||||
|
||||
// BlobHash256Key is the key to address a blob content
|
||||
type BlobHash256Key string
|
||||
|
||||
// ContentStore is a wrapper around ObjectStorage
|
||||
type ContentStore struct {
|
||||
store storage.ObjectStorage
|
||||
}
|
||||
|
||||
// NewContentStore creates the default package store
|
||||
func NewContentStore() *ContentStore {
|
||||
contentStore := &ContentStore{storage.Packages}
|
||||
return contentStore
|
||||
}
|
||||
|
||||
// Get gets a package blob
|
||||
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
|
||||
return s.store.Open(keyToRelativePath(key))
|
||||
}
|
||||
|
||||
// Save stores a package blob
|
||||
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
|
||||
_, err := s.store.Save(keyToRelativePath(key), r, size)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes a package blob
|
||||
func (s *ContentStore) Delete(key BlobHash256Key) error {
|
||||
return s.store.Delete(keyToRelativePath(key))
|
||||
}
|
||||
|
||||
// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
|
||||
func keyToRelativePath(key BlobHash256Key) string {
|
||||
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
|
||||
}
|
||||
70
modules/packages/hashed_buffer.go
Normal file
70
modules/packages/hashed_buffer.go
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"code.gitea.io/gitea/modules/util/filebuffer"
|
||||
)
|
||||
|
||||
// HashedSizeReader provide methods to read, sum hashes and a Size method
|
||||
type HashedSizeReader interface {
|
||||
io.Reader
|
||||
HashSummer
|
||||
Size() int64
|
||||
}
|
||||
|
||||
// HashedBuffer is buffer which calculates multiple checksums
|
||||
type HashedBuffer struct {
|
||||
*filebuffer.FileBackedBuffer
|
||||
|
||||
hash *MultiHasher
|
||||
|
||||
combinedWriter io.Writer
|
||||
}
|
||||
|
||||
// NewHashedBuffer creates a hashed buffer with a specific maximum memory size
|
||||
func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
|
||||
b, err := filebuffer.New(maxMemorySize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hash := NewMultiHasher()
|
||||
|
||||
combinedWriter := io.MultiWriter(b, hash)
|
||||
|
||||
return &HashedBuffer{
|
||||
b,
|
||||
hash,
|
||||
combinedWriter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it.
|
||||
func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
|
||||
b, err := NewHashedBuffer(maxMemorySize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = io.Copy(b, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (b *HashedBuffer) Write(p []byte) (int, error) {
|
||||
return b.combinedWriter.Write(p)
|
||||
}
|
||||
|
||||
// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data
|
||||
func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) {
|
||||
return b.hash.Sums()
|
||||
}
|
||||
89
modules/packages/maven/metadata.go
Normal file
89
modules/packages/maven/metadata.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package maven
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
)
|
||||
|
||||
// Metadata represents the metadata of a Maven package
|
||||
type Metadata struct {
|
||||
GroupID string `json:"group_id,omitempty"`
|
||||
ArtifactID string `json:"artifact_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
Licenses []string `json:"licenses,omitempty"`
|
||||
Dependencies []*Dependency `json:"dependencies,omitempty"`
|
||||
}
|
||||
|
||||
// Dependency represents a dependency of a Maven package
|
||||
type Dependency struct {
|
||||
GroupID string `json:"group_id,omitempty"`
|
||||
ArtifactID string `json:"artifact_id,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
type pomStruct struct {
|
||||
XMLName xml.Name `xml:"project"`
|
||||
GroupID string `xml:"groupId"`
|
||||
ArtifactID string `xml:"artifactId"`
|
||||
Version string `xml:"version"`
|
||||
Name string `xml:"name"`
|
||||
Description string `xml:"description"`
|
||||
URL string `xml:"url"`
|
||||
Licenses []struct {
|
||||
Name string `xml:"name"`
|
||||
URL string `xml:"url"`
|
||||
Distribution string `xml:"distribution"`
|
||||
} `xml:"licenses>license"`
|
||||
Dependencies []struct {
|
||||
GroupID string `xml:"groupId"`
|
||||
ArtifactID string `xml:"artifactId"`
|
||||
Version string `xml:"version"`
|
||||
Scope string `xml:"scope"`
|
||||
} `xml:"dependencies>dependency"`
|
||||
}
|
||||
|
||||
// ParsePackageMetaData parses the metadata of a pom file
|
||||
func ParsePackageMetaData(r io.Reader) (*Metadata, error) {
|
||||
var pom pomStruct
|
||||
if err := xml.NewDecoder(r).Decode(&pom); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(pom.URL) {
|
||||
pom.URL = ""
|
||||
}
|
||||
|
||||
licenses := make([]string, 0, len(pom.Licenses))
|
||||
for _, l := range pom.Licenses {
|
||||
if l.Name != "" {
|
||||
licenses = append(licenses, l.Name)
|
||||
}
|
||||
}
|
||||
|
||||
dependencies := make([]*Dependency, 0, len(pom.Dependencies))
|
||||
for _, d := range pom.Dependencies {
|
||||
dependencies = append(dependencies, &Dependency{
|
||||
GroupID: d.GroupID,
|
||||
ArtifactID: d.ArtifactID,
|
||||
Version: d.Version,
|
||||
})
|
||||
}
|
||||
|
||||
return &Metadata{
|
||||
GroupID: pom.GroupID,
|
||||
ArtifactID: pom.ArtifactID,
|
||||
Name: pom.Name,
|
||||
Description: pom.Description,
|
||||
ProjectURL: pom.URL,
|
||||
Licenses: licenses,
|
||||
Dependencies: dependencies,
|
||||
}, nil
|
||||
}
|
||||
73
modules/packages/maven/metadata_test.go
Normal file
73
modules/packages/maven/metadata_test.go
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package maven
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
groupID = "org.gitea"
|
||||
artifactID = "my-project"
|
||||
version = "1.0.1"
|
||||
name = "My Gitea Project"
|
||||
description = "Package Description"
|
||||
projectURL = "https://gitea.io"
|
||||
license = "MIT"
|
||||
dependencyGroupID = "org.gitea.core"
|
||||
dependencyArtifactID = "git"
|
||||
dependencyVersion = "5.0.0"
|
||||
)
|
||||
|
||||
const pomContent = `<?xml version="1.0"?>
|
||||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<groupId>` + groupID + `</groupId>
|
||||
<artifactId>` + artifactID + `</artifactId>
|
||||
<version>` + version + `</version>
|
||||
<name>` + name + `</name>
|
||||
<description>` + description + `</description>
|
||||
<url>` + projectURL + `</url>
|
||||
<licenses>
|
||||
<license>
|
||||
<name>` + license + `</name>
|
||||
</license>
|
||||
</licenses>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>` + dependencyGroupID + `</groupId>
|
||||
<artifactId>` + dependencyArtifactID + `</artifactId>
|
||||
<version>` + dependencyVersion + `</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>`
|
||||
|
||||
func TestParsePackageMetaData(t *testing.T) {
|
||||
t.Run("InvalidFile", func(t *testing.T) {
|
||||
m, err := ParsePackageMetaData(strings.NewReader(""))
|
||||
assert.Nil(t, m)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
m, err := ParsePackageMetaData(strings.NewReader(pomContent))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, m)
|
||||
|
||||
assert.Equal(t, groupID, m.GroupID)
|
||||
assert.Equal(t, artifactID, m.ArtifactID)
|
||||
assert.Equal(t, name, m.Name)
|
||||
assert.Equal(t, description, m.Description)
|
||||
assert.Equal(t, projectURL, m.ProjectURL)
|
||||
assert.Len(t, m.Licenses, 1)
|
||||
assert.Equal(t, license, m.Licenses[0])
|
||||
assert.Len(t, m.Dependencies, 1)
|
||||
assert.Equal(t, dependencyGroupID, m.Dependencies[0].GroupID)
|
||||
assert.Equal(t, dependencyArtifactID, m.Dependencies[0].ArtifactID)
|
||||
assert.Equal(t, dependencyVersion, m.Dependencies[0].Version)
|
||||
})
|
||||
}
|
||||
123
modules/packages/multi_hasher.go
Normal file
123
modules/packages/multi_hasher.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
marshaledSizeMD5 = 92
|
||||
marshaledSizeSHA1 = 96
|
||||
marshaledSizeSHA256 = 108
|
||||
marshaledSizeSHA512 = 204
|
||||
|
||||
marshaledSize = marshaledSizeMD5 + marshaledSizeSHA1 + marshaledSizeSHA256 + marshaledSizeSHA512
|
||||
)
|
||||
|
||||
// HashSummer provide a Sums method
|
||||
type HashSummer interface {
|
||||
Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte)
|
||||
}
|
||||
|
||||
// MultiHasher calculates multiple checksums
|
||||
type MultiHasher struct {
|
||||
md5 hash.Hash
|
||||
sha1 hash.Hash
|
||||
sha256 hash.Hash
|
||||
sha512 hash.Hash
|
||||
|
||||
combinedWriter io.Writer
|
||||
}
|
||||
|
||||
// NewMultiHasher creates a multi hasher
|
||||
func NewMultiHasher() *MultiHasher {
|
||||
md5 := md5.New()
|
||||
sha1 := sha1.New()
|
||||
sha256 := sha256.New()
|
||||
sha512 := sha512.New()
|
||||
|
||||
combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512)
|
||||
|
||||
return &MultiHasher{
|
||||
md5,
|
||||
sha1,
|
||||
sha256,
|
||||
sha512,
|
||||
combinedWriter,
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalBinary implements encoding.BinaryMarshaler
|
||||
func (h *MultiHasher) MarshalBinary() ([]byte, error) {
|
||||
md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b := make([]byte, 0, marshaledSize)
|
||||
b = append(b, md5Bytes...)
|
||||
b = append(b, sha1Bytes...)
|
||||
b = append(b, sha256Bytes...)
|
||||
b = append(b, sha512Bytes...)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary implements encoding.BinaryUnmarshaler
|
||||
func (h *MultiHasher) UnmarshalBinary(b []byte) error {
|
||||
if len(b) != marshaledSize {
|
||||
return errors.New("invalid hash state size")
|
||||
}
|
||||
|
||||
if err := h.md5.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeMD5]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b = b[marshaledSizeMD5:]
|
||||
if err := h.sha1.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA1]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b = b[marshaledSizeSHA1:]
|
||||
if err := h.sha256.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA256]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b = b[marshaledSizeSHA256:]
|
||||
return h.sha512.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA512])
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (h *MultiHasher) Write(p []byte) (int, error) {
|
||||
return h.combinedWriter.Write(p)
|
||||
}
|
||||
|
||||
// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data
|
||||
func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) {
|
||||
hashMD5 = h.md5.Sum(nil)
|
||||
hashSHA1 = h.sha1.Sum(nil)
|
||||
hashSHA256 = h.sha256.Sum(nil)
|
||||
hashSHA512 = h.sha512.Sum(nil)
|
||||
return
|
||||
}
|
||||
54
modules/packages/multi_hasher_test.go
Normal file
54
modules/packages/multi_hasher_test.go
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
expectedMD5 = "e3bef03c5f3b7f6b3ab3e3053ed71e9c"
|
||||
expectedSHA1 = "060b3b99f88e96085b4a68e095bc9e3d1d91e1bc"
|
||||
expectedSHA256 = "6ccce4863b70f258d691f59609d31b4502e1ba5199942d3bc5d35d17a4ce771d"
|
||||
expectedSHA512 = "7f70e439ba8c52025c1f06cdf6ae443c4b8ed2e90059cdb9bbbf8adf80846f185a24acca9245b128b226d61753b0d7ed46580a69c8999eeff3bc13a4d0bd816c"
|
||||
)
|
||||
|
||||
func TestMultiHasherSums(t *testing.T) {
|
||||
t.Run("Sums", func(t *testing.T) {
|
||||
h := NewMultiHasher()
|
||||
h.Write([]byte("gitea"))
|
||||
|
||||
hashMD5, hashSHA1, hashSHA256, hashSHA512 := h.Sums()
|
||||
|
||||
assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5))
|
||||
assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1))
|
||||
assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256))
|
||||
assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512))
|
||||
})
|
||||
|
||||
t.Run("State", func(t *testing.T) {
|
||||
h := NewMultiHasher()
|
||||
h.Write([]byte("git"))
|
||||
|
||||
state, err := h.MarshalBinary()
|
||||
assert.NoError(t, err)
|
||||
|
||||
h2 := NewMultiHasher()
|
||||
err = h2.UnmarshalBinary(state)
|
||||
assert.NoError(t, err)
|
||||
|
||||
h2.Write([]byte("ea"))
|
||||
|
||||
hashMD5, hashSHA1, hashSHA256, hashSHA512 := h2.Sums()
|
||||
|
||||
assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5))
|
||||
assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1))
|
||||
assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256))
|
||||
assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512))
|
||||
})
|
||||
}
|
||||
256
modules/packages/npm/creator.go
Normal file
256
modules/packages/npm/creator.go
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package npm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInvalidPackage indicates an invalid package
|
||||
ErrInvalidPackage = errors.New("The package is invalid")
|
||||
// ErrInvalidPackageName indicates an invalid name
|
||||
ErrInvalidPackageName = errors.New("The package name is invalid")
|
||||
// ErrInvalidPackageVersion indicates an invalid version
|
||||
ErrInvalidPackageVersion = errors.New("The package version is invalid")
|
||||
// ErrInvalidAttachment indicates a invalid attachment
|
||||
ErrInvalidAttachment = errors.New("The package attachment is invalid")
|
||||
// ErrInvalidIntegrity indicates an integrity validation error
|
||||
ErrInvalidIntegrity = errors.New("Failed to validate integrity")
|
||||
)
|
||||
|
||||
var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~'!\(\)\*]+)\z`)
|
||||
|
||||
// Package represents a npm package
|
||||
type Package struct {
|
||||
Name string
|
||||
Version string
|
||||
DistTags []string
|
||||
Metadata Metadata
|
||||
Filename string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
|
||||
type PackageMetadata struct {
|
||||
ID string `json:"_id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
DistTags map[string]string `json:"dist-tags,omitempty"`
|
||||
Versions map[string]*PackageMetadataVersion `json:"versions"`
|
||||
Readme string `json:"readme,omitempty"`
|
||||
Maintainers []User `json:"maintainers,omitempty"`
|
||||
Time map[string]time.Time `json:"time,omitempty"`
|
||||
Homepage string `json:"homepage,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
Repository Repository `json:"repository,omitempty"`
|
||||
Author User `json:"author"`
|
||||
ReadmeFilename string `json:"readmeFilename,omitempty"`
|
||||
Users map[string]bool `json:"users,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
}
|
||||
|
||||
// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
|
||||
type PackageMetadataVersion struct {
|
||||
ID string `json:"_id"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Author User `json:"author"`
|
||||
Homepage string `json:"homepage,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
Repository Repository `json:"repository,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
Dependencies map[string]string `json:"dependencies,omitempty"`
|
||||
DevDependencies map[string]string `json:"devDependencies,omitempty"`
|
||||
PeerDependencies map[string]string `json:"peerDependencies,omitempty"`
|
||||
OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"`
|
||||
Readme string `json:"readme,omitempty"`
|
||||
Dist PackageDistribution `json:"dist"`
|
||||
Maintainers []User `json:"maintainers,omitempty"`
|
||||
}
|
||||
|
||||
// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
|
||||
type PackageDistribution struct {
|
||||
Integrity string `json:"integrity"`
|
||||
Shasum string `json:"shasum"`
|
||||
Tarball string `json:"tarball"`
|
||||
FileCount int `json:"fileCount,omitempty"`
|
||||
UnpackedSize int `json:"unpackedSize,omitempty"`
|
||||
NpmSignature string `json:"npm-signature,omitempty"`
|
||||
}
|
||||
|
||||
// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
|
||||
type User struct {
|
||||
Username string `json:"username,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON is needed because User objects can be strings or objects
|
||||
func (u *User) UnmarshalJSON(data []byte) error {
|
||||
switch data[0] {
|
||||
case '"':
|
||||
if err := json.Unmarshal(data, &u.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
case '{':
|
||||
var tmp struct {
|
||||
Username string `json:"username"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
u.Username = tmp.Username
|
||||
u.Name = tmp.Name
|
||||
u.Email = tmp.Email
|
||||
u.URL = tmp.URL
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
|
||||
type Repository struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
|
||||
type PackageAttachment struct {
|
||||
ContentType string `json:"content_type"`
|
||||
Data string `json:"data"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
|
||||
type packageUpload struct {
|
||||
PackageMetadata
|
||||
Attachments map[string]*PackageAttachment `json:"_attachments"`
|
||||
}
|
||||
|
||||
// ParsePackage parses the content into a npm package
|
||||
func ParsePackage(r io.Reader) (*Package, error) {
|
||||
var upload packageUpload
|
||||
if err := json.NewDecoder(r).Decode(&upload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, meta := range upload.Versions {
|
||||
if !validateName(meta.Name) {
|
||||
return nil, ErrInvalidPackageName
|
||||
}
|
||||
|
||||
v, err := version.NewSemver(meta.Version)
|
||||
if err != nil {
|
||||
return nil, ErrInvalidPackageVersion
|
||||
}
|
||||
|
||||
scope := ""
|
||||
name := meta.Name
|
||||
nameParts := strings.SplitN(meta.Name, "/", 2)
|
||||
if len(nameParts) == 2 {
|
||||
scope = nameParts[0]
|
||||
name = nameParts[1]
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(meta.Homepage) {
|
||||
meta.Homepage = ""
|
||||
}
|
||||
|
||||
p := &Package{
|
||||
Name: meta.Name,
|
||||
Version: v.String(),
|
||||
DistTags: make([]string, 0, 1),
|
||||
Metadata: Metadata{
|
||||
Scope: scope,
|
||||
Name: name,
|
||||
Description: meta.Description,
|
||||
Author: meta.Author.Name,
|
||||
License: meta.License,
|
||||
ProjectURL: meta.Homepage,
|
||||
Keywords: meta.Keywords,
|
||||
Dependencies: meta.Dependencies,
|
||||
DevelopmentDependencies: meta.DevDependencies,
|
||||
PeerDependencies: meta.PeerDependencies,
|
||||
OptionalDependencies: meta.OptionalDependencies,
|
||||
Readme: meta.Readme,
|
||||
},
|
||||
}
|
||||
|
||||
for tag := range upload.DistTags {
|
||||
p.DistTags = append(p.DistTags, tag)
|
||||
}
|
||||
|
||||
p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version))
|
||||
|
||||
attachment := func() *PackageAttachment {
|
||||
for _, a := range upload.Attachments {
|
||||
return a
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if attachment == nil || len(attachment.Data) == 0 {
|
||||
return nil, ErrInvalidAttachment
|
||||
}
|
||||
|
||||
data, err := base64.StdEncoding.DecodeString(attachment.Data)
|
||||
if err != nil {
|
||||
return nil, ErrInvalidAttachment
|
||||
}
|
||||
p.Data = data
|
||||
|
||||
integrity := strings.SplitN(meta.Dist.Integrity, "-", 2)
|
||||
if len(integrity) != 2 {
|
||||
return nil, ErrInvalidIntegrity
|
||||
}
|
||||
integrityHash, err := base64.StdEncoding.DecodeString(integrity[1])
|
||||
if err != nil {
|
||||
return nil, ErrInvalidIntegrity
|
||||
}
|
||||
var hash []byte
|
||||
switch integrity[0] {
|
||||
case "sha1":
|
||||
tmp := sha1.Sum(data)
|
||||
hash = tmp[:]
|
||||
case "sha512":
|
||||
tmp := sha512.Sum512(data)
|
||||
hash = tmp[:]
|
||||
}
|
||||
if !bytes.Equal(integrityHash, hash) {
|
||||
return nil, ErrInvalidIntegrity
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
return nil, ErrInvalidPackage
|
||||
}
|
||||
|
||||
func validateName(name string) bool {
|
||||
if strings.TrimSpace(name) != name {
|
||||
return false
|
||||
}
|
||||
if len(name) == 0 || len(name) > 214 {
|
||||
return false
|
||||
}
|
||||
return nameMatch.MatchString(name)
|
||||
}
|
||||
272
modules/packages/npm/creator_test.go
Normal file
272
modules/packages/npm/creator_test.go
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package npm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParsePackage(t *testing.T) {
|
||||
packageScope := "@scope"
|
||||
packageName := "test-package"
|
||||
packageFullName := packageScope + "/" + packageName
|
||||
packageVersion := "1.0.1-pre"
|
||||
packageTag := "latest"
|
||||
packageAuthor := "KN4CK3R"
|
||||
packageDescription := "Test Description"
|
||||
data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA"
|
||||
integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg=="
|
||||
|
||||
t.Run("InvalidUpload", func(t *testing.T) {
|
||||
p, err := ParsePackage(bytes.NewReader([]byte{0}))
|
||||
assert.Nil(t, p)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("InvalidUploadNoData", func(t *testing.T) {
|
||||
b, _ := json.Marshal(packageUpload{})
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidPackage)
|
||||
})
|
||||
|
||||
t.Run("InvalidPackageName", func(t *testing.T) {
|
||||
test := func(t *testing.T, name string) {
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: name,
|
||||
Name: name,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: name,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidPackageName)
|
||||
}
|
||||
|
||||
test(t, " test ")
|
||||
test(t, " test")
|
||||
test(t, "test ")
|
||||
test(t, "te st")
|
||||
test(t, "invalid/scope")
|
||||
test(t, "@invalid/_name")
|
||||
test(t, "@invalid/.name")
|
||||
})
|
||||
|
||||
t.Run("ValidPackageName", func(t *testing.T) {
|
||||
test := func(t *testing.T, name string) {
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: name,
|
||||
Name: name,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: name,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidPackageVersion)
|
||||
}
|
||||
|
||||
test(t, "test")
|
||||
test(t, "@scope/name")
|
||||
test(t, packageFullName)
|
||||
})
|
||||
|
||||
t.Run("InvalidPackageVersion", func(t *testing.T) {
|
||||
version := "first-version"
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
version: {
|
||||
Name: packageFullName,
|
||||
Version: version,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidPackageVersion)
|
||||
})
|
||||
|
||||
t.Run("InvalidAttachment", func(t *testing.T) {
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: packageFullName,
|
||||
Version: packageVersion,
|
||||
},
|
||||
},
|
||||
},
|
||||
Attachments: map[string]*PackageAttachment{
|
||||
"dummy.tgz": {},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidAttachment)
|
||||
})
|
||||
|
||||
t.Run("InvalidData", func(t *testing.T) {
|
||||
filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: packageFullName,
|
||||
Version: packageVersion,
|
||||
},
|
||||
},
|
||||
},
|
||||
Attachments: map[string]*PackageAttachment{
|
||||
filename: {
|
||||
Data: "/",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidAttachment)
|
||||
})
|
||||
|
||||
t.Run("InvalidIntegrity", func(t *testing.T) {
|
||||
filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: packageFullName,
|
||||
Version: packageVersion,
|
||||
Dist: PackageDistribution{
|
||||
Integrity: "sha512-test==",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Attachments: map[string]*PackageAttachment{
|
||||
filename: {
|
||||
Data: data,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidIntegrity)
|
||||
})
|
||||
|
||||
t.Run("InvalidIntegrity2", func(t *testing.T) {
|
||||
filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: packageFullName,
|
||||
Version: packageVersion,
|
||||
Dist: PackageDistribution{
|
||||
Integrity: integrity,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Attachments: map[string]*PackageAttachment{
|
||||
filename: {
|
||||
Data: base64.StdEncoding.EncodeToString([]byte("data")),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.Nil(t, p)
|
||||
assert.ErrorIs(t, err, ErrInvalidIntegrity)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
|
||||
b, _ := json.Marshal(packageUpload{
|
||||
PackageMetadata: PackageMetadata{
|
||||
ID: packageFullName,
|
||||
Name: packageFullName,
|
||||
DistTags: map[string]string{
|
||||
packageTag: packageVersion,
|
||||
},
|
||||
Versions: map[string]*PackageMetadataVersion{
|
||||
packageVersion: {
|
||||
Name: packageFullName,
|
||||
Version: packageVersion,
|
||||
Description: packageDescription,
|
||||
Author: User{Name: packageAuthor},
|
||||
License: "MIT",
|
||||
Homepage: "https://gitea.io/",
|
||||
Readme: packageDescription,
|
||||
Dependencies: map[string]string{
|
||||
"package": "1.2.0",
|
||||
},
|
||||
Dist: PackageDistribution{
|
||||
Integrity: integrity,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Attachments: map[string]*PackageAttachment{
|
||||
filename: {
|
||||
Data: data,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
p, err := ParsePackage(bytes.NewReader(b))
|
||||
assert.NotNil(t, p)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, packageFullName, p.Name)
|
||||
assert.Equal(t, packageVersion, p.Version)
|
||||
assert.Equal(t, []string{packageTag}, p.DistTags)
|
||||
assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename)
|
||||
b, _ = base64.StdEncoding.DecodeString(data)
|
||||
assert.Equal(t, b, p.Data)
|
||||
assert.Equal(t, packageName, p.Metadata.Name)
|
||||
assert.Equal(t, packageScope, p.Metadata.Scope)
|
||||
assert.Equal(t, packageDescription, p.Metadata.Description)
|
||||
assert.Equal(t, packageDescription, p.Metadata.Readme)
|
||||
assert.Equal(t, packageAuthor, p.Metadata.Author)
|
||||
assert.Equal(t, "MIT", p.Metadata.License)
|
||||
assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL)
|
||||
assert.Contains(t, p.Metadata.Dependencies, "package")
|
||||
assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"])
|
||||
})
|
||||
}
|
||||
24
modules/packages/npm/metadata.go
Normal file
24
modules/packages/npm/metadata.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package npm
|
||||
|
||||
// TagProperty is the name of the property for tag management
|
||||
const TagProperty = "npm.tag"
|
||||
|
||||
// Metadata represents the metadata of a npm package
|
||||
type Metadata struct {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
Dependencies map[string]string `json:"dependencies,omitempty"`
|
||||
DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"`
|
||||
PeerDependencies map[string]string `json:"peer_dependencies,omitempty"`
|
||||
OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"`
|
||||
Readme string `json:"readme,omitempty"`
|
||||
}
|
||||
187
modules/packages/nuget/metadata.go
Normal file
187
modules/packages/nuget/metadata.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package nuget
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMissingNuspecFile indicates a missing Nuspec file
|
||||
ErrMissingNuspecFile = errors.New("Nuspec file is missing")
|
||||
// ErrNuspecFileTooLarge indicates a Nuspec file which is too large
|
||||
ErrNuspecFileTooLarge = errors.New("Nuspec file is too large")
|
||||
// ErrNuspecInvalidID indicates an invalid id in the Nuspec file
|
||||
ErrNuspecInvalidID = errors.New("Nuspec file contains an invalid id")
|
||||
// ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file
|
||||
ErrNuspecInvalidVersion = errors.New("Nuspec file contains an invalid version")
|
||||
)
|
||||
|
||||
// PackageType specifies the package type the metadata describes
|
||||
type PackageType int
|
||||
|
||||
const (
|
||||
// DependencyPackage represents a package (*.nupkg)
|
||||
DependencyPackage PackageType = iota + 1
|
||||
// SymbolsPackage represents a symbol package (*.snupkg)
|
||||
SymbolsPackage
|
||||
|
||||
PropertySymbolID = "nuget.symbol.id"
|
||||
)
|
||||
|
||||
var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`)
|
||||
|
||||
const maxNuspecFileSize = 3 * 1024 * 1024
|
||||
|
||||
// Package represents a Nuget package
|
||||
type Package struct {
|
||||
PackageType PackageType
|
||||
ID string
|
||||
Version string
|
||||
Metadata *Metadata
|
||||
}
|
||||
|
||||
// Metadata represents the metadata of a Nuget package
|
||||
type Metadata struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||
Authors string `json:"authors,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
RepositoryURL string `json:"repository_url,omitempty"`
|
||||
Dependencies map[string][]Dependency `json:"dependencies,omitempty"`
|
||||
}
|
||||
|
||||
// Dependency represents a dependency of a Nuget package
|
||||
type Dependency struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type nuspecPackage struct {
|
||||
Metadata struct {
|
||||
ID string `xml:"id"`
|
||||
Version string `xml:"version"`
|
||||
Authors string `xml:"authors"`
|
||||
RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"`
|
||||
ProjectURL string `xml:"projectUrl"`
|
||||
Description string `xml:"description"`
|
||||
ReleaseNotes string `xml:"releaseNotes"`
|
||||
PackageTypes struct {
|
||||
PackageType []struct {
|
||||
Name string `xml:"name,attr"`
|
||||
} `xml:"packageType"`
|
||||
} `xml:"packageTypes"`
|
||||
Repository struct {
|
||||
URL string `xml:"url,attr"`
|
||||
} `xml:"repository"`
|
||||
Dependencies struct {
|
||||
Group []struct {
|
||||
TargetFramework string `xml:"targetFramework,attr"`
|
||||
Dependency []struct {
|
||||
ID string `xml:"id,attr"`
|
||||
Version string `xml:"version,attr"`
|
||||
Exclude string `xml:"exclude,attr"`
|
||||
} `xml:"dependency"`
|
||||
} `xml:"group"`
|
||||
} `xml:"dependencies"`
|
||||
} `xml:"metadata"`
|
||||
}
|
||||
|
||||
// ParsePackageMetaData parses the metadata of a Nuget package file
|
||||
func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) {
|
||||
archive, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range archive.File {
|
||||
if filepath.Dir(file.Name) != "." {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") {
|
||||
if file.UncompressedSize64 > maxNuspecFileSize {
|
||||
return nil, ErrNuspecFileTooLarge
|
||||
}
|
||||
f, err := archive.Open(file.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return ParseNuspecMetaData(f)
|
||||
}
|
||||
}
|
||||
return nil, ErrMissingNuspecFile
|
||||
}
|
||||
|
||||
// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package
|
||||
func ParseNuspecMetaData(r io.Reader) (*Package, error) {
|
||||
var p nuspecPackage
|
||||
if err := xml.NewDecoder(r).Decode(&p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !idmatch.MatchString(p.Metadata.ID) {
|
||||
return nil, ErrNuspecInvalidID
|
||||
}
|
||||
|
||||
v, err := version.NewSemver(p.Metadata.Version)
|
||||
if err != nil {
|
||||
return nil, ErrNuspecInvalidVersion
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(p.Metadata.ProjectURL) {
|
||||
p.Metadata.ProjectURL = ""
|
||||
}
|
||||
|
||||
packageType := DependencyPackage
|
||||
for _, pt := range p.Metadata.PackageTypes.PackageType {
|
||||
if pt.Name == "SymbolsPackage" {
|
||||
packageType = SymbolsPackage
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
m := &Metadata{
|
||||
Description: p.Metadata.Description,
|
||||
ReleaseNotes: p.Metadata.ReleaseNotes,
|
||||
Authors: p.Metadata.Authors,
|
||||
ProjectURL: p.Metadata.ProjectURL,
|
||||
RepositoryURL: p.Metadata.Repository.URL,
|
||||
Dependencies: make(map[string][]Dependency),
|
||||
}
|
||||
|
||||
for _, group := range p.Metadata.Dependencies.Group {
|
||||
deps := make([]Dependency, 0, len(group.Dependency))
|
||||
for _, dep := range group.Dependency {
|
||||
if dep.ID == "" || dep.Version == "" {
|
||||
continue
|
||||
}
|
||||
deps = append(deps, Dependency{
|
||||
ID: dep.ID,
|
||||
Version: dep.Version,
|
||||
})
|
||||
}
|
||||
if len(deps) > 0 {
|
||||
m.Dependencies[group.TargetFramework] = deps
|
||||
}
|
||||
}
|
||||
return &Package{
|
||||
PackageType: packageType,
|
||||
ID: p.Metadata.ID,
|
||||
Version: v.String(),
|
||||
Metadata: m,
|
||||
}, nil
|
||||
}
|
||||
163
modules/packages/nuget/metadata_test.go
Normal file
163
modules/packages/nuget/metadata_test.go
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package nuget
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
id = "System.Gitea"
|
||||
semver = "1.0.1"
|
||||
authors = "Gitea Authors"
|
||||
projectURL = "https://gitea.io"
|
||||
description = "Package Description"
|
||||
releaseNotes = "Package Release Notes"
|
||||
repositoryURL = "https://gitea.io/gitea/gitea"
|
||||
targetFramework = ".NETStandard2.1"
|
||||
dependencyID = "System.Text.Json"
|
||||
dependencyVersion = "5.0.0"
|
||||
)
|
||||
|
||||
const nuspecContent = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata>
|
||||
<id>` + id + `</id>
|
||||
<version>` + semver + `</version>
|
||||
<authors>` + authors + `</authors>
|
||||
<requireLicenseAcceptance>true</requireLicenseAcceptance>
|
||||
<projectUrl>` + projectURL + `</projectUrl>
|
||||
<description>` + description + `</description>
|
||||
<releaseNotes>` + releaseNotes + `</releaseNotes>
|
||||
<repository url="` + repositoryURL + `" />
|
||||
<dependencies>
|
||||
<group targetFramework="` + targetFramework + `">
|
||||
<dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" />
|
||||
</group>
|
||||
</dependencies>
|
||||
</metadata>
|
||||
</package>`
|
||||
|
||||
const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata>
|
||||
<id>` + id + `</id>
|
||||
<version>` + semver + `</version>
|
||||
<description>` + description + `</description>
|
||||
<packageTypes>
|
||||
<packageType name="SymbolsPackage" />
|
||||
</packageTypes>
|
||||
<dependencies>
|
||||
<group targetFramework="` + targetFramework + `" />
|
||||
</dependencies>
|
||||
</metadata>
|
||||
</package>`
|
||||
|
||||
func TestParsePackageMetaData(t *testing.T) {
|
||||
createArchive := func(name, content string) []byte {
|
||||
var buf bytes.Buffer
|
||||
archive := zip.NewWriter(&buf)
|
||||
w, _ := archive.Create(name)
|
||||
w.Write([]byte(content))
|
||||
archive.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
t.Run("MissingNuspecFile", func(t *testing.T) {
|
||||
data := createArchive("dummy.txt", "")
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, np)
|
||||
assert.ErrorIs(t, err, ErrMissingNuspecFile)
|
||||
})
|
||||
|
||||
t.Run("MissingNuspecFileInRoot", func(t *testing.T) {
|
||||
data := createArchive("sub/package.nuspec", "")
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, np)
|
||||
assert.ErrorIs(t, err, ErrMissingNuspecFile)
|
||||
})
|
||||
|
||||
t.Run("InvalidNuspecFile", func(t *testing.T) {
|
||||
data := createArchive("package.nuspec", "")
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, np)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("InvalidPackageId", func(t *testing.T) {
|
||||
data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata></metadata>
|
||||
</package>`)
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, np)
|
||||
assert.ErrorIs(t, err, ErrNuspecInvalidID)
|
||||
})
|
||||
|
||||
t.Run("InvalidPackageVersion", func(t *testing.T) {
|
||||
data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata>
|
||||
<id>`+id+`</id>
|
||||
</metadata>
|
||||
</package>`)
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.Nil(t, np)
|
||||
assert.ErrorIs(t, err, ErrNuspecInvalidVersion)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
data := createArchive("package.nuspec", nuspecContent)
|
||||
|
||||
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, np)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseNuspecMetaData(t *testing.T) {
|
||||
t.Run("Dependency Package", func(t *testing.T) {
|
||||
np, err := ParseNuspecMetaData(strings.NewReader(nuspecContent))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, np)
|
||||
assert.Equal(t, DependencyPackage, np.PackageType)
|
||||
|
||||
assert.Equal(t, id, np.ID)
|
||||
assert.Equal(t, semver, np.Version)
|
||||
assert.Equal(t, authors, np.Metadata.Authors)
|
||||
assert.Equal(t, projectURL, np.Metadata.ProjectURL)
|
||||
assert.Equal(t, description, np.Metadata.Description)
|
||||
assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes)
|
||||
assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL)
|
||||
assert.Len(t, np.Metadata.Dependencies, 1)
|
||||
assert.Contains(t, np.Metadata.Dependencies, targetFramework)
|
||||
deps := np.Metadata.Dependencies[targetFramework]
|
||||
assert.Len(t, deps, 1)
|
||||
assert.Equal(t, dependencyID, deps[0].ID)
|
||||
assert.Equal(t, dependencyVersion, deps[0].Version)
|
||||
})
|
||||
|
||||
t.Run("Symbols Package", func(t *testing.T) {
|
||||
np, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, np)
|
||||
assert.Equal(t, SymbolsPackage, np.PackageType)
|
||||
|
||||
assert.Equal(t, id, np.ID)
|
||||
assert.Equal(t, semver, np.Version)
|
||||
assert.Equal(t, description, np.Metadata.Description)
|
||||
assert.Empty(t, np.Metadata.Dependencies)
|
||||
})
|
||||
}
|
||||
187
modules/packages/nuget/symbol_extractor.go
Normal file
187
modules/packages/nuget/symbol_extractor.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package nuget
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/packages"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMissingPdbFiles = errors.New("Package does not contain PDB files")
|
||||
ErrInvalidFiles = errors.New("Package contains invalid files")
|
||||
ErrInvalidPdbMagicNumber = errors.New("Invalid Portable PDB magic number")
|
||||
ErrMissingPdbStream = errors.New("Missing PDB stream")
|
||||
)
|
||||
|
||||
type PortablePdb struct {
|
||||
Name string
|
||||
ID string
|
||||
Content *packages.HashedBuffer
|
||||
}
|
||||
|
||||
type PortablePdbList []*PortablePdb
|
||||
|
||||
func (l PortablePdbList) Close() {
|
||||
for _, pdb := range l {
|
||||
pdb.Content.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractPortablePdb extracts PDB files from a .snupkg file
|
||||
func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
|
||||
archive, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pdbs PortablePdbList
|
||||
|
||||
err = func() error {
|
||||
for _, file := range archive.File {
|
||||
if strings.HasSuffix(file.Name, "/") {
|
||||
continue
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(file.Name))
|
||||
|
||||
switch ext {
|
||||
case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s":
|
||||
continue
|
||||
case ".pdb":
|
||||
f, err := archive.Open(file.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024)
|
||||
|
||||
f.Close()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
id, err := ParseDebugHeaderID(buf)
|
||||
if err != nil {
|
||||
buf.Close()
|
||||
return fmt.Errorf("Invalid PDB file: %v", err)
|
||||
}
|
||||
|
||||
if _, err := buf.Seek(0, io.SeekStart); err != nil {
|
||||
buf.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
pdbs = append(pdbs, &PortablePdb{
|
||||
Name: path.Base(file.Name),
|
||||
ID: id,
|
||||
Content: buf,
|
||||
})
|
||||
default:
|
||||
return ErrInvalidFiles
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
pdbs.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(pdbs) == 0 {
|
||||
return nil, ErrMissingPdbFiles
|
||||
}
|
||||
|
||||
return pdbs, nil
|
||||
}
|
||||
|
||||
// ParseDebugHeaderID TODO
|
||||
func ParseDebugHeaderID(r io.ReadSeeker) (string, error) {
|
||||
var magic uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if magic != 0x424A5342 {
|
||||
return "", ErrInvalidPdbMagicNumber
|
||||
}
|
||||
|
||||
if _, err := r.Seek(8, io.SeekCurrent); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var versionStringSize int32
|
||||
if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := r.Seek(2, io.SeekCurrent); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var streamCount int16
|
||||
if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
read4ByteAlignedString := func(r io.Reader) (string, error) {
|
||||
b := make([]byte, 4)
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
if _, err := r.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if i := bytes.IndexByte(b, 0); i != -1 {
|
||||
buf.Write(b[:i])
|
||||
return buf.String(), nil
|
||||
}
|
||||
buf.Write(b)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < int(streamCount); i++ {
|
||||
var offset uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &offset); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := r.Seek(4, io.SeekCurrent); err != nil {
|
||||
return "", err
|
||||
}
|
||||
name, err := read4ByteAlignedString(r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if name == "#Pdb" {
|
||||
if _, err := r.Seek(int64(offset), io.SeekStart); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
b := make([]byte, 16)
|
||||
if _, err := r.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data1 := binary.LittleEndian.Uint32(b[0:4])
|
||||
data2 := binary.LittleEndian.Uint16(b[4:6])
|
||||
data3 := binary.LittleEndian.Uint16(b[6:8])
|
||||
data4 := b[8:16]
|
||||
|
||||
return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", ErrMissingPdbStream
|
||||
}
|
||||
82
modules/packages/nuget/symbol_extractor_test.go
Normal file
82
modules/packages/nuget/symbol_extractor_test.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package nuget
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj
|
||||
fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB
|
||||
AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`
|
||||
|
||||
func TestExtractPortablePdb(t *testing.T) {
|
||||
createArchive := func(name string, content []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
archive := zip.NewWriter(&buf)
|
||||
w, _ := archive.Create(name)
|
||||
w.Write(content)
|
||||
archive.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
t.Run("MissingPdbFiles", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
zip.NewWriter(&buf).Close()
|
||||
|
||||
pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
|
||||
assert.ErrorIs(t, err, ErrMissingPdbFiles)
|
||||
assert.Empty(t, pdbs)
|
||||
})
|
||||
|
||||
t.Run("InvalidFiles", func(t *testing.T) {
|
||||
data := createArchive("sub/test.bin", []byte{})
|
||||
|
||||
pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data)))
|
||||
assert.ErrorIs(t, err, ErrInvalidFiles)
|
||||
assert.Empty(t, pdbs)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
b, _ := base64.StdEncoding.DecodeString(pdbContent)
|
||||
data := createArchive("test.pdb", b)
|
||||
|
||||
pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data)))
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, pdbs, 1)
|
||||
assert.Equal(t, "test.pdb", pdbs[0].Name)
|
||||
assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID)
|
||||
pdbs.Close()
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseDebugHeaderID(t *testing.T) {
|
||||
t.Run("InvalidPdbMagicNumber", func(t *testing.T) {
|
||||
id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0}))
|
||||
assert.ErrorIs(t, err, ErrInvalidPdbMagicNumber)
|
||||
assert.Empty(t, id)
|
||||
})
|
||||
|
||||
t.Run("MissingPdbStream", func(t *testing.T) {
|
||||
b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`)
|
||||
|
||||
id, err := ParseDebugHeaderID(bytes.NewReader(b))
|
||||
assert.ErrorIs(t, err, ErrMissingPdbStream)
|
||||
assert.Empty(t, id)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
b, _ := base64.StdEncoding.DecodeString(pdbContent)
|
||||
|
||||
id, err := ParseDebugHeaderID(bytes.NewReader(b))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id)
|
||||
})
|
||||
}
|
||||
16
modules/packages/pypi/metadata.go
Normal file
16
modules/packages/pypi/metadata.go
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pypi
|
||||
|
||||
// Metadata represents the metadata of a PyPI package
|
||||
type Metadata struct {
|
||||
Author string `json:"author,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
LongDescription string `json:"long_description,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
RequiresPython string `json:"requires_python,omitempty"`
|
||||
}
|
||||
311
modules/packages/rubygems/marshal.go
Normal file
311
modules/packages/rubygems/marshal.go
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rubygems
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
const (
|
||||
majorVersion = 4
|
||||
minorVersion = 8
|
||||
|
||||
typeNil = '0'
|
||||
typeTrue = 'T'
|
||||
typeFalse = 'F'
|
||||
typeFixnum = 'i'
|
||||
typeString = '"'
|
||||
typeSymbol = ':'
|
||||
typeSymbolLink = ';'
|
||||
typeArray = '['
|
||||
typeIVar = 'I'
|
||||
typeUserMarshal = 'U'
|
||||
typeUserDef = 'u'
|
||||
typeObject = 'o'
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrUnsupportedType indicates an unsupported type
|
||||
ErrUnsupportedType = errors.New("Type is unsupported")
|
||||
// ErrInvalidIntRange indicates an invalid number range
|
||||
ErrInvalidIntRange = errors.New("Number is not in valid range")
|
||||
)
|
||||
|
||||
// RubyUserMarshal is a Ruby object that has a marshal_load function.
|
||||
type RubyUserMarshal struct {
|
||||
Name string
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// RubyUserDef is a Ruby object that has a _load function.
|
||||
type RubyUserDef struct {
|
||||
Name string
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// RubyObject is a default Ruby object.
|
||||
type RubyObject struct {
|
||||
Name string
|
||||
Member map[string]interface{}
|
||||
}
|
||||
|
||||
// MarshalEncoder mimics Rubys Marshal class.
|
||||
// Note: Only supports types used by the RubyGems package registry.
|
||||
type MarshalEncoder struct {
|
||||
w *bufio.Writer
|
||||
symbols map[string]int
|
||||
}
|
||||
|
||||
// NewMarshalEncoder creates a new MarshalEncoder
|
||||
func NewMarshalEncoder(w io.Writer) *MarshalEncoder {
|
||||
return &MarshalEncoder{
|
||||
w: bufio.NewWriter(w),
|
||||
symbols: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
// Encode encodes the given type
|
||||
func (e *MarshalEncoder) Encode(v interface{}) error {
|
||||
if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshal(v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.w.Flush()
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshal(v interface{}) error {
|
||||
if v == nil {
|
||||
return e.marshalNil()
|
||||
}
|
||||
|
||||
val := reflect.ValueOf(v)
|
||||
typ := reflect.TypeOf(v)
|
||||
|
||||
if typ.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
typ = typ.Elem()
|
||||
}
|
||||
|
||||
switch typ.Kind() {
|
||||
case reflect.Bool:
|
||||
return e.marshalBool(val.Bool())
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
|
||||
return e.marshalInt(val.Int())
|
||||
case reflect.String:
|
||||
return e.marshalString(val.String())
|
||||
case reflect.Slice, reflect.Array:
|
||||
return e.marshalArray(val)
|
||||
}
|
||||
|
||||
switch typ.Name() {
|
||||
case "RubyUserMarshal":
|
||||
return e.marshalUserMarshal(val.Interface().(RubyUserMarshal))
|
||||
case "RubyUserDef":
|
||||
return e.marshalUserDef(val.Interface().(RubyUserDef))
|
||||
case "RubyObject":
|
||||
return e.marshalObject(val.Interface().(RubyObject))
|
||||
}
|
||||
|
||||
return ErrUnsupportedType
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalNil() error {
|
||||
return e.w.WriteByte(typeNil)
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalBool(b bool) error {
|
||||
if b {
|
||||
return e.w.WriteByte(typeTrue)
|
||||
}
|
||||
return e.w.WriteByte(typeFalse)
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalInt(i int64) error {
|
||||
if err := e.w.WriteByte(typeFixnum); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.marshalIntInternal(i)
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalIntInternal(i int64) error {
|
||||
if i == 0 {
|
||||
return e.w.WriteByte(0)
|
||||
} else if 0 < i && i < 123 {
|
||||
return e.w.WriteByte(byte(i + 5))
|
||||
} else if -124 < i && i <= -1 {
|
||||
return e.w.WriteByte(byte(i - 5))
|
||||
}
|
||||
|
||||
var len int
|
||||
if 122 < i && i <= 0xff {
|
||||
len = 1
|
||||
} else if 0xff < i && i <= 0xffff {
|
||||
len = 2
|
||||
} else if 0xffff < i && i <= 0xffffff {
|
||||
len = 3
|
||||
} else if 0xffffff < i && i <= 0x3fffffff {
|
||||
len = 4
|
||||
} else if -0x100 <= i && i < -123 {
|
||||
len = -1
|
||||
} else if -0x10000 <= i && i < -0x100 {
|
||||
len = -2
|
||||
} else if -0x1000000 <= i && i < -0x100000 {
|
||||
len = -3
|
||||
} else if -0x40000000 <= i && i < -0x1000000 {
|
||||
len = -4
|
||||
} else {
|
||||
return ErrInvalidIntRange
|
||||
}
|
||||
|
||||
if err := e.w.WriteByte(byte(len)); err != nil {
|
||||
return err
|
||||
}
|
||||
if len < 0 {
|
||||
len = -len
|
||||
}
|
||||
|
||||
for c := 0; c < len; c++ {
|
||||
if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalString(str string) error {
|
||||
if err := e.w.WriteByte(typeIVar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalRawString(str); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalIntInternal(1); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalSymbol("E"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.marshalBool(true)
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalRawString(str string) error {
|
||||
if err := e.w.WriteByte(typeString); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalIntInternal(int64(len(str))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := e.w.WriteString(str)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalSymbol(str string) error {
|
||||
if index, ok := e.symbols[str]; ok {
|
||||
if err := e.w.WriteByte(typeSymbolLink); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.marshalIntInternal(int64(index))
|
||||
}
|
||||
|
||||
e.symbols[str] = len(e.symbols)
|
||||
|
||||
if err := e.w.WriteByte(typeSymbol); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalIntInternal(int64(len(str))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := e.w.WriteString(str)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalArray(arr reflect.Value) error {
|
||||
if err := e.w.WriteByte(typeArray); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
len := arr.Len()
|
||||
|
||||
if err := e.marshalIntInternal(int64(len)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < len; i++ {
|
||||
if err := e.marshal(arr.Index(i).Interface()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error {
|
||||
if err := e.w.WriteByte(typeUserMarshal); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.marshalSymbol(userMarshal.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.marshal(userMarshal.Value)
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error {
|
||||
var buf bytes.Buffer
|
||||
if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.w.WriteByte(typeUserDef); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.marshalSymbol(userDef.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.marshalIntInternal(int64(buf.Len())); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := e.w.Write(buf.Bytes())
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *MarshalEncoder) marshalObject(obj RubyObject) error {
|
||||
if err := e.w.WriteByte(typeObject); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.marshalSymbol(obj.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range obj.Member {
|
||||
if err := e.marshalSymbol(k); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.marshal(v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
99
modules/packages/rubygems/marshal_test.go
Normal file
99
modules/packages/rubygems/marshal_test.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rubygems
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMinimalEncoder(t *testing.T) {
|
||||
cases := []struct {
|
||||
Value interface{}
|
||||
Expected []byte
|
||||
Error error
|
||||
}{
|
||||
{
|
||||
Value: nil,
|
||||
Expected: []byte{4, 8, 0x30},
|
||||
},
|
||||
{
|
||||
Value: true,
|
||||
Expected: []byte{4, 8, 'T'},
|
||||
},
|
||||
{
|
||||
Value: false,
|
||||
Expected: []byte{4, 8, 'F'},
|
||||
},
|
||||
{
|
||||
Value: 0,
|
||||
Expected: []byte{4, 8, 'i', 0},
|
||||
},
|
||||
{
|
||||
Value: 1,
|
||||
Expected: []byte{4, 8, 'i', 6},
|
||||
},
|
||||
{
|
||||
Value: -1,
|
||||
Expected: []byte{4, 8, 'i', 0xfa},
|
||||
},
|
||||
{
|
||||
Value: 0x1fffffff,
|
||||
Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f},
|
||||
},
|
||||
{
|
||||
Value: 0x41000000,
|
||||
Error: ErrInvalidIntRange,
|
||||
},
|
||||
{
|
||||
Value: "test",
|
||||
Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'},
|
||||
},
|
||||
{
|
||||
Value: []int{1, 2},
|
||||
Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7},
|
||||
},
|
||||
{
|
||||
Value: &RubyUserMarshal{
|
||||
Name: "Test",
|
||||
Value: 4,
|
||||
},
|
||||
Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9},
|
||||
},
|
||||
{
|
||||
Value: &RubyUserDef{
|
||||
Name: "Test",
|
||||
Value: 4,
|
||||
},
|
||||
Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9},
|
||||
},
|
||||
{
|
||||
Value: &RubyObject{
|
||||
Name: "Test",
|
||||
Member: map[string]interface{}{
|
||||
"test": 4,
|
||||
},
|
||||
},
|
||||
Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9},
|
||||
},
|
||||
{
|
||||
Value: &struct {
|
||||
Name string
|
||||
}{
|
||||
"test",
|
||||
},
|
||||
Error: ErrUnsupportedType,
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
var b bytes.Buffer
|
||||
err := NewMarshalEncoder(&b).Encode(c.Value)
|
||||
assert.ErrorIs(t, err, c.Error)
|
||||
assert.Equal(t, c.Expected, b.Bytes(), "case %d", i)
|
||||
}
|
||||
}
|
||||
222
modules/packages/rubygems/metadata.go
Normal file
222
modules/packages/rubygems/metadata.go
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rubygems
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMissingMetadataFile indicates a missing metadata.gz file
|
||||
ErrMissingMetadataFile = errors.New("Metadata file is missing")
|
||||
// ErrInvalidName indicates an invalid id in the metadata.gz file
|
||||
ErrInvalidName = errors.New("Metadata file contains an invalid name")
|
||||
// ErrInvalidVersion indicates an invalid version in the metadata.gz file
|
||||
ErrInvalidVersion = errors.New("Metadata file contains an invalid version")
|
||||
)
|
||||
|
||||
var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`)
|
||||
|
||||
// Package represents a RubyGems package
|
||||
type Package struct {
|
||||
Name string
|
||||
Version string
|
||||
Metadata *Metadata
|
||||
}
|
||||
|
||||
// Metadata represents the metadata of a RubyGems package
|
||||
type Metadata struct {
|
||||
Platform string `json:"platform,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Authors []string `json:"authors,omitempty"`
|
||||
Licenses []string `json:"licenses,omitempty"`
|
||||
RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"`
|
||||
RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"`
|
||||
ProjectURL string `json:"project_url,omitempty"`
|
||||
RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"`
|
||||
DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"`
|
||||
}
|
||||
|
||||
// VersionRequirement represents a version restriction
|
||||
type VersionRequirement struct {
|
||||
Restriction string `json:"restriction"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// Dependency represents a dependency of a RubyGems package
|
||||
type Dependency struct {
|
||||
Name string `json:"name"`
|
||||
Version []VersionRequirement `json:"version"`
|
||||
}
|
||||
|
||||
type gemspec struct {
|
||||
Name string `yaml:"name"`
|
||||
Version struct {
|
||||
Version string `yaml:"version"`
|
||||
} `yaml:"version"`
|
||||
Platform string `yaml:"platform"`
|
||||
Authors []string `yaml:"authors"`
|
||||
Autorequire interface{} `yaml:"autorequire"`
|
||||
Bindir string `yaml:"bindir"`
|
||||
CertChain []interface{} `yaml:"cert_chain"`
|
||||
Date string `yaml:"date"`
|
||||
Dependencies []struct {
|
||||
Name string `yaml:"name"`
|
||||
Requirement requirement `yaml:"requirement"`
|
||||
Type string `yaml:"type"`
|
||||
Prerelease bool `yaml:"prerelease"`
|
||||
VersionRequirements requirement `yaml:"version_requirements"`
|
||||
} `yaml:"dependencies"`
|
||||
Description string `yaml:"description"`
|
||||
Email string `yaml:"email"`
|
||||
Executables []string `yaml:"executables"`
|
||||
Extensions []interface{} `yaml:"extensions"`
|
||||
ExtraRdocFiles []string `yaml:"extra_rdoc_files"`
|
||||
Files []string `yaml:"files"`
|
||||
Homepage string `yaml:"homepage"`
|
||||
Licenses []string `yaml:"licenses"`
|
||||
Metadata struct {
|
||||
BugTrackerURI string `yaml:"bug_tracker_uri"`
|
||||
ChangelogURI string `yaml:"changelog_uri"`
|
||||
DocumentationURI string `yaml:"documentation_uri"`
|
||||
SourceCodeURI string `yaml:"source_code_uri"`
|
||||
} `yaml:"metadata"`
|
||||
PostInstallMessage interface{} `yaml:"post_install_message"`
|
||||
RdocOptions []interface{} `yaml:"rdoc_options"`
|
||||
RequirePaths []string `yaml:"require_paths"`
|
||||
RequiredRubyVersion requirement `yaml:"required_ruby_version"`
|
||||
RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"`
|
||||
Requirements []interface{} `yaml:"requirements"`
|
||||
RubygemsVersion string `yaml:"rubygems_version"`
|
||||
SigningKey interface{} `yaml:"signing_key"`
|
||||
SpecificationVersion int `yaml:"specification_version"`
|
||||
Summary string `yaml:"summary"`
|
||||
TestFiles []interface{} `yaml:"test_files"`
|
||||
}
|
||||
|
||||
type requirement struct {
|
||||
Requirements [][]interface{} `yaml:"requirements"`
|
||||
}
|
||||
|
||||
// AsVersionRequirement converts into []VersionRequirement
|
||||
func (r requirement) AsVersionRequirement() []VersionRequirement {
|
||||
requirements := make([]VersionRequirement, 0, len(r.Requirements))
|
||||
for _, req := range r.Requirements {
|
||||
if len(req) != 2 {
|
||||
continue
|
||||
}
|
||||
restriction, ok := req[0].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
vm, ok := req[1].(map[interface{}]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
versionInt, ok := vm["version"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
version, ok := versionInt.(string)
|
||||
if !ok || version == "0" {
|
||||
continue
|
||||
}
|
||||
|
||||
requirements = append(requirements, VersionRequirement{
|
||||
Restriction: restriction,
|
||||
Version: version,
|
||||
})
|
||||
}
|
||||
return requirements
|
||||
}
|
||||
|
||||
// ParsePackageMetaData parses the metadata of a Gem package file
|
||||
func ParsePackageMetaData(r io.Reader) (*Package, error) {
|
||||
archive := tar.NewReader(r)
|
||||
for {
|
||||
hdr, err := archive.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if hdr.Name == "metadata.gz" {
|
||||
return parseMetadataFile(archive)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, ErrMissingMetadataFile
|
||||
}
|
||||
|
||||
func parseMetadataFile(r io.Reader) (*Package, error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
var spec gemspec
|
||||
if err := yaml.NewDecoder(zr).Decode(&spec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") {
|
||||
return nil, ErrInvalidName
|
||||
}
|
||||
|
||||
if !versionMatcher.MatchString(spec.Version.Version) {
|
||||
return nil, ErrInvalidVersion
|
||||
}
|
||||
|
||||
if !validation.IsValidURL(spec.Homepage) {
|
||||
spec.Homepage = ""
|
||||
}
|
||||
if !validation.IsValidURL(spec.Metadata.SourceCodeURI) {
|
||||
spec.Metadata.SourceCodeURI = ""
|
||||
}
|
||||
|
||||
m := &Metadata{
|
||||
Platform: spec.Platform,
|
||||
Description: spec.Description,
|
||||
Summary: spec.Summary,
|
||||
Authors: spec.Authors,
|
||||
Licenses: spec.Licenses,
|
||||
ProjectURL: spec.Homepage,
|
||||
RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(),
|
||||
RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(),
|
||||
DevelopmentDependencies: make([]Dependency, 0, 5),
|
||||
RuntimeDependencies: make([]Dependency, 0, 5),
|
||||
}
|
||||
|
||||
for _, gemdep := range spec.Dependencies {
|
||||
dep := Dependency{
|
||||
Name: gemdep.Name,
|
||||
Version: gemdep.Requirement.AsVersionRequirement(),
|
||||
}
|
||||
if gemdep.Type == ":runtime" {
|
||||
m.RuntimeDependencies = append(m.RuntimeDependencies, dep)
|
||||
} else {
|
||||
m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep)
|
||||
}
|
||||
}
|
||||
|
||||
return &Package{
|
||||
Name: spec.Name,
|
||||
Version: spec.Version.Version,
|
||||
Metadata: m,
|
||||
}, nil
|
||||
}
|
||||
89
modules/packages/rubygems/metadata_test.go
Normal file
89
modules/packages/rubygems/metadata_test.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rubygems
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParsePackageMetaData(t *testing.T) {
|
||||
createArchive := func(filename string, content []byte) io.Reader {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
hdr := &tar.Header{
|
||||
Name: filename,
|
||||
Mode: 0o600,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
tw.WriteHeader(hdr)
|
||||
tw.Write(content)
|
||||
tw.Close()
|
||||
return &buf
|
||||
}
|
||||
|
||||
t.Run("MissingMetadataFile", func(t *testing.T) {
|
||||
data := createArchive("dummy.txt", []byte{0})
|
||||
|
||||
rp, err := ParsePackageMetaData(data)
|
||||
assert.ErrorIs(t, err, ErrMissingMetadataFile)
|
||||
assert.Nil(t, rp)
|
||||
})
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=")
|
||||
data := createArchive("metadata.gz", content)
|
||||
|
||||
rp, err := ParsePackageMetaData(data)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, rp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseMetadataFile(t *testing.T) {
|
||||
content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd
|
||||
0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4
|
||||
bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS
|
||||
R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z
|
||||
d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n
|
||||
uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0
|
||||
+TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS
|
||||
dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/
|
||||
yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi
|
||||
4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`)
|
||||
rp, err := parseMetadataFile(bytes.NewReader(content))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, rp)
|
||||
|
||||
assert.Equal(t, "gitea", rp.Name)
|
||||
assert.Equal(t, "1.0.5", rp.Version)
|
||||
assert.Equal(t, "ruby", rp.Metadata.Platform)
|
||||
assert.Equal(t, "Gitea package", rp.Metadata.Summary)
|
||||
assert.Equal(t, "RubyGems package test", rp.Metadata.Description)
|
||||
assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors)
|
||||
assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL)
|
||||
assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses)
|
||||
assert.Empty(t, rp.Metadata.RequiredRubygemsVersion)
|
||||
assert.Len(t, rp.Metadata.RequiredRubyVersion, 1)
|
||||
assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction)
|
||||
assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version)
|
||||
assert.Len(t, rp.Metadata.RuntimeDependencies, 1)
|
||||
assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name)
|
||||
assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2)
|
||||
assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction)
|
||||
assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version)
|
||||
assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction)
|
||||
assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version)
|
||||
assert.Len(t, rp.Metadata.DevelopmentDependencies, 1)
|
||||
assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name)
|
||||
assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1)
|
||||
assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction)
|
||||
assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version)
|
||||
}
|
||||
47
modules/setting/packages.go
Normal file
47
modules/setting/packages.go
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package setting
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
)
|
||||
|
||||
// Package registry settings
|
||||
var (
|
||||
Packages = struct {
|
||||
Storage
|
||||
Enabled bool
|
||||
ChunkedUploadPath string
|
||||
RegistryHost string
|
||||
}{
|
||||
Enabled: true,
|
||||
}
|
||||
)
|
||||
|
||||
func newPackages() {
|
||||
sec := Cfg.Section("packages")
|
||||
if err := sec.MapTo(&Packages); err != nil {
|
||||
log.Fatal("Failed to map Packages settings: %v", err)
|
||||
}
|
||||
|
||||
Packages.Storage = getStorage("packages", "", nil)
|
||||
|
||||
Packages.RegistryHost = Domain
|
||||
if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") {
|
||||
Packages.RegistryHost += ":" + HTTPPort
|
||||
}
|
||||
|
||||
Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload"))
|
||||
if !filepath.IsAbs(Packages.ChunkedUploadPath) {
|
||||
Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath))
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
|
||||
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -212,6 +212,7 @@ var (
|
|||
MembersPagingNum int
|
||||
FeedMaxCommitNum int
|
||||
FeedPagingNum int
|
||||
PackagesPagingNum int
|
||||
GraphMaxCommitNum int
|
||||
CodeCommentLines int
|
||||
ReactionMaxUserNum int
|
||||
|
|
@ -264,6 +265,7 @@ var (
|
|||
MembersPagingNum: 20,
|
||||
FeedMaxCommitNum: 5,
|
||||
FeedPagingNum: 20,
|
||||
PackagesPagingNum: 20,
|
||||
GraphMaxCommitNum: 100,
|
||||
CodeCommentLines: 4,
|
||||
ReactionMaxUserNum: 10,
|
||||
|
|
@ -1016,6 +1018,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
|
|||
|
||||
newPictureService()
|
||||
|
||||
newPackages()
|
||||
|
||||
if err = Cfg.Section("ui").MapTo(&UI); err != nil {
|
||||
log.Fatal("Failed to map UI settings: %v", err)
|
||||
} else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil {
|
||||
|
|
|
|||
|
|
@ -123,6 +123,9 @@ var (
|
|||
|
||||
// RepoArchives represents repository archives storage
|
||||
RepoArchives ObjectStorage
|
||||
|
||||
// Packages represents packages storage
|
||||
Packages ObjectStorage
|
||||
)
|
||||
|
||||
// Init init the stoarge
|
||||
|
|
@ -143,7 +146,11 @@ func Init() error {
|
|||
return err
|
||||
}
|
||||
|
||||
return initRepoArchives()
|
||||
if err := initRepoArchives(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return initPackages()
|
||||
}
|
||||
|
||||
// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
|
||||
|
|
@ -188,3 +195,9 @@ func initRepoArchives() (err error) {
|
|||
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
|
||||
return
|
||||
}
|
||||
|
||||
func initPackages() (err error) {
|
||||
log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type)
|
||||
Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ var (
|
|||
_ Payloader = &PullRequestPayload{}
|
||||
_ Payloader = &RepositoryPayload{}
|
||||
_ Payloader = &ReleasePayload{}
|
||||
_ Payloader = &PackagePayload{}
|
||||
)
|
||||
|
||||
// _________ __
|
||||
|
|
@ -425,3 +426,27 @@ type RepositoryPayload struct {
|
|||
func (p *RepositoryPayload) JSONPayload() ([]byte, error) {
|
||||
return json.MarshalIndent(p, "", " ")
|
||||
}
|
||||
|
||||
// HookPackageAction an action that happens to a package
|
||||
type HookPackageAction string
|
||||
|
||||
const (
|
||||
// HookPackageCreated created
|
||||
HookPackageCreated HookPackageAction = "created"
|
||||
// HookPackageDeleted deleted
|
||||
HookPackageDeleted HookPackageAction = "deleted"
|
||||
)
|
||||
|
||||
// PackagePayload represents a package payload
|
||||
type PackagePayload struct {
|
||||
Action HookPackageAction `json:"action"`
|
||||
Repository *Repository `json:"repository"`
|
||||
Package *Package `json:"package"`
|
||||
Organization *User `json:"organization"`
|
||||
Sender *User `json:"sender"`
|
||||
}
|
||||
|
||||
// JSONPayload implements Payload
|
||||
func (p *PackagePayload) JSONPayload() ([]byte, error) {
|
||||
return json.MarshalIndent(p, "", " ")
|
||||
}
|
||||
|
|
|
|||
33
modules/structs/package.go
Normal file
33
modules/structs/package.go
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package structs
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Package represents a package
|
||||
type Package struct {
|
||||
ID int64 `json:"id"`
|
||||
Owner *User `json:"owner"`
|
||||
Repository *Repository `json:"repository"`
|
||||
Creator *User `json:"creator"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
// swagger:strfmt date-time
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// PackageFile represents a package file
|
||||
type PackageFile struct {
|
||||
ID int64 `json:"id"`
|
||||
Size int64
|
||||
Name string `json:"name"`
|
||||
HashMD5 string `json:"md5"`
|
||||
HashSHA1 string `json:"sha1"`
|
||||
HashSHA256 string `json:"sha256"`
|
||||
HashSHA512 string `json:"sha512"`
|
||||
}
|
||||
|
|
@ -34,6 +34,7 @@ import (
|
|||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/markup"
|
||||
"code.gitea.io/gitea/modules/markup/markdown"
|
||||
"code.gitea.io/gitea/modules/repository"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/svg"
|
||||
|
|
@ -161,7 +162,16 @@ func NewFuncMap() []template.FuncMap {
|
|||
"RenderEmojiPlain": emoji.ReplaceAliases,
|
||||
"ReactionToEmoji": ReactionToEmoji,
|
||||
"RenderNote": RenderNote,
|
||||
"IsMultilineCommitMessage": IsMultilineCommitMessage,
|
||||
"RenderMarkdownToHtml": func(input string) template.HTML {
|
||||
output, err := markdown.RenderString(&markup.RenderContext{
|
||||
URLPrefix: setting.AppSubURL,
|
||||
}, input)
|
||||
if err != nil {
|
||||
log.Error("RenderString: %v", err)
|
||||
}
|
||||
return template.HTML(output)
|
||||
},
|
||||
"IsMultilineCommitMessage": IsMultilineCommitMessage,
|
||||
"ThemeColorMetaTag": func() string {
|
||||
return setting.UI.ThemeColorMetaTag
|
||||
},
|
||||
|
|
|
|||
147
modules/util/filebuffer/file_backed_buffer.go
Normal file
147
modules/util/filebuffer/file_backed_buffer.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package filebuffer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer
|
||||
|
||||
var (
|
||||
// ErrInvalidMemorySize occurs if the memory size is not in a valid range
|
||||
ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32")
|
||||
// ErrWriteAfterRead occurs if Write is called after a read operation
|
||||
ErrWriteAfterRead = errors.New("Write is unsupported after a read operation")
|
||||
)
|
||||
|
||||
type readAtSeeker interface {
|
||||
io.ReadSeeker
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
// FileBackedBuffer uses a memory buffer with a fixed size.
|
||||
// If more data is written a temporary file is used instead.
|
||||
// It implements io.ReadWriteCloser, io.ReadSeekCloser and io.ReaderAt
|
||||
type FileBackedBuffer struct {
|
||||
maxMemorySize int64
|
||||
size int64
|
||||
buffer bytes.Buffer
|
||||
file *os.File
|
||||
reader readAtSeeker
|
||||
}
|
||||
|
||||
// New creates a file backed buffer with a specific maximum memory size
|
||||
func New(maxMemorySize int) (*FileBackedBuffer, error) {
|
||||
if maxMemorySize < 0 || maxMemorySize > maxInt {
|
||||
return nil, ErrInvalidMemorySize
|
||||
}
|
||||
|
||||
return &FileBackedBuffer{
|
||||
maxMemorySize: int64(maxMemorySize),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateFromReader creates a file backed buffer and copies the provided reader data into it.
|
||||
func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) {
|
||||
b, err := New(maxMemorySize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = io.Copy(b, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (b *FileBackedBuffer) Write(p []byte) (int, error) {
|
||||
if b.reader != nil {
|
||||
return 0, ErrWriteAfterRead
|
||||
}
|
||||
|
||||
var n int
|
||||
var err error
|
||||
|
||||
if b.file != nil {
|
||||
n, err = b.file.Write(p)
|
||||
} else {
|
||||
if b.size+int64(len(p)) > b.maxMemorySize {
|
||||
b.file, err = os.CreateTemp("", "gitea-buffer-")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
_, err = io.Copy(b.file, &b.buffer)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return b.Write(p)
|
||||
}
|
||||
|
||||
n, err = b.buffer.Write(p)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
b.size += int64(n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Size returns the byte size of the buffered data
|
||||
func (b *FileBackedBuffer) Size() int64 {
|
||||
return b.size
|
||||
}
|
||||
|
||||
func (b *FileBackedBuffer) switchToReader() {
|
||||
if b.reader != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if b.file != nil {
|
||||
b.reader = b.file
|
||||
} else {
|
||||
b.reader = bytes.NewReader(b.buffer.Bytes())
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (b *FileBackedBuffer) Read(p []byte) (int, error) {
|
||||
b.switchToReader()
|
||||
|
||||
return b.reader.Read(p)
|
||||
}
|
||||
|
||||
// ReadAt implements io.ReaderAt
|
||||
func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) {
|
||||
b.switchToReader()
|
||||
|
||||
return b.reader.ReadAt(p, off)
|
||||
}
|
||||
|
||||
// Seek implements io.Seeker
|
||||
func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) {
|
||||
b.switchToReader()
|
||||
|
||||
return b.reader.Seek(offset, whence)
|
||||
}
|
||||
|
||||
// Close implements io.Closer
|
||||
func (b *FileBackedBuffer) Close() error {
|
||||
if b.file != nil {
|
||||
err := b.file.Close()
|
||||
os.Remove(b.file.Name())
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue