Add Debian package registry (#24426)

Co-authored-by: @awkwardbunny

This PR adds a Debian package registry.
You can follow [this
tutorial](https://www.baeldung.com/linux/create-debian-package) to build
a *.deb package for testing.
Source packages are not supported at the moment and I did not find
documentation of the architecture "all" and how these packages should be
treated.


![grafik](https://user-images.githubusercontent.com/1666336/218126879-eb80a866-775c-4c8e-8529-5797203a64e6.png)

Part of #20751.

Revised copy of #22854.

---------

Co-authored-by: Brian Hong <brian@hongs.me>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Giteabot <teabot@gitea.io>

modules/packages/debian/metadata.go

@@ -0,0 +1,218 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"io"
+	"net/mail"
+	"regexp"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/validation"
+
+	"github.com/blakesmith/ar"
+	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
+)
+
+const (
+	PropertyDistribution               = "debian.distribution"
+	PropertyComponent                  = "debian.component"
+	PropertyArchitecture               = "debian.architecture"
+	PropertyControl                    = "debian.control"
+	PropertyRepositoryIncludeInRelease = "debian.repository.include_in_release"
+
+	SettingKeyPrivate = "debian.key.private"
+	SettingKeyPublic  = "debian.key.public"
+
+	RepositoryPackage = "_debian"
+	RepositoryVersion = "_repository"
+
+	controlTar = "control.tar"
+)
+
+var (
+	ErrMissingControlFile     = util.NewInvalidArgumentErrorf("control file is missing")
+	ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithm")
+	ErrInvalidName            = util.NewInvalidArgumentErrorf("package name is invalid")
+	ErrInvalidVersion         = util.NewInvalidArgumentErrorf("package version is invalid")
+	ErrInvalidArchitecture    = util.NewInvalidArgumentErrorf("package architecture is invalid")
+
+	// https://www.debian.org/doc/debian-policy/ch-controlfields.html#source
+	namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`)
+	// https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
+	versionPattern = regexp.MustCompile(`\A(?:[0-9]:)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`)
+)
+
+type Package struct {
+	Name         string
+	Version      string
+	Architecture string
+	Control      string
+	Metadata     *Metadata
+}
+
+type Metadata struct {
+	Maintainer   string   `json:"maintainer,omitempty"`
+	ProjectURL   string   `json:"project_url,omitempty"`
+	Description  string   `json:"description,omitempty"`
+	Dependencies []string `json:"dependencies,omitempty"`
+}
+
+// ParsePackage parses the Debian package file
+// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html
+func ParsePackage(r io.Reader) (*Package, error) {
+	arr := ar.NewReader(r)
+
+	for {
+		hd, err := arr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		if strings.HasPrefix(hd.Name, controlTar) {
+			var inner io.Reader
+			switch hd.Name[len(controlTar):] {
+			case "":
+				inner = arr
+			case ".gz":
+				gzr, err := gzip.NewReader(arr)
+				if err != nil {
+					return nil, err
+				}
+				defer gzr.Close()
+
+				inner = gzr
+			case ".xz":
+				xzr, err := xz.NewReader(arr)
+				if err != nil {
+					return nil, err
+				}
+
+				inner = xzr
+			case ".zst":
+				zr, err := zstd.NewReader(arr)
+				if err != nil {
+					return nil, err
+				}
+				defer zr.Close()
+
+				inner = zr
+			default:
+				return nil, ErrUnsupportedCompression
+			}
+
+			tr := tar.NewReader(inner)
+			for {
+				hd, err := tr.Next()
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					return nil, err
+				}
+
+				if hd.Typeflag != tar.TypeReg {
+					continue
+				}
+
+				if hd.FileInfo().Name() == "control" {
+					return ParseControlFile(tr)
+				}
+			}
+		}
+	}
+
+	return nil, ErrMissingControlFile
+}
+
+// ParseControlFile parses a Debian control file to retrieve the metadata
+func ParseControlFile(r io.Reader) (*Package, error) {
+	p := &Package{
+		Metadata: &Metadata{},
+	}
+
+	key := ""
+	var depends strings.Builder
+	var control strings.Builder
+
+	s := bufio.NewScanner(io.TeeReader(r, &control))
+	for s.Scan() {
+		line := s.Text()
+
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			continue
+		}
+
+		if line[0] == ' ' || line[0] == '\t' {
+			switch key {
+			case "Description":
+				p.Metadata.Description += line
+			case "Depends":
+				depends.WriteString(trimmed)
+			}
+		} else {
+			parts := strings.SplitN(trimmed, ":", 2)
+			if len(parts) < 2 {
+				continue
+			}
+
+			key = parts[0]
+			value := strings.TrimSpace(parts[1])
+			switch key {
+			case "Package":
+				if !namePattern.MatchString(value) {
+					return nil, ErrInvalidName
+				}
+				p.Name = value
+			case "Version":
+				if !versionPattern.MatchString(value) {
+					return nil, ErrInvalidVersion
+				}
+				p.Version = value
+			case "Architecture":
+				if value == "" {
+					return nil, ErrInvalidArchitecture
+				}
+				p.Architecture = value
+			case "Maintainer":
+				a, err := mail.ParseAddress(value)
+				if err != nil || a.Name == "" {
+					p.Metadata.Maintainer = value
+				} else {
+					p.Metadata.Maintainer = a.Name
+				}
+			case "Description":
+				p.Metadata.Description = value
+			case "Depends":
+				depends.WriteString(value)
+			case "Homepage":
+				if validation.IsValidURL(value) {
+					p.Metadata.ProjectURL = value
+				}
+			}
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, err
+	}
+
+	dependencies := strings.Split(depends.String(), ",")
+	for i := range dependencies {
+		dependencies[i] = strings.TrimSpace(dependencies[i])
+	}
+	p.Metadata.Dependencies = dependencies
+
+	p.Control = control.String()
+
+	return p, nil
+}

modules/packages/debian/metadata_test.go

@@ -0,0 +1,171 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"io"
+	"testing"
+
+	"github.com/blakesmith/ar"
+	"github.com/klauspost/compress/zstd"
+	"github.com/stretchr/testify/assert"
+	"github.com/ulikunitz/xz"
+)
+
+const (
+	packageName         = "gitea"
+	packageVersion      = "0:1.0.1-te~st"
+	packageArchitecture = "amd64"
+	packageAuthor       = "KN4CK3R"
+	description         = "Description with multiple lines."
+	projectURL          = "https://gitea.io"
+)
+
+func TestParsePackage(t *testing.T) {
+	createArchive := func(files map[string][]byte) io.Reader {
+		var buf bytes.Buffer
+		aw := ar.NewWriter(&buf)
+		aw.WriteGlobalHeader()
+		for filename, content := range files {
+			hdr := &ar.Header{
+				Name: filename,
+				Mode: 0o600,
+				Size: int64(len(content)),
+			}
+			aw.WriteHeader(hdr)
+			aw.Write(content)
+		}
+		return &buf
+	}
+
+	t.Run("MissingControlFile", func(t *testing.T) {
+		data := createArchive(map[string][]byte{"dummy.txt": {}})
+
+		p, err := ParsePackage(data)
+		assert.Nil(t, p)
+		assert.ErrorIs(t, err, ErrMissingControlFile)
+	})
+
+	t.Run("Compression", func(t *testing.T) {
+		t.Run("Unsupported", func(t *testing.T) {
+			data := createArchive(map[string][]byte{"control.tar.foo": {}})
+
+			p, err := ParsePackage(data)
+			assert.Nil(t, p)
+			assert.ErrorIs(t, err, ErrUnsupportedCompression)
+		})
+
+		var buf bytes.Buffer
+		tw := tar.NewWriter(&buf)
+		tw.WriteHeader(&tar.Header{
+			Name: "control",
+			Mode: 0o600,
+			Size: 50,
+		})
+		tw.Write([]byte("Package: gitea\nVersion: 1.0.0\nArchitecture: amd64\n"))
+		tw.Close()
+
+		t.Run("None", func(t *testing.T) {
+			data := createArchive(map[string][]byte{"control.tar": buf.Bytes()})
+
+			p, err := ParsePackage(data)
+			assert.NotNil(t, p)
+			assert.NoError(t, err)
+			assert.Equal(t, "gitea", p.Name)
+		})
+
+		t.Run("gz", func(t *testing.T) {
+			var zbuf bytes.Buffer
+			zw := gzip.NewWriter(&zbuf)
+			zw.Write(buf.Bytes())
+			zw.Close()
+
+			data := createArchive(map[string][]byte{"control.tar.gz": zbuf.Bytes()})
+
+			p, err := ParsePackage(data)
+			assert.NotNil(t, p)
+			assert.NoError(t, err)
+			assert.Equal(t, "gitea", p.Name)
+		})
+
+		t.Run("xz", func(t *testing.T) {
+			var xbuf bytes.Buffer
+			xw, _ := xz.NewWriter(&xbuf)
+			xw.Write(buf.Bytes())
+			xw.Close()
+
+			data := createArchive(map[string][]byte{"control.tar.xz": xbuf.Bytes()})
+
+			p, err := ParsePackage(data)
+			assert.NotNil(t, p)
+			assert.NoError(t, err)
+			assert.Equal(t, "gitea", p.Name)
+		})
+
+		t.Run("zst", func(t *testing.T) {
+			var zbuf bytes.Buffer
+			zw, _ := zstd.NewWriter(&zbuf)
+			zw.Write(buf.Bytes())
+			zw.Close()
+
+			data := createArchive(map[string][]byte{"control.tar.zst": zbuf.Bytes()})
+
+			p, err := ParsePackage(data)
+			assert.NotNil(t, p)
+			assert.NoError(t, err)
+			assert.Equal(t, "gitea", p.Name)
+		})
+	})
+}
+
+func TestParseControlFile(t *testing.T) {
+	buildContent := func(name, version, architecture string) *bytes.Buffer {
+		var buf bytes.Buffer
+		buf.WriteString("Package: " + name + "\nVersion: " + version + "\nArchitecture: " + architecture + "\nMaintainer: " + packageAuthor + " <kn4ck3r@gitea.io>\nHomepage: " + projectURL + "\nDepends: a,\n b\nDescription: Description\n with multiple\n lines.")
+		return &buf
+	}
+
+	t.Run("InvalidName", func(t *testing.T) {
+		for _, name := range []string{"", "-cd"} {
+			p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture))
+			assert.Nil(t, p)
+			assert.ErrorIs(t, err, ErrInvalidName)
+		}
+	})
+
+	t.Run("InvalidVersion", func(t *testing.T) {
+		for _, version := range []string{"", "1-", ":1.0", "1_0"} {
+			p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture))
+			assert.Nil(t, p)
+			assert.ErrorIs(t, err, ErrInvalidVersion)
+		}
+	})
+
+	t.Run("InvalidArchitecture", func(t *testing.T) {
+		p, err := ParseControlFile(buildContent(packageName, packageVersion, ""))
+		assert.Nil(t, p)
+		assert.ErrorIs(t, err, ErrInvalidArchitecture)
+	})
+
+	t.Run("Valid", func(t *testing.T) {
+		content := buildContent(packageName, packageVersion, packageArchitecture)
+		full := content.String()
+
+		p, err := ParseControlFile(content)
+		assert.NoError(t, err)
+		assert.NotNil(t, p)
+
+		assert.Equal(t, packageName, p.Name)
+		assert.Equal(t, packageVersion, p.Version)
+		assert.Equal(t, packageArchitecture, p.Architecture)
+		assert.Equal(t, description, p.Metadata.Description)
+		assert.Equal(t, projectURL, p.Metadata.ProjectURL)
+		assert.Equal(t, packageAuthor, p.Metadata.Maintainer)
+		assert.Equal(t, []string{"a", "b"}, p.Metadata.Dependencies)
+		assert.Equal(t, full, p.Control)
+	})
+}

modules/packages/hashed_buffer.go

@@ -25,8 +25,15 @@ type HashedBuffer struct {
 	combinedWriter io.Writer
 }
 
-// NewHashedBuffer creates a hashed buffer with a specific maximum memory size
-func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
+const DefaultMemorySize = 32 * 1024 * 1024
+
+// NewHashedBuffer creates a hashed buffer with the default memory size
+func NewHashedBuffer() (*HashedBuffer, error) {
+	return NewHashedBufferWithSize(DefaultMemorySize)
+}
+
+// NewHashedBuffer creates a hashed buffer with a specific memory size
+func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) {
 	b, err := filebuffer.New(maxMemorySize)
 	if err != nil {
 		return nil, err
@@ -43,9 +50,14 @@ func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
 	}, nil
 }
 
-// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it.
-func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
-	b, err := NewHashedBuffer(maxMemorySize)
+// CreateHashedBufferFromReader creates a hashed buffer with the default memory size and copies the provided reader data into it.
+func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) {
+	return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize)
+}
+
+// CreateHashedBufferFromReaderWithSize creates a hashed buffer and copies the provided reader data into it.
+func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
+	b, err := NewHashedBufferWithSize(maxMemorySize)
 	if err != nil {
 		return nil, err
 	}

modules/packages/hashed_buffer_test.go

@@ -26,7 +26,7 @@ func TestHashedBuffer(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize)
+		buf, err := CreateHashedBufferFromReaderWithSize(strings.NewReader(c.Data), c.MaxMemorySize)
 		assert.NoError(t, err)
 
 		assert.EqualValues(t, len(c.Data), buf.Size())

modules/packages/nuget/symbol_extractor.go

@@ -63,7 +63,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
 					return err
 				}
 
-				buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024)
+				buf, err := packages.CreateHashedBufferFromReader(f)
 
 				f.Close()
 

modules/setting/packages.go

@@ -30,6 +30,7 @@ var (
 		LimitSizeConan       int64
 		LimitSizeConda       int64
 		LimitSizeContainer   int64
+		LimitSizeDebian      int64
 		LimitSizeGeneric     int64
 		LimitSizeHelm        int64
 		LimitSizeMaven       int64
@@ -73,6 +74,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
 	Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
 	Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA")
 	Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
+	Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN")
 	Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
 	Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
 	Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")

modules/util/filebuffer/file_backed_buffer.go

@@ -7,11 +7,10 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"math"
 	"os"
 )
 
-const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer
-
 var (
 	// ErrInvalidMemorySize occurs if the memory size is not in a valid range
 	ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32")
@@ -37,7 +36,7 @@ type FileBackedBuffer struct {
 
 // New creates a file backed buffer with a specific maximum memory size
 func New(maxMemorySize int) (*FileBackedBuffer, error) {
-	if maxMemorySize < 0 || maxMemorySize > maxInt {
+	if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 {
 		return nil, ErrInvalidMemorySize
 	}