From 600c703ef4e4adaee000fb78df2283721a408a96 Mon Sep 17 00:00:00 2001 From: dragon Date: Mon, 26 Aug 2024 10:45:24 +0800 Subject: [PATCH 1/4] Fix arch OptDepends version check and add gzip compression --- modules/packages/arch/metadata.go | 8 ++++++-- modules/packages/arch/metadata_test.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 8fba53e58..069d7e8c7 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -39,11 +39,12 @@ const ( var ( reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) - reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) - rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?(:.*)?$`) magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} + magicGZ = []byte{0x1F, 0x8B} ) type Package struct { @@ -111,6 +112,9 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { tarballType = "xz" tarball = archiver.NewTarXz() + } else if bytes.Equal(header[:len(magicGZ)], magicGZ) { + tarballType = "gz" + tarball = archiver.NewTarGz() } else { return nil, errors.New("not supported compression") } diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index bd4c2a9ad..c328c7674 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -199,7 +199,7 @@ func TestValidatePackageSpec(t *testing.T) { Provides: []string{"abc", "def"}, License: []string{"GPL"}, Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, - OptDepends: []string{"git: something", "make"}, + OptDepends: []string{"git: something", "make", "llvm<=17: not support llvm 18+"}, MakeDepends: []string{"chrom"}, CheckDepends: []string{"bariy"}, Backup: []string{"etc/pacman.d/filo"}, From a3ef037ccb44d3a588dbd1667287255e6fcc7abb Mon Sep 17 00:00:00 2001 From: dragon Date: Mon, 26 Aug 2024 11:13:22 +0800 Subject: [PATCH 2/4] clean style and add more tests --- modules/packages/arch/metadata.go | 2 +- modules/packages/arch/metadata_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 069d7e8c7..0e0867031 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -39,8 +39,8 @@ const ( var ( reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) - rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`) reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?(:.*)?$`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`) magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index c328c7674..ddb35ca83 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -199,7 +199,7 @@ func TestValidatePackageSpec(t *testing.T) { Provides: []string{"abc", "def"}, License: []string{"GPL"}, Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, - OptDepends: []string{"git: something", "make", "llvm<=17: not support llvm 18+"}, + OptDepends: []string{"git", "libgcc=1.0", "gzip>1.0", "gz>=1.0", "lz<1.0", "gzip<=1.0", "zstd>1.0:foo bar"}, MakeDepends: []string{"chrom"}, CheckDepends: []string{"bariy"}, Backup: []string{"etc/pacman.d/filo"}, From 97ec997a04fc46227246401a369bf5550fb58c5d Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 27 Aug 2024 09:58:08 +0800 Subject: [PATCH 3/4] add zst,xz,gz tests --- tests/integration/api_packages_arch_test.go | 47 ++++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 74a6bb8be..05febf912 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -80,7 +80,7 @@ I2dkIEMjxsSOiAlJjH4HIwbTjayZJidXVxKQYH2gICOCBhK7KqMlLZ4gMCU1BapYlsTAXnywepyy jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`), - "other": unPack(` + "otherXZ": unPack(` /Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl @@ -93,6 +93,32 @@ MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ Wg==`), // this is tar.xz file + "otherZST": unPack(` +KLUv/QRYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj +ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi +sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz +q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf +A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz +5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd +GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW +gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX +Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv +qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE +wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd +nYARrByncQ==`), // this is zstd file + "otherGZ": unPack(` +H4sIAAAAAAAAA9PzDQlydWWgKTAwMDAzMVEA0UCAThsYGBuZKRiamBmbm5qZGJqbKBgYGpobGzMo +GNDWWRBQWlySWAR0SlF+fgk+dYTk0T03RIB8NweEwVx71tDviIFA60O75Rtc5s+9YbxteUHzhUWi +HBkWDcbGcUqCukrLGi4Lv8jIqNsbXhueXW8uzTe79Lr9/TVbnl69c3wR652f21+7rnU5kmjTc/38 +8t+zLx/+ePFr6lajpZ2dzCkyB3NPTxdVOfFk2/RXmq+Ktq2dbnY6RcPCMW8Kg9aGszs1f6+YsTlf +x5j5eIpXnXzStAbJvQvPP3su//3lu2/2pj++XO9hbJS+puPmqJKREff4X+RUqdYTbpGTBGYuefH9 +mNbGzKNdiUmS+xgt7J+5iTMObIgOLaAX4O3u6efmT0s7COV/UwNztPxvZGhqOpr/6QGUFdxT81KL +EktSUxSSKhVyE7NTC7LTFcz0DPUMuJQVSosz89IV0oCiIP8rlKUWFWfm5ykY6hmbcgHV5SXmpirY +KpSkFpcYgfhJicUIfkVKYkkikAcUL6ksSLUF0iA1QDOAgkDj9Qx0DUECKanFyVBNCgWJydmJ6alc +pUU5QKGMkpKCYit9/dSKxNyCnFS95Pxcfa6k0sycFKDRIIsMzQ0tTS2NDSxMuKA6QWaH5mXn5Zfn +KQRAhbiKM6tAqg24EouSM4CMxLxKrpzM5NQ8sGuTgUkgP5crOT8vDShYAhSpKs7gKijKL8sEOg2k +HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA +`), // this is tar.gz file } t.Run("RepositoryKey", func(t *testing.T) { @@ -228,7 +254,7 @@ Wg==`), // this is tar.xz file t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) { defer tests.PrintCurrentTest(t)() // test data - req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])). + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["otherXZ"])). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusCreated) @@ -252,6 +278,23 @@ Wg==`), // this is tar.xz file req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") MakeRequest(t, req, http.StatusNotFound) }) + + for tp, key := range map[string]string{ + "XZ": "otherXZ", + "GZ": "otherGZ", + "ZST": "otherZST", + } { + t.Run(fmt.Sprintf("Upload%s[%s]", tp, group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs[key])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + }) + } + } } From 60b54c42ec6eb1eab54e5850d584fb0ef610e3c2 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 27 Aug 2024 10:02:43 +0800 Subject: [PATCH 4/4] add zst,xz,gz tests --- tests/integration/api_packages_arch_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 05febf912..af275de44 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -92,7 +92,7 @@ MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv 0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ -Wg==`), // this is tar.xz file +Wg==`), "otherZST": unPack(` KLUv/QRYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi @@ -105,7 +105,7 @@ gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd -nYARrByncQ==`), // this is zstd file +nYARrByncQ==`), "otherGZ": unPack(` H4sIAAAAAAAAA9PzDQlydWWgKTAwMDAzMVEA0UCAThsYGBuZKRiamBmbm5qZGJqbKBgYGpobGzMo GNDWWRBQWlySWAR0SlF+fgk+dYTk0T03RIB8NweEwVx71tDviIFA60O75Rtc5s+9YbxteUHzhUWi @@ -118,7 +118,7 @@ KpSkFpcYgfhJicUIfkVKYkkikAcUL6ksSLUF0iA1QDOAgkDj9Qx0DUECKanFyVBNCgWJydmJ6alc pUU5QKGMkpKCYit9/dSKxNyCnFS95Pxcfa6k0sycFKDRIIsMzQ0tTS2NDSxMuKA6QWaH5mXn5Zfn KQRAhbiKM6tAqg24EouSM4CMxLxKrpzM5NQ8sGuTgUkgP5crOT8vDShYAhSpKs7gKijKL8sEOg2k HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA -`), // this is tar.gz file +`), } t.Run("RepositoryKey", func(t *testing.T) { @@ -280,8 +280,8 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA }) for tp, key := range map[string]string{ - "XZ": "otherXZ", "GZ": "otherGZ", + "XZ": "otherXZ", "ZST": "otherZST", } { t.Run(fmt.Sprintf("Upload%s[%s]", tp, group), func(t *testing.T) { @@ -289,12 +289,16 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs[key])). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "GET", groupURL+"/x86_64/test2-1.0.0-1-any.pkg.tar."+strings.ToLower(tp)) + resp := MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs[key], resp.Body.Bytes()) + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNoContent) }) } - } }