Add minimal implementation for RubyGems compact index API. (#3811)

Current package registry for RubyGems does not work with Bundler, because it implements neither the [compact index](https://guides.rubygems.org/rubygems-org-compact-index-api/) or the [dependency API](https://guides.rubygems.org/rubygems-org-api/). As a result, bundler complains about finding non-existing dependencies when installing anything with dependency: `revealed dependencies not in the API or the lockfile`.

This patch provides a minimal implementation for the compact index API to solve this issue. Specifically, we implemented a version that does not cache the results / do incremental updates; which is consistent with the current implementation.

Testing:
  * Modified existing integration tests.
  * Manually Verified bundler is able to parse the served versions / info file.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3811
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Haoyuan (Bill) Xing <me@hoppinglife.com>
Co-committed-by: Haoyuan (Bill) Xing <me@hoppinglife.com>
This commit is contained in:
Haoyuan (Bill) Xing 2024-05-19 23:30:41 +00:00 committed by Earl Warren
parent 3351ce2bc5
commit 6cb8c81de1
4 changed files with 323 additions and 10 deletions

View file

@ -5,10 +5,13 @@ package integration
import (
"bytes"
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"fmt"
"mime/multipart"
"net/http"
"strings"
"testing"
"code.gitea.io/gitea/models/db"
@ -29,6 +32,9 @@ func TestPackageRubyGems(t *testing.T) {
packageName := "gitea"
packageVersion := "1.0.5"
packageFilename := "gitea-1.0.5.gem"
packageDependency := "runtime-dep:>= 1.2.0&< 2.0"
rubyRequirements := "ruby:>= 2.3.0"
sep := "---"
gemContent, _ := base64.StdEncoding.DecodeString(`bWV0YWRhdGEuZ3oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDA0NDQAMDAwMDAw
@ -111,11 +117,93 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`)
checksum := fmt.Sprintf("%x", sha256.Sum256(gemContent))
holaPackageName := "hola"
holaPackageVersion := "0.0.1"
// holaPackageFilename := "hola-0.0.1.gem"
holaPackageDependency := "example:~> 1.1&>= 1.1.4,zero:>= 0"
holaRubyGemsRequirements := "rubygems:= 1.2.3"
// sep := "---"
holaGemContent, _ := base64.StdEncoding.DecodeString(`bWV0YWRhdGEuZ3oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDA0NDQAMDAwMDAw
MAAwMDAwMDAwADAwMDAwMDAwNzYyADE0NjIyMjU1MzY0ADAxMzQ1NAAgMAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhcgAwMHdoZWVsAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAd2hlZWwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwMDAwADAwMDAw
MDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAf
iwgA9FpJZgID1VVNb9QwEL37V7i95JQ02W0PWGpPSHBCAiQOIBRNnNmNqT+C7aBdEPx2xtnNbtNW
BVokRBLJ8Xhm/Oa9eJLnOT/xQ7M9c80nlFG8QCPE2x6lWikJUTnLLBgUvHMa2Bf0gUzinph3uyXG
+cGpLMqiYr2GuHLeCJ5iGAyxcz4IlvNXSl7z1wN4sNGlBefx86A8CtYo2yovOI1Moo+17EBRyg8f
WQuR4CzKxXleXuTVM16WYnyKcrr4e9Zij7ZFKxWOe90F/Hzy2BLmXY24AdNrpPkeiEEb7yv2zXGZ
nGfutFuy5HSf/rg6HSdp+hBju+vAW1YVVXbMcnX5qCyUpDgnc9z2VJrwg43KpNp6jx41QiDzCnTA
o2b1rJD/uvDflPwrevfX9H4k4KzM/pFOTwDcYpBe9alDCIYGlKZhg3KI0Gg6c+mo4iaiTSGHqYfa
t07WKzX57N5ILq2as9RkCt+wzhnsYU2NQCtJKfa+BiPQ8QfBv31nvQuxVjZE0Lo2GMLoP2Z3I6xd
zL7yuofYTftMxrZONdcPdLU5j7dZnHH4awZn/M0grNGEp8HILrM/RVEVi2LJ5l9SIuwOnmWxLBYX
LKi1VXZdX+NWsHDzF3HDlYXBGPBbwV+SlicsIql0VPsn64DO03AGAAAAAAAAAAAAAAAAAAAAAGRh
dGEudGFyLmd6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwNDQ0ADAwMDAwMDAA
MDAwMDAwMAAwMDAwMDAwMDE1MQAxNDYyMjI1NTM2NAAwMTMzNjIAIDAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdXN0YXIAMDB3aGVlbAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAHdoZWVsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDAwMDAwMAAwMDAwMDAw
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH4sI
APRaSWYCA8rJTNLPyM9J1CtKYqAVMDA0MDAzMWEwgAB0Gsw2NDEzMjIyNTU2A6ozNDY2NmFQMGCg
AygtLkksAjqlPCM1NQePOkLy6J4bBaNgFIyCQQ4AAAAA//8DAMJiTFMABgAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjaGVj
a3N1bXMueWFtbC5negAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDAwMDQ0NAAwMDAwMDAwADAw
MDAwMDAAMDAwMDAwMDA0NTMAMTQ2MjIyNTUzNjQAMDE0NjE3ACAwAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHVzdGFyADAwd2hlZWwAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAB3aGVlbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDAwMDAAMDAwMDAwMAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB+LCAD0
WklmAgNlkD2uFDAMBvs9xV5gke04/nkdHT0nsGObigZtxekJr4QijaWMZr7X6/X4/u0rbfl4PJ8/
+x0V7/jy4/fH0xIRx4PkkBT7Qt8cG0DSNq2iBIkMwD0ETCCgQ6Xh5WZWeHmfrHf8+uSRBivatKy8
n6UT249x1CbVnAEHsbSDNEJAfRDuOytsa27767mR/vPkPtXpakdXyRBdsXti1lkjiye7uCeyHath
7VzMRxAOxNo3L31AiJu7ryPe7BsZR91Tcp21chYaSyvDwZaQE+SxlOn09fqnM8nUVcUb5CSEbljA
LH4HrZhC5YJMNyRevKZtKiqTZroEkKvuoHmS0iYWQFXYnTqOz0Wpxqw6RqnHhf0uah45WkjqtfPx
B6h0MiLUAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==`)
holaChecksum := fmt.Sprintf("%x", sha256.Sum256(holaGemContent))
root := fmt.Sprintf("/api/packages/%s/rubygems", user.Name)
uploadFile := func(t *testing.T, expectedStatus int) {
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/api/v1/gems", root), bytes.NewReader(gemContent)).
uploadFile := func(t *testing.T, content []byte, expectedStatus int) {
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/api/v1/gems", root), bytes.NewReader(content)).
AddBasicAuth(user.Name)
MakeRequest(t, req, expectedStatus)
}
@ -123,7 +211,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`)
t.Run("Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
uploadFile(t, http.StatusCreated)
uploadFile(t, gemContent, http.StatusCreated)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeRubyGems)
assert.NoError(t, err)
@ -150,7 +238,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`)
t.Run("UploadExists", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
uploadFile(t, http.StatusConflict)
uploadFile(t, gemContent, http.StatusConflict)
})
t.Run("Download", func(t *testing.T) {
@ -206,6 +294,72 @@ gAAAAP//MS06Gw==`)
enumeratePackages(t, "prerelease_specs.4.8.gz", b)
})
t.Run("UploadHola", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
uploadFile(t, holaGemContent, http.StatusCreated)
})
t.Run("PackageInfo", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/info/%s", root, packageName)).
AddBasicAuth(user.Name)
resp := MakeRequest(t, req, http.StatusOK)
expected := fmt.Sprintf("%s\n%s %s|checksum:%s,%s\n",
sep, packageVersion, packageDependency, checksum, rubyRequirements)
assert.Equal(t, expected, resp.Body.String())
})
t.Run("HolaPackageInfo", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/info/%s", root, holaPackageName)).
AddBasicAuth(user.Name)
resp := MakeRequest(t, req, http.StatusOK)
expected := fmt.Sprintf("%s\n%s %s|checksum:%s,%s\n",
sep, holaPackageVersion, holaPackageDependency, holaChecksum, holaRubyGemsRequirements)
assert.Equal(t, expected, resp.Body.String())
})
t.Run("Versions", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
versionsReq := NewRequest(t, "GET", fmt.Sprintf("%s/versions", root)).
AddBasicAuth(user.Name)
versionsResp := MakeRequest(t, versionsReq, http.StatusOK)
infoReq := NewRequest(t, "GET", fmt.Sprintf("%s/info/%s", root, packageName)).
AddBasicAuth(user.Name)
infoResp := MakeRequest(t, infoReq, http.StatusOK)
holaInfoReq := NewRequest(t, "GET", fmt.Sprintf("%s/info/%s", root, holaPackageName)).
AddBasicAuth(user.Name)
holaInfoResp := MakeRequest(t, holaInfoReq, http.StatusOK)
// expected := fmt.Sprintf("%s\n%s %s %x\n",
// sep, packageName, packageVersion, md5.Sum(infoResp.Body.Bytes()))
lines := versionsResp.Body.String()
assert.ElementsMatch(t, strings.Split(lines, "\n"), []string{
sep,
fmt.Sprintf("%s %s %x", packageName, packageVersion, md5.Sum(infoResp.Body.Bytes())),
fmt.Sprintf("%s %s %x", holaPackageName, holaPackageVersion, md5.Sum(holaInfoResp.Body.Bytes())),
"",
})
})
t.Run("DeleteHola", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
body := bytes.Buffer{}
writer := multipart.NewWriter(&body)
writer.WriteField("gem_name", holaPackageName)
writer.WriteField("version", holaPackageVersion)
writer.Close()
req := NewRequestWithBody(t, "DELETE", fmt.Sprintf("%s/api/v1/gems/yank", root), &body).
SetHeader("Content-Type", writer.FormDataContentType()).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusOK)
})
t.Run("Delete", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
@ -224,4 +378,20 @@ gAAAAP//MS06Gw==`)
assert.NoError(t, err)
assert.Empty(t, pvs)
})
t.Run("NonExistingGem", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/info/%s", root, packageName)).
AddBasicAuth(user.Name)
_ = MakeRequest(t, req, http.StatusNotFound)
})
t.Run("EmptyVersions", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/versions", root)).
AddBasicAuth(user.Name)
resp := MakeRequest(t, req, http.StatusOK)
assert.Equal(t, sep+"\n", resp.Body.String())
})
}