Add minimal implementation for RubyGems compact index API. (#3811)

Current package registry for RubyGems does not work with Bundler, because it implements neither the [compact index](https://guides.rubygems.org/rubygems-org-compact-index-api/) or the [dependency API](https://guides.rubygems.org/rubygems-org-api/). As a result, bundler complains about finding non-existing dependencies when installing anything with dependency: `revealed dependencies not in the API or the lockfile`.

This patch provides a minimal implementation for the compact index API to solve this issue. Specifically, we implemented a version that does not cache the results / do incremental updates; which is consistent with the current implementation.

Testing:
  * Modified existing integration tests.
  * Manually Verified bundler is able to parse the served versions / info file.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3811
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Haoyuan (Bill) Xing <me@hoppinglife.com>
Co-committed-by: Haoyuan (Bill) Xing <me@hoppinglife.com>

routers/api/packages/api.go

@@ -586,6 +586,8 @@ func CommonRoutes() *web.Route {
 			r.Get("/specs.4.8.gz", rubygems.EnumeratePackages)
 			r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest)
 			r.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease)
+			r.Get("/info/{package}", rubygems.ServePackageInfo)
+			r.Get("/versions", rubygems.ServeVersionsFile)
 			r.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification)
 			r.Get("/gems/{filename}", rubygems.DownloadPackageFile)
 			r.Group("/api/v1/gems", func() {

routers/api/packages/rubygems/rubygems.go

@@ -6,6 +6,7 @@ package rubygems
 import (
 	"compress/gzip"
 	"compress/zlib"
+	"crypto/md5"
 	"errors"
 	"fmt"
 	"io"
@@ -22,6 +23,10 @@ import (
 	packages_service "code.gitea.io/gitea/services/packages"
 )
 
+const (
+	Sep = "---\n"
+)
+
 func apiError(ctx *context.Context, status int, obj any) {
 	helper.LogAndProcessError(ctx, status, obj, func(message string) {
 		ctx.PlainText(status, message)
@@ -92,6 +97,69 @@ func enumeratePackages(ctx *context.Context, filename string, pvs []*packages_mo
 	}
 }
 
+// Serves info file for rubygems.org compatible /info/{gem} file.
+// See also https://guides.rubygems.org/rubygems-org-compact-index-api/.
+func ServePackageInfo(ctx *context.Context) {
+	packageName := ctx.Params("package")
+	versions, err := packages_model.GetVersionsByPackageName(
+		ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, packageName)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+	}
+	if len(versions) == 0 {
+		apiError(ctx, http.StatusNotFound, fmt.Sprintf("Could not find package %s", packageName))
+	}
+
+	result, err := buildInfoFileForPackage(ctx, versions)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.PlainText(http.StatusOK, *result)
+}
+
+// ServeVersionsFile creates rubygems.org compatible /versions file.
+// See also https://guides.rubygems.org/rubygems-org-compact-index-api/.
+func ServeVersionsFile(ctx *context.Context) {
+	packages, err := packages_model.GetPackagesByType(
+		ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	result := new(strings.Builder)
+	result.WriteString(Sep)
+	for _, pack := range packages {
+		versions, err := packages_model.GetVersionsByPackageName(
+			ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, pack.Name)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		if len(versions) == 0 {
+			// No versions left for this package, we should continue.
+			continue
+		}
+
+		fmt.Fprintf(result, "%s ", pack.Name)
+		for i, v := range versions {
+			result.WriteString(v.Version)
+			if i != len(versions)-1 {
+				result.WriteString(",")
+			}
+		}
+
+		info, err := buildInfoFileForPackage(ctx, versions)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+
+		checksum := md5.Sum([]byte(*info))
+		fmt.Fprintf(result, " %x\n", checksum)
+	}
+	ctx.PlainText(http.StatusOK, result.String())
+}
+
 // ServePackageSpecification serves the compressed Gemspec file of a package
 func ServePackageSpecification(ctx *context.Context) {
 	filename := ctx.Params("filename")
@@ -227,12 +295,7 @@ func UploadPackageFile(ctx *context.Context) {
 		return
 	}
 
-	var filename string
-	if rp.Metadata.Platform == "" || rp.Metadata.Platform == "ruby" {
-		filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", rp.Name, rp.Version))
-	} else {
-		filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", rp.Name, rp.Version, rp.Metadata.Platform))
-	}
+	filename := getFullFilename(rp.Name, rp.Version, rp.Metadata.Platform)
 
 	_, _, err = packages_service.CreatePackageAndAddFile(
 		ctx,
@@ -300,6 +363,83 @@ func DeletePackage(ctx *context.Context) {
 	}
 }
 
+func writeRequirements(reqs []rubygems_module.VersionRequirement, result *strings.Builder) {
+	if len(reqs) == 0 {
+		reqs = []rubygems_module.VersionRequirement{{Restriction: ">=", Version: "0"}}
+	}
+	for i, req := range reqs {
+		if i != 0 {
+			result.WriteString("&")
+		}
+		result.WriteString(req.Restriction)
+		result.WriteString(" ")
+		result.WriteString(req.Version)
+	}
+}
+
+func buildRequirementStringFromVersion(ctx *context.Context, version *packages_model.PackageVersion) (string, error) {
+	pd, err := packages_model.GetPackageDescriptor(ctx, version)
+	if err != nil {
+		return "", err
+	}
+	metadata := pd.Metadata.(*rubygems_module.Metadata)
+	dependencyRequirements := new(strings.Builder)
+	for i, dep := range metadata.RuntimeDependencies {
+		if i != 0 {
+			dependencyRequirements.WriteString(",")
+		}
+
+		dependencyRequirements.WriteString(dep.Name)
+		dependencyRequirements.WriteString(":")
+		reqs := dep.Version
+		writeRequirements(reqs, dependencyRequirements)
+	}
+	fullname := getFullFilename(pd.Package.Name, version.Version, metadata.Platform)
+	file, err := packages_model.GetFileForVersionByName(ctx, version.ID, fullname, "")
+	if err != nil {
+		return "", err
+	}
+	blob, err := packages_model.GetBlobByID(ctx, file.BlobID)
+	if err != nil {
+		return "", err
+	}
+	additionalRequirements := new(strings.Builder)
+	fmt.Fprintf(additionalRequirements, "checksum:%s", blob.HashSHA256)
+	if len(metadata.RequiredRubyVersion) != 0 {
+		additionalRequirements.WriteString(",ruby:")
+		writeRequirements(metadata.RequiredRubyVersion, additionalRequirements)
+	}
+	if len(metadata.RequiredRubygemsVersion) != 0 {
+		additionalRequirements.WriteString(",rubygems:")
+		writeRequirements(metadata.RequiredRubygemsVersion, additionalRequirements)
+	}
+	return fmt.Sprintf("%s %s|%s", version.Version, dependencyRequirements, additionalRequirements), nil
+}
+
+func buildInfoFileForPackage(ctx *context.Context, versions []*packages_model.PackageVersion) (*string, error) {
+	result := "---\n"
+	for _, v := range versions {
+		str, err := buildRequirementStringFromVersion(ctx, v)
+		if err != nil {
+			return nil, err
+		}
+		result += str
+		result += "\n"
+	}
+	return &result, nil
+}
+
+func getFullFilename(gemName, version, platform string) string {
+	return strings.ToLower(getFullName(gemName, version, platform)) + ".gem"
+}
+
+func getFullName(gemName, version, platform string) string {
+	if platform == "" || platform == "ruby" {
+		return fmt.Sprintf("%s-%s", gemName, version)
+	}
+	return fmt.Sprintf("%s-%s-%s", gemName, version, platform)
+}
+
 func getVersionsByFilename(ctx *context.Context, filename string) ([]*packages_model.PackageVersion, error) {
 	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
 		OwnerID:         ctx.Package.Owner.ID,