finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request '[gitea] week 2024-23 cherry pick (gitea/main -> forgejo)' (#3989) from earl-warren/wcp/2024-23 into forgejo

Browse source 
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3989
Reviewed-by: Otto <otto@codeberg.org>
This commit is contained in:
Earl Warren 2024-06-04 07:40:35 +00:00
commit c2382d4f5b
98 changed files with 1520 additions and 975 deletions
Download patch file Download diff file Expand all files Collapse all files

4
modules/indexer/issues/bleve/bleve.go
View file

@ -230,8 +230,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
if options.ProjectID.Has() {
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectID.Value(), "project_id"))
}
if options.ProjectBoardID.Has() {
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectBoardID.Value(), "project_board_id"))
if options.ProjectColumnID.Has() {
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id"))
}
if options.PosterID.Has() {

2
modules/indexer/issues/db/options.go
View file

@ -61,7 +61,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m
ReviewedID: convertID(options.ReviewedID),
SubscriberID: convertID(options.SubscriberID),
ProjectID: convertID(options.ProjectID),
ProjectBoardID: convertID(options.ProjectBoardID),
ProjectColumnID: convertID(options.ProjectColumnID),
IsClosed: options.IsClosed,
IsPull: options.IsPull,
IncludedLabelNames: nil,

2
modules/indexer/issues/dboptions.go
View file

@ -50,7 +50,7 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
}
searchOpt.ProjectID = convertID(opts.ProjectID)
searchOpt.ProjectBoardID = convertID(opts.ProjectBoardID)
searchOpt.ProjectColumnID = convertID(opts.ProjectColumnID)
searchOpt.PosterID = convertID(opts.PosterID)
searchOpt.AssigneeID = convertID(opts.AssigneeID)
searchOpt.MentionID = convertID(opts.MentionedID)

4
modules/indexer/issues/elasticsearch/elasticsearch.go
View file

@ -197,8 +197,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
if options.ProjectID.Has() {
query.Must(elastic.NewTermQuery("project_id", options.ProjectID.Value()))
}
if options.ProjectBoardID.Has() {
query.Must(elastic.NewTermQuery("project_board_id", options.ProjectBoardID.Value()))
if options.ProjectColumnID.Has() {
query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value()))
}
if options.PosterID.Has() {

4
modules/indexer/issues/indexer_test.go
View file

@ -369,13 +369,13 @@ func searchIssueInProject(t *testing.T) {
},
{
SearchOptions{
ProjectBoardID: optional.Some(int64(1)),
ProjectColumnID: optional.Some(int64(1)),
},
[]int64{1},
},
{
SearchOptions{
ProjectBoardID: optional.Some(int64(0)), // issue with in default board
ProjectColumnID: optional.Some(int64(0)), // issue with in default column
},
[]int64{2},
},

6
modules/indexer/issues/internal/model.go
View file

@ -27,7 +27,7 @@ type IndexerData struct {
NoLabel bool `json:"no_label"` // True if LabelIDs is empty
MilestoneID int64 `json:"milestone_id"`
ProjectID int64 `json:"project_id"`
ProjectBoardID int64 `json:"project_board_id"`
ProjectColumnID int64 `json:"project_board_id"` // the key should be kept as project_board_id to keep compatible
PosterID int64 `json:"poster_id"`
AssigneeID int64 `json:"assignee_id"`
MentionIDs []int64 `json:"mention_ids"`
@ -89,8 +89,8 @@ type SearchOptions struct {
MilestoneIDs []int64 // milestones the issues have
ProjectID optional.Option[int64] // project the issues belong to
ProjectBoardID optional.Option[int64] // project board the issues belong to
ProjectID optional.Option[int64] // project the issues belong to
ProjectColumnID optional.Option[int64] // project column the issues belong to
PosterID optional.Option[int64] // poster of the issues

18
modules/indexer/issues/internal/tests/tests.go
View file

@ -352,38 +352,38 @@ var cases = []*testIndexerCase{
},
},
{
Name: "ProjectBoardID",
Name: "ProjectColumnID",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
},
ProjectBoardID: optional.Some(int64(1)),
ProjectColumnID: optional.Some(int64(1)),
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Equal(t, 5, len(result.Hits))
for _, v := range result.Hits {
assert.Equal(t, int64(1), data[v.ID].ProjectBoardID)
assert.Equal(t, int64(1), data[v.ID].ProjectColumnID)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectBoardID == 1
return v.ProjectColumnID == 1
}), result.Total)
},
},
{
Name: "no ProjectBoardID",
Name: "no ProjectColumnID",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
},
ProjectBoardID: optional.Some(int64(0)),
ProjectColumnID: optional.Some(int64(0)),
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Equal(t, 5, len(result.Hits))
for _, v := range result.Hits {
assert.Equal(t, int64(0), data[v.ID].ProjectBoardID)
assert.Equal(t, int64(0), data[v.ID].ProjectColumnID)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectBoardID == 0
return v.ProjectColumnID == 0
}), result.Total)
},
},
@ -720,7 +720,7 @@ func generateDefaultIndexerData() []*internal.IndexerData {
NoLabel: len(labelIDs) == 0,
MilestoneID: issueIndex % 4,
ProjectID: issueIndex % 5,
ProjectBoardID: issueIndex % 6,
ProjectColumnID: issueIndex % 6,
PosterID: id%10 + 1, // PosterID should not be 0
AssigneeID: issueIndex % 10,
MentionIDs: mentionIDs,

4
modules/indexer/issues/meilisearch/meilisearch.go
View file

@ -174,8 +174,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
if options.ProjectID.Has() {
query.And(inner_meilisearch.NewFilterEq("project_id", options.ProjectID.Value()))
}
if options.ProjectBoardID.Has() {
query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectBoardID.Value()))
if options.ProjectColumnID.Has() {
query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value()))
}
if options.PosterID.Has() {

2
modules/indexer/issues/util.go
View file

@ -105,7 +105,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD
NoLabel: len(labels) == 0,
MilestoneID: issue.MilestoneID,
ProjectID: projectID,
ProjectBoardID: issue.ProjectBoardID(ctx),
ProjectColumnID: issue.ProjectColumnID(ctx),
PosterID: issue.PosterID,
AssigneeID: issue.AssigneeID,
MentionIDs: mentionIDs,

14
modules/metrics/collector.go
View file

@ -36,7 +36,7 @@ type Collector struct {
Oauths *prometheus.Desc
Organizations *prometheus.Desc
Projects *prometheus.Desc
ProjectBoards *prometheus.Desc
ProjectColumns *prometheus.Desc
PublicKeys *prometheus.Desc
Releases *prometheus.Desc
Repositories *prometheus.Desc
@ -146,9 +146,9 @@ func NewCollector() Collector {
"Number of projects",
nil, nil,
),
ProjectBoards: prometheus.NewDesc(
namespace+"projects_boards",
"Number of project boards",
ProjectColumns: prometheus.NewDesc(
namespace+"projects_boards", // TODO: change the key name will affect the consume's result history
"Number of project columns",
nil, nil,
),
PublicKeys: prometheus.NewDesc(
@ -219,7 +219,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.Oauths
ch <- c.Organizations
ch <- c.Projects
ch <- c.ProjectBoards
ch <- c.ProjectColumns
ch <- c.PublicKeys
ch <- c.Releases
ch <- c.Repositories
@ -336,9 +336,9 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) {
float64(stats.Counter.Project),
)
ch <- prometheus.MustNewConstMetric(
c.ProjectBoards,
c.ProjectColumns,
prometheus.GaugeValue,
float64(stats.Counter.ProjectBoard),
float64(stats.Counter.ProjectColumn),
)
ch <- prometheus.MustNewConstMetric(
c.PublicKeys,

31
modules/storage/minio.go
View file

@ -97,7 +97,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
minioClient, err := minio.New(config.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
Secure: config.UseSSL,
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
Region: config.Location,
@ -164,6 +164,35 @@ func (m *MinioStorage) buildMinioDirPrefix(p string) string {
return p
}
func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
// If static credentials are provided, use those
if config.AccessKeyID != "" {
return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
}
// Otherwise, fallback to a credentials chain for S3 access
chain := []credentials.Provider{
// configure based upon MINIO_ prefixed environment variables
&credentials.EnvMinio{},
// configure based upon AWS_ prefixed environment variables
&credentials.EnvAWS{},
// read credentials from MINIO_SHARED_CREDENTIALS_FILE
// environment variable, or default json config files
&credentials.FileMinioClient{},
// read credentials from AWS_SHARED_CREDENTIALS_FILE
// environment variable, or default credentials file
&credentials.FileAWSCredentials{},
// read IAM role from EC2 metadata endpoint if available
&credentials.IAM{
Endpoint: iamEndpoint,
Client: &http.Client{
Transport: http.DefaultTransport,
},
},
}
return credentials.NewChainCredentials(chain)
}
// Open opens a file
func (m *MinioStorage) Open(path string) (Object, error) {
opts := minio.GetObjectOptions{}

104
modules/storage/minio_test.go
View file

@ -6,6 +6,7 @@ package storage
import (
"context"
"net/http"
"net/http/httptest"
"os"
"testing"
@ -109,3 +110,106 @@ func TestS3StorageBadRequest(t *testing.T) {
_, err := NewStorage(setting.MinioStorageType, cfg)
assert.ErrorContains(t, err, message)
}
func TestMinioCredentials(t *testing.T) {
const (
ExpectedAccessKey = "ExampleAccessKeyID"
ExpectedSecretAccessKey = "ExampleSecretAccessKeyID"
// Use a FakeEndpoint for IAM credentials to avoid logging any
// potential real IAM credentials when running in EC2.
FakeEndpoint = "http://localhost"
)
t.Run("Static Credentials", func(t *testing.T) {
cfg := setting.MinioStorageConfig{
AccessKeyID: ExpectedAccessKey,
SecretAccessKey: ExpectedSecretAccessKey,
}
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey, v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey, v.SecretAccessKey)
})
t.Run("Chain", func(t *testing.T) {
cfg := setting.MinioStorageConfig{}
t.Run("EnvMinio", func(t *testing.T) {
t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio")
t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"Minio", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"Minio", v.SecretAccessKey)
})
t.Run("EnvAWS", func(t *testing.T) {
t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS")
t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"AWS", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"AWS", v.SecretAccessKey)
})
t.Run("FileMinio", func(t *testing.T) {
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
// prevent loading any actual credentials files from the user
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"MinioFile", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"MinioFile", v.SecretAccessKey)
})
t.Run("FileAWS", func(t *testing.T) {
// prevent loading any actual credentials files from the user
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"AWSFile", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"AWSFile", v.SecretAccessKey)
})
t.Run("IAM", func(t *testing.T) {
// prevent loading any actual credentials files from the user
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
// Spawn a server to emulate the EC2 Instance Metadata
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The client will actually make 3 requests here,
// first will be to get the IMDSv2 token, second to
// get the role, and third for the actual
// credentials. However, we can return credentials
// every request since we're not emulating a full
// IMDSv2 flow.
w.Write([]byte(`{"Code":"Success","AccessKeyId":"ExampleAccessKeyIDIAM","SecretAccessKey":"ExampleSecretAccessKeyIDIAM"}`))
}))
defer server.Close()
// Use the provided EC2 Instance Metadata server
creds := buildMinioCredentials(cfg, server.URL)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"IAM", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"IAM", v.SecretAccessKey)
})
})
}

3
modules/storage/testdata/aws_credentials vendored Normal file
View file

@ -0,0 +1,3 @@
[default]
aws_access_key_id=ExampleAccessKeyIDAWSFile
aws_secret_access_key=ExampleSecretAccessKeyIDAWSFile

12
modules/storage/testdata/minio.json vendored Normal file
View file

@ -0,0 +1,12 @@
{
"version": "10",
"aliases": {
"s3": {
"url": "https://s3.amazonaws.com",
"accessKey": "ExampleAccessKeyIDMinioFile",
"secretKey": "ExampleSecretAccessKeyIDMinioFile",
"api": "S3v4",
"path": "dns"
}
}
}

1
modules/structs/repo.go
View file

@ -114,6 +114,7 @@ type Repository struct {
// swagger:strfmt date-time
MirrorUpdated time.Time `json:"mirror_updated,omitempty"`
RepoTransfer *RepoTransfer `json:"repo_transfer"`
Topics []string `json:"topics"`
}
// GetName implements the gitrepo.Repository interface