Add generic set type (#21408)

This PR adds a generic set type to get rid of maps used as sets.

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
KN4CK3R 2022-10-12 07:18:26 +02:00 committed by GitHub
parent e84558b093
commit 0e57ff7eee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 328 additions and 324 deletions

View file

@ -241,15 +241,6 @@ func Int64sToStrings(ints []int64) []string {
return strs
}
// Int64sToMap converts a slice of int64 to a int64 map.
func Int64sToMap(ints []int64) map[int64]bool {
m := make(map[int64]bool)
for _, i := range ints {
m[i] = true
}
return m
}
// Int64sContains returns if a int64 in a slice of int64
func Int64sContains(intsSlice []int64, a int64) bool {
for _, c := range intsSlice {

View file

@ -214,16 +214,7 @@ func TestInt64sToStrings(t *testing.T) {
)
}
func TestInt64sToMap(t *testing.T) {
assert.Equal(t, map[int64]bool{}, Int64sToMap([]int64{}))
assert.Equal(t,
map[int64]bool{1: true, 4: true, 16: true},
Int64sToMap([]int64{1, 4, 16}),
)
}
func TestInt64sContains(t *testing.T) {
assert.Equal(t, map[int64]bool{}, Int64sToMap([]int64{}))
assert.True(t, Int64sContains([]int64{6, 44324, 4324, 32, 1, 2323}, 1))
assert.True(t, Int64sContains([]int64{2323}, 2323))
assert.False(t, Int64sContains([]int64{6, 44324, 4324, 32, 1, 2323}, 232))

View file

@ -1,14 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package container
// KeysInt64 returns keys slice for a map with int64 key
func KeysInt64(m map[int64]struct{}) []int64 {
keys := make([]int64, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}

57
modules/container/set.go Normal file
View file

@ -0,0 +1,57 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package container
type Set[T comparable] map[T]struct{}
// SetOf creates a set and adds the specified elements to it.
func SetOf[T comparable](values ...T) Set[T] {
s := make(Set[T], len(values))
s.AddMultiple(values...)
return s
}
// Add adds the specified element to a set.
// Returns true if the element is added; false if the element is already present.
func (s Set[T]) Add(value T) bool {
if _, has := s[value]; !has {
s[value] = struct{}{}
return true
}
return false
}
// AddMultiple adds the specified elements to a set.
func (s Set[T]) AddMultiple(values ...T) {
for _, value := range values {
s.Add(value)
}
}
// Contains determines whether a set contains the specified element.
// Returns true if the set contains the specified element; otherwise, false.
func (s Set[T]) Contains(value T) bool {
_, has := s[value]
return has
}
// Remove removes the specified element.
// Returns true if the element is successfully found and removed; otherwise, false.
func (s Set[T]) Remove(value T) bool {
if _, has := s[value]; has {
delete(s, value)
return true
}
return false
}
// Values gets a list of all elements in the set.
func (s Set[T]) Values() []T {
keys := make([]T, 0, len(s))
for k := range s {
keys = append(keys, k)
}
return keys
}

View file

@ -0,0 +1,37 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package container
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSet(t *testing.T) {
s := make(Set[string])
assert.True(t, s.Add("key1"))
assert.False(t, s.Add("key1"))
assert.True(t, s.Add("key2"))
assert.True(t, s.Contains("key1"))
assert.True(t, s.Contains("key2"))
assert.False(t, s.Contains("key3"))
assert.True(t, s.Remove("key2"))
assert.False(t, s.Contains("key2"))
assert.False(t, s.Remove("key3"))
s.AddMultiple("key4", "key5")
assert.True(t, s.Contains("key4"))
assert.True(t, s.Contains("key5"))
s = SetOf("key6", "key7")
assert.False(t, s.Contains("key1"))
assert.True(t, s.Contains("key6"))
assert.True(t, s.Contains("key7"))
}

View file

@ -14,6 +14,7 @@ import (
"strings"
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
@ -40,7 +41,7 @@ func checkAuthorizedKeys(ctx context.Context, logger log.Logger, autofix bool) e
}
defer f.Close()
linesInAuthorizedKeys := map[string]bool{}
linesInAuthorizedKeys := make(container.Set[string])
scanner := bufio.NewScanner(f)
for scanner.Scan() {
@ -48,7 +49,7 @@ func checkAuthorizedKeys(ctx context.Context, logger log.Logger, autofix bool) e
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
linesInAuthorizedKeys[line] = true
linesInAuthorizedKeys.Add(line)
}
f.Close()
@ -64,7 +65,7 @@ func checkAuthorizedKeys(ctx context.Context, logger log.Logger, autofix bool) e
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
if ok := linesInAuthorizedKeys[line]; ok {
if linesInAuthorizedKeys.Contains(line) {
continue
}
if !autofix {

View file

@ -14,6 +14,8 @@ import (
"sort"
"strings"
"code.gitea.io/gitea/modules/container"
"github.com/djherbis/buffer"
"github.com/djherbis/nio/v3"
)
@ -339,7 +341,7 @@ func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath st
lastEmptyParent := head.ID.String()
commitSinceLastEmptyParent := uint64(0)
commitSinceNextRestart := uint64(0)
parentRemaining := map[string]bool{}
parentRemaining := make(container.Set[string])
changed := make([]bool, len(paths))
@ -365,7 +367,7 @@ heaploop:
if current == nil {
break heaploop
}
delete(parentRemaining, current.CommitID)
parentRemaining.Remove(current.CommitID)
if current.Paths != nil {
for i, found := range current.Paths {
if !found {
@ -410,14 +412,12 @@ heaploop:
}
}
g = NewLogNameStatusRepoParser(ctx, repo.Path, lastEmptyParent, treepath, remainingPaths...)
parentRemaining = map[string]bool{}
parentRemaining = make(container.Set[string])
nextRestart = (remaining * 3) / 4
continue heaploop
}
}
for _, parent := range current.ParentIDs {
parentRemaining[parent] = true
}
parentRemaining.AddMultiple(current.ParentIDs...)
}
g.Close()

View file

@ -13,6 +13,8 @@ import (
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/container"
)
// CodeActivityStats represents git statistics data
@ -80,7 +82,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string)
stats.Additions = 0
stats.Deletions = 0
authors := make(map[string]*CodeActivityAuthor)
files := make(map[string]bool)
files := make(container.Set[string])
var author string
p := 0
for scanner.Scan() {
@ -119,9 +121,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string)
stats.Deletions += c
}
}
if _, ok := files[parts[2]]; !ok {
files[parts[2]] = true
}
files.Add(parts[2])
}
}
}

View file

@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/container"
api "code.gitea.io/gitea/modules/structs"
"gitea.com/go-chi/binding"
@ -43,7 +44,7 @@ func validateYaml(template *api.IssueTemplate) error {
if len(template.Fields) == 0 {
return fmt.Errorf("'body' is required")
}
ids := map[string]struct{}{}
ids := make(container.Set[string])
for idx, field := range template.Fields {
if err := validateID(field, idx, ids); err != nil {
return err
@ -125,7 +126,7 @@ func validateRequired(field *api.IssueFormField, idx int) error {
return validateBoolItem(newErrorPosition(idx, field.Type), field.Validations, "required")
}
func validateID(field *api.IssueFormField, idx int, ids map[string]struct{}) error {
func validateID(field *api.IssueFormField, idx int, ids container.Set[string]) error {
if field.Type == api.IssueFormFieldTypeMarkdown {
// The ID is not required for a markdown field
return nil
@ -139,10 +140,9 @@ func validateID(field *api.IssueFormField, idx int, ids map[string]struct{}) err
if binding.AlphaDashPattern.MatchString(field.ID) {
return position.Errorf("'id' should contain only alphanumeric, '-' and '_'")
}
if _, ok := ids[field.ID]; ok {
if !ids.Add(field.ID) {
return position.Errorf("'id' should be unique")
}
ids[field.ID] = struct{}{}
return nil
}

View file

@ -10,6 +10,7 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/common"
"code.gitea.io/gitea/modules/setting"
@ -198,7 +199,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
}
type prefixedIDs struct {
values map[string]bool
values container.Set[string]
}
// Generate generates a new element id.
@ -219,14 +220,12 @@ func (p *prefixedIDs) GenerateWithDefault(value, dft []byte) []byte {
if !bytes.HasPrefix(result, []byte("user-content-")) {
result = append([]byte("user-content-"), result...)
}
if _, ok := p.values[util.BytesToReadOnlyString(result)]; !ok {
p.values[util.BytesToReadOnlyString(result)] = true
if p.values.Add(util.BytesToReadOnlyString(result)) {
return result
}
for i := 1; ; i++ {
newResult := fmt.Sprintf("%s-%d", result, i)
if _, ok := p.values[newResult]; !ok {
p.values[newResult] = true
if p.values.Add(newResult) {
return []byte(newResult)
}
}
@ -234,12 +233,12 @@ func (p *prefixedIDs) GenerateWithDefault(value, dft []byte) []byte {
// Put puts a given element id to the used ids table.
func (p *prefixedIDs) Put(value []byte) {
p.values[util.BytesToReadOnlyString(value)] = true
p.values.Add(util.BytesToReadOnlyString(value))
}
func newPrefixedIDs() *prefixedIDs {
return &prefixedIDs{
values: map[string]bool{},
values: make(container.Set[string]),
}
}

View file

@ -10,6 +10,7 @@ import (
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification/base"
@ -123,14 +124,14 @@ func (ns *notificationService) NotifyNewPullRequest(pr *issues_model.PullRequest
log.Error("Unable to load issue: %d for pr: %d: Error: %v", pr.IssueID, pr.ID, err)
return
}
toNotify := make(map[int64]struct{}, 32)
toNotify := make(container.Set[int64], 32)
repoWatchers, err := repo_model.GetRepoWatchersIDs(db.DefaultContext, pr.Issue.RepoID)
if err != nil {
log.Error("GetRepoWatchersIDs: %v", err)
return
}
for _, id := range repoWatchers {
toNotify[id] = struct{}{}
toNotify.Add(id)
}
issueParticipants, err := issues_model.GetParticipantsIDsByIssueID(pr.IssueID)
if err != nil {
@ -138,11 +139,11 @@ func (ns *notificationService) NotifyNewPullRequest(pr *issues_model.PullRequest
return
}
for _, id := range issueParticipants {
toNotify[id] = struct{}{}
toNotify.Add(id)
}
delete(toNotify, pr.Issue.PosterID)
for _, mention := range mentions {
toNotify[mention.ID] = struct{}{}
toNotify.Add(mention.ID)
}
for receiverID := range toNotify {
_ = ns.issueQueue.Push(issueNotificationOpts{

View file

@ -11,6 +11,7 @@ import (
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@ -83,11 +84,11 @@ func AssetsHandlerFunc(opts *Options) http.HandlerFunc {
}
// parseAcceptEncoding parse Accept-Encoding: deflate, gzip;q=1.0, *;q=0.5 as compress methods
func parseAcceptEncoding(val string) map[string]bool {
func parseAcceptEncoding(val string) container.Set[string] {
parts := strings.Split(val, ";")
types := make(map[string]bool)
types := make(container.Set[string])
for _, v := range strings.Split(parts[0], ",") {
types[strings.TrimSpace(v)] = true
types.Add(strings.TrimSpace(v))
}
return types
}

View file

@ -7,28 +7,23 @@ package public
import (
"testing"
"code.gitea.io/gitea/modules/container"
"github.com/stretchr/testify/assert"
)
func TestParseAcceptEncoding(t *testing.T) {
kases := []struct {
Header string
Expected map[string]bool
Expected container.Set[string]
}{
{
Header: "deflate, gzip;q=1.0, *;q=0.5",
Expected: map[string]bool{
"deflate": true,
"gzip": true,
},
Header: "deflate, gzip;q=1.0, *;q=0.5",
Expected: container.SetOf("deflate", "gzip"),
},
{
Header: " gzip, deflate, br",
Expected: map[string]bool{
"deflate": true,
"gzip": true,
"br": true,
},
Header: " gzip, deflate, br",
Expected: container.SetOf("deflate", "gzip", "br"),
},
}

View file

@ -60,7 +60,7 @@ func AssetIsDir(name string) (bool, error) {
// serveContent serve http content
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
if encodings["gzip"] {
if encodings.Contains("gzip") {
if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok {
rdGzip := bytes.NewReader(cf.GzipBytes())
// all static files are managed by Gitea, so we can make sure every file has the correct ext name

View file

@ -12,6 +12,7 @@ import (
"sync/atomic"
"time"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
)
@ -33,7 +34,7 @@ type ChannelUniqueQueueConfiguration ChannelQueueConfiguration
type ChannelUniqueQueue struct {
*WorkerPool
lock sync.Mutex
table map[string]bool
table container.Set[string]
shutdownCtx context.Context
shutdownCtxCancel context.CancelFunc
terminateCtx context.Context
@ -58,7 +59,7 @@ func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue
shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx)
queue := &ChannelUniqueQueue{
table: map[string]bool{},
table: make(container.Set[string]),
shutdownCtx: shutdownCtx,
shutdownCtxCancel: shutdownCtxCancel,
terminateCtx: terminateCtx,
@ -73,7 +74,7 @@ func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue
bs, _ := json.Marshal(datum)
queue.lock.Lock()
delete(queue.table, string(bs))
queue.table.Remove(string(bs))
queue.lock.Unlock()
if u := handle(datum); u != nil {
@ -127,16 +128,15 @@ func (q *ChannelUniqueQueue) PushFunc(data Data, fn func() error) error {
q.lock.Unlock()
}
}()
if _, ok := q.table[string(bs)]; ok {
if !q.table.Add(string(bs)) {
return ErrAlreadyInQueue
}
// FIXME: We probably need to implement some sort of limit here
// If the downstream queue blocks this table will grow without limit
q.table[string(bs)] = true
if fn != nil {
err := fn()
if err != nil {
delete(q.table, string(bs))
q.table.Remove(string(bs))
return err
}
}
@ -155,8 +155,7 @@ func (q *ChannelUniqueQueue) Has(data Data) (bool, error) {
q.lock.Lock()
defer q.lock.Unlock()
_, has := q.table[string(bs)]
return has, nil
return q.table.Contains(string(bs)), nil
}
// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager

View file

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/models/organization"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
@ -275,7 +276,7 @@ func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository)
return pullMirrorReleaseSync(repo, gitRepo)
}
existingRelTags := make(map[string]struct{})
existingRelTags := make(container.Set[string])
opts := repo_model.FindReleasesOptions{
IncludeDrafts: true,
IncludeTags: true,
@ -303,14 +304,14 @@ func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository)
return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
existingRelTags.Add(strings.ToLower(rel.TagName))
}
}
}
_, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error {
tagName := strings.TrimPrefix(refname, git.TagPrefix)
if _, ok := existingRelTags[strings.ToLower(tagName)]; ok {
if existingRelTags.Contains(strings.ToLower(tagName)) {
return nil
}

View file

@ -9,6 +9,7 @@ import (
"strconv"
"time"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/log"
ini "gopkg.in/ini.v1"
@ -109,8 +110,8 @@ func NewQueueService() {
// Now handle the old issue_indexer configuration
// FIXME: DEPRECATED to be removed in v1.18.0
section := Cfg.Section("queue.issue_indexer")
directlySet := toDirectlySetKeysMap(section)
if !directlySet["TYPE"] && defaultType == "" {
directlySet := toDirectlySetKeysSet(section)
if !directlySet.Contains("TYPE") && defaultType == "" {
switch typ := Cfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(""); typ {
case "levelqueue":
_, _ = section.NewKey("TYPE", "level")
@ -124,25 +125,25 @@ func NewQueueService() {
log.Fatal("Unsupported indexer queue type: %v", typ)
}
}
if !directlySet["LENGTH"] {
if !directlySet.Contains("LENGTH") {
length := Cfg.Section("indexer").Key("UPDATE_BUFFER_LEN").MustInt(0)
if length != 0 {
_, _ = section.NewKey("LENGTH", strconv.Itoa(length))
}
}
if !directlySet["BATCH_LENGTH"] {
if !directlySet.Contains("BATCH_LENGTH") {
fallback := Cfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(0)
if fallback != 0 {
_, _ = section.NewKey("BATCH_LENGTH", strconv.Itoa(fallback))
}
}
if !directlySet["DATADIR"] {
if !directlySet.Contains("DATADIR") {
queueDir := filepath.ToSlash(Cfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_DIR").MustString(""))
if queueDir != "" {
_, _ = section.NewKey("DATADIR", queueDir)
}
}
if !directlySet["CONN_STR"] {
if !directlySet.Contains("CONN_STR") {
connStr := Cfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("")
if connStr != "" {
_, _ = section.NewKey("CONN_STR", connStr)
@ -178,19 +179,19 @@ func handleOldLengthConfiguration(queueName, oldSection, oldKey string, defaultV
}
section := Cfg.Section("queue." + queueName)
directlySet := toDirectlySetKeysMap(section)
if !directlySet["LENGTH"] {
directlySet := toDirectlySetKeysSet(section)
if !directlySet.Contains("LENGTH") {
_, _ = section.NewKey("LENGTH", strconv.Itoa(value))
}
}
// toDirectlySetKeysMap returns a bool map of keys directly set by this section
// toDirectlySetKeysSet returns a set of keys directly set by this section
// Note: we cannot use section.HasKey(...) as that will immediately set the Key if a parent section has the Key
// but this section does not.
func toDirectlySetKeysMap(section *ini.Section) map[string]bool {
sectionMap := map[string]bool{}
func toDirectlySetKeysSet(section *ini.Section) container.Set[string] {
sections := make(container.Set[string])
for _, key := range section.Keys() {
sectionMap[key.Name()] = true
sections.Add(key.Name())
}
return sectionMap
return sections
}

View file

@ -21,6 +21,7 @@ import (
"text/template"
"time"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/user"
@ -234,7 +235,7 @@ var (
DefaultTheme string
Themes []string
Reactions []string
ReactionsMap map[string]bool `ini:"-"`
ReactionsLookup container.Set[string] `ini:"-"`
CustomEmojis []string
CustomEmojisMap map[string]string `ini:"-"`
SearchRepoDescription bool
@ -1100,9 +1101,9 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
newMarkup()
UI.ReactionsMap = make(map[string]bool)
UI.ReactionsLookup = make(container.Set[string])
for _, reaction := range UI.Reactions {
UI.ReactionsMap[reaction] = true
UI.ReactionsLookup.Add(reaction)
}
UI.CustomEmojisMap = make(map[string]string)
for _, emoji := range UI.CustomEmojis {

View file

@ -6,6 +6,8 @@ package sync
import (
"sync"
"code.gitea.io/gitea/modules/container"
)
// StatusTable is a table maintains true/false values.
@ -14,13 +16,13 @@ import (
// in different goroutines.
type StatusTable struct {
lock sync.RWMutex
pool map[string]struct{}
pool container.Set[string]
}
// NewStatusTable initializes and returns a new StatusTable object.
func NewStatusTable() *StatusTable {
return &StatusTable{
pool: make(map[string]struct{}),
pool: make(container.Set[string]),
}
}
@ -28,32 +30,29 @@ func NewStatusTable() *StatusTable {
// Returns whether set value was set to true
func (p *StatusTable) StartIfNotRunning(name string) bool {
p.lock.Lock()
_, ok := p.pool[name]
if !ok {
p.pool[name] = struct{}{}
}
added := p.pool.Add(name)
p.lock.Unlock()
return !ok
return added
}
// Start sets value of given name to true in the pool.
func (p *StatusTable) Start(name string) {
p.lock.Lock()
p.pool[name] = struct{}{}
p.pool.Add(name)
p.lock.Unlock()
}
// Stop sets value of given name to false in the pool.
func (p *StatusTable) Stop(name string) {
p.lock.Lock()
delete(p.pool, name)
p.pool.Remove(name)
p.lock.Unlock()
}
// IsRunning checks if value of given name is set to true in the pool.
func (p *StatusTable) IsRunning(name string) bool {
p.lock.RLock()
_, ok := p.pool[name]
exists := p.pool.Contains(name)
p.lock.RUnlock()
return ok
return exists
}