Add more tests and docs for issue indexer, add db indexer type for searching from database (#6144)

* add more tests and docs for issue indexer, add db indexer type for searching from database

* fix typo

* fix typo

* fix lint

* improve docs

modules/indexer/issues/db.go

@@ -0,0 +1,45 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package issues
+
+import "code.gitea.io/gitea/models"
+
+// DBIndexer implements Indexer inteface to use database's like search
+type DBIndexer struct {
+}
+
+// Init dummy function
+func (db *DBIndexer) Init() (bool, error) {
+	return false, nil
+}
+
+// Index dummy function
+func (db *DBIndexer) Index(issue []*IndexerData) error {
+	return nil
+}
+
+// Delete dummy function
+func (db *DBIndexer) Delete(ids ...int64) error {
+	return nil
+}
+
+// Search dummy function
+func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchResult, error) {
+	total, ids, err := models.SearchIssueIDsByKeyword(kw, repoID, limit, start)
+	if err != nil {
+		return nil, err
+	}
+	var result = SearchResult{
+		Total: total,
+		Hits:  make([]Match, 0, limit),
+	}
+	for _, id := range ids {
+		result.Hits = append(result.Hits, Match{
+			ID:     id,
+			RepoID: repoID,
+		})
+	}
+	return &result, nil
+}

modules/indexer/issues/indexer.go

@@ -33,7 +33,8 @@ type Match struct {
 
 // SearchResult represents search results
 type SearchResult struct {
-	Hits []Match
+	Total int64
+	Hits  []Match
 }
 
 // Indexer defines an inteface to indexer issues contents
@@ -54,6 +55,7 @@ var (
 // all issue index done.
 func InitIssueIndexer(syncReindex bool) error {
 	var populate bool
+	var dummyQueue bool
 	switch setting.Indexer.IssueType {
 	case "bleve":
 		issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
@@ -62,10 +64,17 @@ func InitIssueIndexer(syncReindex bool) error {
 			return err
 		}
 		populate = !exist
+	case "db":
+		issueIndexer = &DBIndexer{}
+		dummyQueue = true
 	default:
 		return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
 	}
 
+	if dummyQueue {
+		return nil
+	}
+
 	var err error
 	switch setting.Indexer.IssueIndexerQueueType {
 	case setting.LevelQueueType:

modules/indexer/issues/indexer_test.go

@@ -48,4 +48,8 @@ func TestSearchIssues(t *testing.T) {
 	ids, err = SearchIssuesByKeyword(1, "for")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
+
+	ids, err = SearchIssuesByKeyword(1, "good")
+	assert.NoError(t, err)
+	assert.EqualValues(t, []int64{1}, ids)
 }

modules/indexer/issues/queue.go

@@ -7,5 +7,19 @@ package issues
 // Queue defines an interface to save an issue indexer queue
 type Queue interface {
 	Run() error
-	Push(*IndexerData)
+	Push(*IndexerData) error
+}
+
+// DummyQueue represents an empty queue
+type DummyQueue struct {
+}
+
+// Run starts to run the queue
+func (b *DummyQueue) Run() error {
+	return nil
+}
+
+// Push pushes data to indexer
+func (b *DummyQueue) Push(*IndexerData) error {
+	return nil
 }

modules/indexer/issues/queue_channel.go

@@ -33,6 +33,11 @@ func (c *ChannelQueue) Run() error {
 	for {
 		select {
 		case data := <-c.queue:
+			if data.IsDelete {
+				c.indexer.Delete(data.IDs...)
+				continue
+			}
+
 			datas = append(datas, data)
 			if len(datas) >= c.batchNumber {
 				c.indexer.Index(datas)
@@ -51,6 +56,7 @@ func (c *ChannelQueue) Run() error {
 }
 
 // Push will push the indexer data to queue
-func (c *ChannelQueue) Push(data *IndexerData) {
+func (c *ChannelQueue) Push(data *IndexerData) error {
 	c.queue <- data
+	return nil
 }

modules/indexer/issues/queue_disk.go

@@ -94,14 +94,10 @@ func (l *LevelQueue) Run() error {
 }
 
 // Push will push the indexer data to queue
-func (l *LevelQueue) Push(data *IndexerData) {
+func (l *LevelQueue) Push(data *IndexerData) error {
 	bs, err := json.Marshal(data)
 	if err != nil {
-		log.Error(4, "Marshal: %v", err)
-		return
-	}
-	err = l.queue.LPush(bs)
-	if err != nil {
-		log.Error(4, "LPush: %v", err)
+		return err
 	}
+	return l.queue.LPush(bs)
 }

modules/setting/indexer.go

@@ -38,6 +38,7 @@ var (
 
 func newIndexerService() {
 	sec := Cfg.Section("indexer")
+	Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve")
 	Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve"))
 	if !filepath.IsAbs(Indexer.IssuePath) {
 		Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath)