Code/repo search (#2582)

Indexed search of repository contents (for default branch only)

modules/context/repo.go

@@ -367,6 +367,7 @@ func RepoAssignment() macaron.Handler {
 		ctx.Data["DisableSSH"] = setting.SSH.Disabled
 		ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
 		ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
+		ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
 		ctx.Data["CloneLink"] = repo.CloneLink()
 		ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
 

modules/indexer/repo.go

@@ -0,0 +1,199 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package indexer
+
+import (
+	"os"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
+	"github.com/blevesearch/bleve/analysis/token/camelcase"
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+)
+
+const repoIndexerAnalyzer = "repoIndexerAnalyzer"
+
+// repoIndexer (thread-safe) index for repository contents
+var repoIndexer bleve.Index
+
+// RepoIndexerOp type of operation to perform on repo indexer
+type RepoIndexerOp int
+
+const (
+	// RepoIndexerOpUpdate add/update a file's contents
+	RepoIndexerOpUpdate = iota
+
+	// RepoIndexerOpDelete delete a file
+	RepoIndexerOpDelete
+)
+
+// RepoIndexerData data stored in the repo indexer
+type RepoIndexerData struct {
+	RepoID  int64
+	Content string
+}
+
+// RepoIndexerUpdate an update to the repo indexer
+type RepoIndexerUpdate struct {
+	Filepath string
+	Op       RepoIndexerOp
+	Data     *RepoIndexerData
+}
+
+func (update RepoIndexerUpdate) addToBatch(batch *bleve.Batch) error {
+	id := filenameIndexerID(update.Data.RepoID, update.Filepath)
+	switch update.Op {
+	case RepoIndexerOpUpdate:
+		return batch.Index(id, update.Data)
+	case RepoIndexerOpDelete:
+		batch.Delete(id)
+	default:
+		log.Error(4, "Unrecognized repo indexer op: %d", update.Op)
+	}
+	return nil
+}
+
+// InitRepoIndexer initialize repo indexer
+func InitRepoIndexer(populateIndexer func() error) {
+	_, err := os.Stat(setting.Indexer.RepoPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if err = createRepoIndexer(); err != nil {
+				log.Fatal(4, "CreateRepoIndexer: %v", err)
+			}
+			if err = populateIndexer(); err != nil {
+				log.Fatal(4, "PopulateRepoIndex: %v", err)
+			}
+		} else {
+			log.Fatal(4, "InitRepoIndexer: %v", err)
+		}
+	} else {
+		repoIndexer, err = bleve.Open(setting.Indexer.RepoPath)
+		if err != nil {
+			log.Fatal(4, "InitRepoIndexer, open index: %v", err)
+		}
+	}
+}
+
+// createRepoIndexer create a repo indexer if one does not already exist
+func createRepoIndexer() error {
+	docMapping := bleve.NewDocumentMapping()
+	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
+
+	textFieldMapping := bleve.NewTextFieldMapping()
+	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
+
+	mapping := bleve.NewIndexMapping()
+	if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
+		return err
+	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
+		"type":          custom.Name,
+		"char_filters":  []string{},
+		"tokenizer":     unicode.Name,
+		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
+	}); err != nil {
+		return err
+	}
+	mapping.DefaultAnalyzer = repoIndexerAnalyzer
+	mapping.AddDocumentMapping("repo", docMapping)
+	var err error
+	repoIndexer, err = bleve.New(setting.Indexer.RepoPath, mapping)
+	return err
+}
+
+func filenameIndexerID(repoID int64, filename string) string {
+	return indexerID(repoID) + "_" + filename
+}
+
+func filenameOfIndexerID(indexerID string) string {
+	index := strings.IndexByte(indexerID, '_')
+	if index == -1 {
+		log.Error(4, "Unexpected ID in repo indexer: %s", indexerID)
+	}
+	return indexerID[index+1:]
+}
+
+// RepoIndexerBatch batch to add updates to
+func RepoIndexerBatch() *Batch {
+	return &Batch{
+		batch: repoIndexer.NewBatch(),
+		index: repoIndexer,
+	}
+}
+
+// DeleteRepoFromIndexer delete all of a repo's files from indexer
+func DeleteRepoFromIndexer(repoID int64) error {
+	query := numericEqualityQuery(repoID, "RepoID")
+	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
+	result, err := repoIndexer.Search(searchRequest)
+	if err != nil {
+		return err
+	}
+	batch := RepoIndexerBatch()
+	for _, hit := range result.Hits {
+		batch.batch.Delete(hit.ID)
+		if err = batch.flushIfFull(); err != nil {
+			return err
+		}
+	}
+	return batch.Flush()
+}
+
+// RepoSearchResult result of performing a search in a repo
+type RepoSearchResult struct {
+	StartIndex int
+	EndIndex   int
+	Filename   string
+	Content    string
+}
+
+// SearchRepoByKeyword searches for files in the specified repo.
+// Returns the matching file-paths
+func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
+	phraseQuery := bleve.NewMatchPhraseQuery(keyword)
+	phraseQuery.FieldVal = "Content"
+	phraseQuery.Analyzer = repoIndexerAnalyzer
+	indexerQuery := bleve.NewConjunctionQuery(
+		numericEqualityQuery(repoID, "RepoID"),
+		phraseQuery,
+	)
+	from := (page - 1) * pageSize
+	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
+	searchRequest.Fields = []string{"Content"}
+	searchRequest.IncludeLocations = true
+
+	result, err := repoIndexer.Search(searchRequest)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	searchResults := make([]*RepoSearchResult, len(result.Hits))
+	for i, hit := range result.Hits {
+		var startIndex, endIndex int = -1, -1
+		for _, locations := range hit.Locations["Content"] {
+			location := locations[0]
+			locationStart := int(location.Start)
+			locationEnd := int(location.End)
+			if startIndex < 0 || locationStart < startIndex {
+				startIndex = locationStart
+			}
+			if endIndex < 0 || locationEnd > endIndex {
+				endIndex = locationEnd
+			}
+		}
+		searchResults[i] = &RepoSearchResult{
+			StartIndex: startIndex,
+			EndIndex:   endIndex,
+			Filename:   filenameOfIndexerID(hit.ID),
+			Content:    hit.Fields["Content"].(string),
+		}
+	}
+	return int64(result.Total), searchResults, nil
+}

modules/search/search.go

@@ -0,0 +1,128 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package search
+
+import (
+	"bytes"
+	gotemplate "html/template"
+	"strings"
+
+	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/indexer"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// Result a search result to display
+type Result struct {
+	Filename       string
+	HighlightClass string
+	LineNumbers    []int
+	FormattedLines gotemplate.HTML
+}
+
+func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
+	startIndex := selectionStartIndex
+	numLinesBefore := 0
+	for ; startIndex > 0; startIndex-- {
+		if content[startIndex-1] == '\n' {
+			if numLinesBefore == 1 {
+				break
+			}
+			numLinesBefore++
+		}
+	}
+
+	endIndex := selectionEndIndex
+	numLinesAfter := 0
+	for ; endIndex < len(content); endIndex++ {
+		if content[endIndex] == '\n' {
+			if numLinesAfter == 1 {
+				break
+			}
+			numLinesAfter++
+		}
+	}
+
+	return startIndex, endIndex
+}
+
+func writeStrings(buf *bytes.Buffer, strs ...string) error {
+	for _, s := range strs {
+		_, err := buf.WriteString(s)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*Result, error) {
+	startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
+
+	var formattedLinesBuffer bytes.Buffer
+
+	contentLines := strings.SplitAfter(result.Content[startIndex:endIndex], "\n")
+	lineNumbers := make([]int, len(contentLines))
+	index := startIndex
+	for i, line := range contentLines {
+		var err error
+		if index < result.EndIndex &&
+			result.StartIndex < index+len(line) &&
+			result.StartIndex < result.EndIndex {
+			openActiveIndex := util.Max(result.StartIndex-index, 0)
+			closeActiveIndex := util.Min(result.EndIndex-index, len(line))
+			err = writeStrings(&formattedLinesBuffer,
+				`<li>`,
+				line[:openActiveIndex],
+				`<span class='active'>`,
+				line[openActiveIndex:closeActiveIndex],
+				`</span>`,
+				line[closeActiveIndex:],
+				`</li>`,
+			)
+		} else {
+			err = writeStrings(&formattedLinesBuffer,
+				`<li>`,
+				line,
+				`</li>`,
+			)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		lineNumbers[i] = startLineNum + i
+		index += len(line)
+	}
+	return &Result{
+		Filename:       result.Filename,
+		HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
+		LineNumbers:    lineNumbers,
+		FormattedLines: gotemplate.HTML(formattedLinesBuffer.String()),
+	}, nil
+}
+
+// PerformSearch perform a search on a repository
+func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
+	if len(keyword) == 0 {
+		return 0, nil, nil
+	}
+
+	total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	displayResults := make([]*Result, len(results))
+
+	for i, result := range results {
+		startIndex, endIndex := indices(result.Content, result.StartIndex, result.EndIndex)
+		displayResults[i], err = searchResult(result, startIndex, endIndex)
+		if err != nil {
+			return 0, nil, err
+		}
+	}
+	return int(total), displayResults, nil
+}

modules/setting/setting.go

@@ -140,8 +140,11 @@ var (
 
 	// Indexer settings
 	Indexer struct {
-		IssuePath         string
-		UpdateQueueLength int
+		IssuePath          string
+		RepoIndexerEnabled bool
+		RepoPath           string
+		UpdateQueueLength  int
+		MaxIndexerFileSize int64
 	}
 
 	// Webhook settings
@@ -234,12 +237,13 @@ var (
 
 	// UI settings
 	UI = struct {
-		ExplorePagingNum   int
-		IssuePagingNum     int
-		FeedMaxCommitNum   int
-		ThemeColorMetaTag  string
-		MaxDisplayFileSize int64
-		ShowUserEmail      bool
+		ExplorePagingNum    int
+		IssuePagingNum      int
+		RepoSearchPagingNum int
+		FeedMaxCommitNum    int
+		ThemeColorMetaTag   string
+		MaxDisplayFileSize  int64
+		ShowUserEmail       bool
 
 		Admin struct {
 			UserPagingNum   int
@@ -256,11 +260,12 @@ var (
 			Keywords    string
 		} `ini:"ui.meta"`
 	}{
-		ExplorePagingNum:   20,
-		IssuePagingNum:     10,
-		FeedMaxCommitNum:   5,
-		ThemeColorMetaTag:  `#6cc644`,
-		MaxDisplayFileSize: 8388608,
+		ExplorePagingNum:    20,
+		IssuePagingNum:      10,
+		RepoSearchPagingNum: 10,
+		FeedMaxCommitNum:    5,
+		ThemeColorMetaTag:   `#6cc644`,
+		MaxDisplayFileSize:  8388608,
 		Admin: struct {
 			UserPagingNum   int
 			RepoPagingNum   int

modules/util/path.go

@@ -0,0 +1,16 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import "path/filepath"
+
+// EnsureAbsolutePath ensure that a path is absolute, making it
+// relative to absoluteBase if necessary
+func EnsureAbsolutePath(path string, absoluteBase string) string {
+	if filepath.IsAbs(path) {
+		return path
+	}
+	return filepath.Join(absoluteBase, path)
+}

modules/util/util.go

@@ -38,3 +38,19 @@ func OptionalBoolOf(b bool) OptionalBool {
 	}
 	return OptionalBoolFalse
 }
+
+// Max max of two ints
+func Max(a, b int) int {
+	if a < b {
+		return b
+	}
+	return a
+}
+
+// Min min of two ints
+func Min(a, b int) int {
+	if a > b {
+		return b
+	}
+	return a
+}