Improve listing performance by using go-git (#6478)

* Use go-git for tree reading and commit info lookup.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use TreeEntry.IsRegular() instead of ObjectType that was removed.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use the treePath to optimize commit info search.

Signed-off-by: Filip Navara <navara@emclient.com>

* Extract the latest commit at treePath along with the other commits.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix listing commit info for a directory that was created in one commit and never modified after.

Signed-off-by: Filip Navara <navara@emclient.com>

* Avoid nearly all external 'git' invocations when doing directory listing (.editorconfig code path is still hit).

Signed-off-by: Filip Navara <navara@emclient.com>

* Use go-git for reading blobs.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make SHA1 type alias for plumbing.Hash in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make Signature type alias for object.Signature in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix GetCommitsInfo for repository with only one commit.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix PGP signature verification.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix issues with walking commit graph across merges.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix typo in condition.

Signed-off-by: Filip Navara <navara@emclient.com>

* Speed up loading branch list by keeping the repository reference (and thus all the loaded packfile indexes).

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix lising submodules.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix build

Signed-off-by: Filip Navara <navara@emclient.com>

* Add back commit cache because of name-rev

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix tests

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix code style

* Fix spelling

* Address PR feedback

Signed-off-by: Filip Navara <navara@emclient.com>

* Update vendor module list

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix getting trees by commit id

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix remaining unit test failures

* Fix GetTreeBySHA

* Avoid running `git name-rev` if not necessary

Signed-off-by: Filip Navara <navara@emclient.com>

* Move Branch code to git module

* Clean up GPG signature verification and fix it for tagged commits

* Address PR feedback (import formatting, copyright headers)

* Make blob lookup by SHA working

* Update tests to use public API

* Allow getting content from any type of object through the blob interface

* Change test to actually expect the object content that is in the GIT repository

* Change one more test to actually expect the object content that is in the GIT repository

* Add comments

modules/git/tree.go

@@ -1,29 +1,31 @@
 // Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
 package git
 
 import (
+	"io"
 	"strings"
+
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
 
 // Tree represents a flat directory listing.
 type Tree struct {
-	ID   SHA1
-	repo *Repository
+	ID       SHA1
+	CommitID SHA1
+	repo     *Repository
+
+	gogitTree *object.Tree
 
 	// parent tree
 	ptree *Tree
-
-	entries       Entries
-	entriesParsed bool
-
-	entriesRecursive       Entries
-	entriesRecursiveParsed bool
 }
 
-// NewTree create a new tree according the repository and commit id
+// NewTree create a new tree according the repository and tree id
 func NewTree(repo *Repository, id SHA1) *Tree {
 	return &Tree{
 		ID:   id,
@@ -60,39 +62,68 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
 	return g, nil
 }
 
+func (t *Tree) loadTreeObject() error {
+	gogitTree, err := t.repo.gogitRepo.TreeObject(plumbing.Hash(t.ID))
+	if err != nil {
+		return err
+	}
+
+	t.gogitTree = gogitTree
+	return nil
+}
+
 // ListEntries returns all entries of current tree.
 func (t *Tree) ListEntries() (Entries, error) {
-	if t.entriesParsed {
-		return t.entries, nil
+	if t.gogitTree == nil {
+		err := t.loadTreeObject()
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path)
-	if err != nil {
-		return nil, err
+	entries := make([]*TreeEntry, len(t.gogitTree.Entries))
+	for i, entry := range t.gogitTree.Entries {
+		entries[i] = &TreeEntry{
+			ID:             entry.Hash,
+			gogitTreeEntry: &t.gogitTree.Entries[i],
+			ptree:          t,
+		}
 	}
 
-	t.entries, err = parseTreeEntries(stdout, t)
-	if err == nil {
-		t.entriesParsed = true
-	}
-
-	return t.entries, err
+	return entries, nil
 }
 
 // ListEntriesRecursive returns all entries of current tree recursively including all subtrees
 func (t *Tree) ListEntriesRecursive() (Entries, error) {
-	if t.entriesRecursiveParsed {
-		return t.entriesRecursive, nil
-	}
-	stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path)
-	if err != nil {
-		return nil, err
+	if t.gogitTree == nil {
+		err := t.loadTreeObject()
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	t.entriesRecursive, err = parseTreeEntries(stdout, t)
-	if err == nil {
-		t.entriesRecursiveParsed = true
+	var entries []*TreeEntry
+	seen := map[plumbing.Hash]bool{}
+	walker := object.NewTreeWalker(t.gogitTree, true, seen)
+	for {
+		_, entry, err := walker.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if seen[entry.Hash] {
+			continue
+		}
+
+		convertedEntry := &TreeEntry{
+			ID:             entry.Hash,
+			gogitTreeEntry: &entry,
+			ptree:          t,
+		}
+		entries = append(entries, convertedEntry)
 	}
 
-	return t.entriesRecursive, err
+	return entries, nil
 }