Memory usage improvements (#3013)

* govendor update code.gitea.io/git

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>

* Greatly improve memory usage

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>

vendor/code.gitea.io/git/blob.go

@@ -6,7 +6,11 @@ package git
 
 import (
 	"bytes"
+	"fmt"
 	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
 )
 
 // Blob represents a Git object.
@@ -18,14 +22,52 @@ type Blob struct {
 // Data gets content of blob all at once and wrap it as io.Reader.
 // This can be very slow and memory consuming for huge content.
 func (b *Blob) Data() (io.Reader, error) {
-	stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path)
-	if err != nil {
-		return nil, err
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+
+	// Preallocate memory to save ~50% memory usage on big files.
+	stdout.Grow(int(b.Size() + 2048))
+
+	if err := b.DataPipeline(stdout, stderr); err != nil {
+		return nil, concatenateError(err, stderr.String())
 	}
-	return bytes.NewBuffer(stdout), nil
+	return stdout, nil
 }
 
 // DataPipeline gets content of blob and write the result or error to stdout or stderr
 func (b *Blob) DataPipeline(stdout, stderr io.Writer) error {
 	return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr)
 }
+
+type cmdReadCloser struct {
+	cmd    *exec.Cmd
+	stdout io.Reader
+}
+
+func (c cmdReadCloser) Read(p []byte) (int, error) {
+	return c.stdout.Read(p)
+}
+
+func (c cmdReadCloser) Close() error {
+	io.Copy(ioutil.Discard, c.stdout)
+	return c.cmd.Wait()
+}
+
+// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
+// Calling the Close function on the result will discard all unread output.
+func (b *Blob) DataAsync() (io.ReadCloser, error) {
+	cmd := exec.Command("git", "show", b.ID.String())
+	cmd.Dir = b.repo.Path
+	cmd.Stderr = os.Stderr
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("StdoutPipe: %v", err)
+	}
+
+	if err = cmd.Start(); err != nil {
+		return nil, fmt.Errorf("Start: %v", err)
+	}
+
+	return cmdReadCloser{stdout: stdout, cmd: cmd}, nil
+}

vendor/code.gitea.io/git/commit.go

@@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool {
 		return false
 	}
 
-	dataRc, err := blob.Data()
+	dataRc, err := blob.DataAsync()
 	if err != nil {
 		return false
 	}
+	defer dataRc.Close()
 	buf := make([]byte, 1024)
 	n, _ := dataRc.Read(buf)
 	buf = buf[:n]

vendor/code.gitea.io/git/git.go

@@ -25,7 +25,7 @@ var (
 	// Prefix the log prefix
 	Prefix = "[git-module] "
 	// GitVersionRequired is the minimum Git version required
-	GitVersionRequired = "1.8.1.6"
+	GitVersionRequired = "1.7.2"
 )
 
 func log(format string, args ...interface{}) {

vendor/vendor.json

@@ -3,10 +3,10 @@
 	"ignore": "test appengine",
 	"package": [
 		{
-			"checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=",
+			"checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=",
 			"path": "code.gitea.io/git",
-			"revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a",
-			"revisionTime": "2017-10-23T00:52:09Z"
+			"revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54",
+			"revisionTime": "2017-11-28T15:25:05Z"
 		},
 		{
 			"checksumSHA1": "QQ7g7B9+EIzGjO14KCGEs9TNEzM=",