toutf8 improved & add max git diff lines

modules/base/template.go

@@ -8,13 +8,16 @@ import (
 	"bytes"
 	"container/list"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"runtime"
 	"strings"
 	"time"
+	"code.google.com/p/mahonia"
 
 	"github.com/gogits/gogs/modules/setting"
+	"github.com/saintfish/chardet"
 )
 
 func Str2html(raw string) template.HTML {
@@ -45,6 +48,29 @@ func ShortSha(sha1 string) string {
 	return sha1
 }
 
+func ToUtf8WithErr(content []byte) (error, string) {
+	detector := chardet.NewTextDetector()
+	result, err := detector.DetectBest(content)
+	if err != nil {
+		return err, ""
+	}
+
+	if result.Charset == "utf8" {
+		return nil, string(content)
+	}
+
+	decoder := mahonia.NewDecoder(result.Charset)
+	if decoder != nil {
+		return nil, decoder.ConvertString(string(content))
+	}
+	return errors.New("unknow char decoder"), string(content)
+}
+
+func ToUtf8(content string) string {
+	_, res := ToUtf8WithErr([]byte(content))
+	return res
+}
+
 var mailDomains = map[string]string{
 	"gmail.com": "gmail.com",
 }
@@ -103,6 +129,7 @@ var TemplateFuncs template.FuncMap = map[string]interface{}{
 	"ActionContent2Commits": ActionContent2Commits,
 	"Oauth2Icon":            Oauth2Icon,
 	"Oauth2Name":            Oauth2Name,
+	"ToUtf8":                ToUtf8,
 }
 
 type Actioner interface {

modules/setting/setting.go

@@ -64,6 +64,7 @@ var (
 	// Picture settings.
 	PictureService  string
 	DisableGravatar bool
+	MaxGitDiffLines int
 
 	// Log settings.
 	LogRootPath string
@@ -241,6 +242,8 @@ func NewConfigContext() {
 		[]string{"server"})
 	DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR")
 
+	MaxGitDiffLines = Cfg.MustInt("git", "MAX_GITDIFF_LINES", 5000)
+
 	Langs = Cfg.MustValueArray("i18n", "LANGS", ",")
 	Names = Cfg.MustValueArray("i18n", "NAMES", ",")
 }