Detect charset and convert non UTF-8 files for display (#4950)

* Detect charset and convert non UTF-8 files for display

* Refactor and move function to correct module

* Revert unrelated changes

* More unrelated changes

* Duplicate content for small text to have better encoding detection

* Check if original content is valid before duplicating it
This commit is contained in:
Lauris BH 2018-09-29 11:33:54 +03:00 committed by Lunny Xiao
parent 6780661192
commit 81702e6ec9
3 changed files with 44 additions and 4 deletions

View file

@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates"
"github.com/Unknwon/paginater"
)
@ -99,7 +100,8 @@ func renderDirectory(ctx *context.Context, treeLink string) {
ctx.Data["FileSize"] = readmeFile.Size()
} else {
d, _ := ioutil.ReadAll(dataRc)
buf = append(buf, d...)
buf = templates.ToUTF8WithFallback(append(buf, d...))
if markup.Type(readmeFile.Name()) != "" {
ctx.Data["IsMarkup"] = true
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
@ -203,7 +205,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
d, _ := ioutil.ReadAll(dataRc)
buf = append(buf, d...)
buf = templates.ToUTF8WithFallback(append(buf, d...))
readmeExist := markup.IsReadmeFile(blob.Name())
ctx.Data["ReadmeExist"] = readmeExist