finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor renders (#15175)

Browse source 
* Refactor renders

* Some performance optimization

* Fix comment

* Transform reader

* Fix csv test

* Fix test

* Fix tests

* Improve optimaziation

* Fix test

* Fix test

* Detect file encoding with reader

* Improve optimaziation

* reduce memory usage

* improve code

* fix build

* Fix test

* Fix for go1.15

* Fix render

* Fix comment

* Fix lint

* Fix test

* Don't use NormalEOF when unnecessary

* revert change on util.go

* Apply suggestions from code review

Co-authored-by: zeripath <art27@cantab.net>

* rename function

* Take NormalEOF back

Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
Lunny Xiao 2021-04-20 06:25:08 +08:00 • committed by GitHub
parent c9cc6698d2
commit 9d99f6ab19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1027 additions and 627 deletions
Download patch file Download diff file Expand all files Collapse all files

38
routers/api/v1/misc/markdown.go
View file

@ -5,11 +5,11 @@
package misc
import (
"io/ioutil"
"net/http"
"strings"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
@ -55,7 +55,6 @@ func Markdown(ctx *context.APIContext) {
case "comment":
fallthrough
case "gfm":
md := []byte(form.Text)
urlPrefix := form.Context
meta := map[string]string{}
if !strings.HasPrefix(setting.AppSubURL+"/", urlPrefix) {
@ -77,22 +76,19 @@ func Markdown(ctx *context.APIContext) {
if form.Mode == "gfm" {
meta["mode"] = "document"
}
if form.Wiki {
_, err := ctx.Write([]byte(markdown.RenderWiki(md, urlPrefix, meta)))
if err != nil {
ctx.InternalServerError(err)
return
}
} else {
_, err := ctx.Write(markdown.Render(md, urlPrefix, meta))
if err != nil {
ctx.InternalServerError(err)
return
}
if err := markdown.Render(&markup.RenderContext{
URLPrefix: urlPrefix,
Metas: meta,
IsWiki: form.Wiki,
}, strings.NewReader(form.Text), ctx.Resp); err != nil {
ctx.InternalServerError(err)
return
}
default:
_, err := ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false))
if err != nil {
if err := markdown.RenderRaw(&markup.RenderContext{
URLPrefix: form.Context,
}, strings.NewReader(form.Text), ctx.Resp); err != nil {
ctx.InternalServerError(err)
return
}
@ -120,14 +116,8 @@ func MarkdownRaw(ctx *context.APIContext) {
// "$ref": "#/responses/MarkdownRender"
// "422":
// "$ref": "#/responses/validationError"
body, err := ioutil.ReadAll(ctx.Req.Body)
if err != nil {
ctx.Error(http.StatusUnprocessableEntity, "", err)
return
}
_, err = ctx.Write(markdown.RenderRaw(body, "", false))
if err != nil {
defer ctx.Req.Body.Close()
if err := markdown.RenderRaw(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil {
ctx.InternalServerError(err)
return
}

2
routers/init.go
View file

@ -143,7 +143,7 @@ func GlobalInit(ctx context.Context) {
NewServices()
highlight.NewContext()
external.RegisterParsers()
external.RegisterRenderers()
markup.Init()
if setting.EnableSQLite3 {

11
routers/org/home.go
View file

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
)
@ -37,7 +38,15 @@ func Home(ctx *context.Context) {
ctx.Data["PageIsUserProfile"] = true
ctx.Data["Title"] = org.DisplayName()
if len(org.Description) != 0 {
ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(org.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"}))
desc, err := markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: map[string]string{"mode": "document"},
}, org.Description)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.Data["RenderedDescription"] = desc
}
var orderBy models.SearchOrderBy

10
routers/repo/compare.go
View file

@ -10,7 +10,6 @@ import (
"errors"
"fmt"
"html"
"io/ioutil"
"net/http"
"path"
"path/filepath"
@ -117,14 +116,7 @@ func setCsvCompareContext(ctx *context.Context) {
}
defer reader.Close()
b, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}
b = charset.ToUTF8WithFallback(b)
return csv_module.CreateReaderAndGuessDelimiter(b), nil
return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader))
}
baseReader, err := csvReaderFromCommit(baseCommit)

55
routers/repo/issue.go
View file

@ -1131,8 +1131,14 @@ func ViewIssue(ctx *context.Context) {
}
ctx.Data["IssueWatch"] = iw
issue.RenderedContent = string(markdown.Render([]byte(issue.Content), ctx.Repo.RepoLink,
ctx.Repo.Repository.ComposeMetas()))
issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, issue.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
repo := ctx.Repo.Repository
@ -1289,9 +1295,14 @@ func ViewIssue(ctx *context.Context) {
return
}
comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
ctx.Repo.Repository.ComposeMetas()))
comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, comment.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
// Check tag.
tag, ok = marked[comment.PosterID]
if ok {
@ -1359,8 +1370,14 @@ func ViewIssue(ctx *context.Context) {
}
}
} else if comment.Type == models.CommentTypeCode || comment.Type == models.CommentTypeReview || comment.Type == models.CommentTypeDismissReview {
comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
ctx.Repo.Repository.ComposeMetas()))
comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, comment.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
if err = comment.LoadReview(); err != nil && !models.IsErrReviewNotExist(err) {
ctx.ServerError("LoadReview", err)
return
@ -1708,10 +1725,20 @@ func UpdateIssueContent(ctx *context.Context) {
files := ctx.QueryStrings("files[]")
if err := updateAttachments(issue, files); err != nil {
ctx.ServerError("UpdateAttachments", err)
return
}
content, err := markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Query("context"),
Metas: ctx.Repo.Repository.ComposeMetas(),
}, issue.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"content": string(markdown.Render([]byte(issue.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
"content": content,
"attachments": attachmentsHTML(ctx, issue.Attachments, issue.Content),
})
}
@ -2125,10 +2152,20 @@ func UpdateCommentContent(ctx *context.Context) {
files := ctx.QueryStrings("files[]")
if err := updateAttachments(comment, files); err != nil {
ctx.ServerError("UpdateAttachments", err)
return
}
content, err := markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Query("context"),
Metas: ctx.Repo.Repository.ComposeMetas(),
}, comment.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"content": string(markdown.Render([]byte(comment.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
"content": content,
"attachments": attachmentsHTML(ctx, comment.Attachments, comment.Content),
})
}

13
routers/repo/lfs.go
View file

@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) {
break
}
d, _ := ioutil.ReadAll(dataRc)
buf = charset.ToUTF8WithFallback(append(buf, d...))
buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
// Building code view blocks with line number on server side.
var fileContent string
if content, err := charset.ToUTF8WithErr(buf); err != nil {
log.Error("ToUTF8WithErr: %v", err)
fileContent = string(buf)
} else {
fileContent = content
}
fileContent, _ := ioutil.ReadAll(buf)
var output bytes.Buffer
lines := strings.Split(fileContent, "\n")
lines := strings.Split(string(fileContent), "\n")
//Remove blank line at the end of file
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]

19
routers/repo/milestone.go
View file

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
@ -84,7 +85,14 @@ func Milestones(ctx *context.Context) {
}
}
for _, m := range miles {
m.RenderedContent = string(markdown.Render([]byte(m.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, m.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
}
ctx.Data["Milestones"] = miles
@ -269,7 +277,14 @@ func MilestoneIssuesAndPulls(ctx *context.Context) {
return
}
milestone.RenderedContent = string(markdown.Render([]byte(milestone.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, milestone.Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.Data["Title"] = milestone.Name
ctx.Data["Milestone"] = milestone

19
routers/repo/projects.go
View file

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -77,7 +78,14 @@ func Projects(ctx *context.Context) {
}
for i := range projects {
projects[i].RenderedContent = string(markdown.Render([]byte(projects[i].Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, projects[i].Description)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
}
ctx.Data["Projects"] = projects
@ -311,7 +319,14 @@ func ViewProject(ctx *context.Context) {
}
ctx.Data["LinkedPRs"] = linkedPrsMap
project.RenderedContent = string(markdown.Render([]byte(project.Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, project.Description)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.Data["CanWriteProjects"] = ctx.Repo.Permission.CanWrite(models.UnitTypeProjects)
ctx.Data["Project"] = project

19
routers/repo/release.go
View file

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/upload"
@ -132,7 +133,14 @@ func releasesOrTags(ctx *context.Context, isTagList bool) {
ctx.ServerError("calReleaseNumCommitsBehind", err)
return
}
r.Note = markdown.RenderString(r.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())
r.Note, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, r.Note)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
}
ctx.Data["Releases"] = releases
@ -182,7 +190,14 @@ func SingleRelease(ctx *context.Context) {
ctx.ServerError("calReleaseNumCommitsBehind", err)
return
}
release.Note = markdown.RenderString(release.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())
release.Note, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeMetas(),
}, release.Note)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.Data["Releases"] = []*models.Release{release}
ctx.HTML(http.StatusOK, tplReleases)

51
routers/repo/view.go
View file

@ -324,13 +324,26 @@ func renderDirectory(ctx *context.Context, treeLink string) {
ctx.Data["IsTextFile"] = true
ctx.Data["FileSize"] = fileSize
} else {
d, _ := ioutil.ReadAll(dataRc)
buf = charset.ToUTF8WithFallback(append(buf, d...))
rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
if markupType := markup.Type(readmeFile.name); markupType != "" {
ctx.Data["IsMarkup"] = true
ctx.Data["MarkupType"] = string(markupType)
ctx.Data["FileContent"] = string(markup.Render(readmeFile.name, buf, readmeTreelink, ctx.Repo.Repository.ComposeDocumentMetas()))
var result strings.Builder
err := markup.Render(&markup.RenderContext{
Filename: readmeFile.name,
URLPrefix: readmeTreelink,
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
}, rd, &result)
if err != nil {
log.Error("Render failed: %v then fallback", err)
bs, _ := ioutil.ReadAll(rd)
ctx.Data["FileContent"] = strings.ReplaceAll(
gotemplate.HTMLEscapeString(string(bs)), "\n", `<br>`,
)
} else {
ctx.Data["FileContent"] = result.String()
}
} else {
ctx.Data["IsRenderedHTML"] = true
ctx.Data["FileContent"] = strings.ReplaceAll(
@ -481,21 +494,30 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
break
}
d, _ := ioutil.ReadAll(dataRc)
buf = charset.ToUTF8WithFallback(append(buf, d...))
rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
readmeExist := markup.IsReadmeFile(blob.Name())
ctx.Data["ReadmeExist"] = readmeExist
if markupType := markup.Type(blob.Name()); markupType != "" {
ctx.Data["IsMarkup"] = true
ctx.Data["MarkupType"] = markupType
ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas()))
var result strings.Builder
err := markup.Render(&markup.RenderContext{
Filename: blob.Name(),
URLPrefix: path.Dir(treeLink),
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
}, rd, &result)
if err != nil {
ctx.ServerError("Render", err)
return
}
ctx.Data["FileContent"] = result.String()
} else if readmeExist {
ctx.Data["IsRenderedHTML"] = true
ctx.Data["FileContent"] = strings.ReplaceAll(
gotemplate.HTMLEscapeString(string(buf)), "\n", `<br>`,
)
} else {
buf = charset.ToUTF8WithFallback(buf)
buf, _ := ioutil.ReadAll(rd)
lineNums := linesBytesCount(buf)
ctx.Data["NumLines"] = strconv.Itoa(lineNums)
ctx.Data["NumLinesSet"] = true
@ -532,11 +554,20 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
if markupType := markup.Type(blob.Name()); markupType != "" {
d, _ := ioutil.ReadAll(dataRc)
buf = append(buf, d...)
rd := io.MultiReader(bytes.NewReader(buf), dataRc)
ctx.Data["IsMarkup"] = true
ctx.Data["MarkupType"] = markupType
ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas()))
var result strings.Builder
err := markup.Render(&markup.RenderContext{
Filename: blob.Name(),
URLPrefix: path.Dir(treeLink),
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
}, rd, &result)
if err != nil {
ctx.ServerError("Render", err)
return
}
ctx.Data["FileContent"] = result.String()
}
}

31
routers/repo/wiki.go
View file

@ -6,6 +6,7 @@
package repo
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
@ -211,12 +212,34 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) {
return nil, nil
}
metas := ctx.Repo.Repository.ComposeDocumentMetas()
ctx.Data["content"] = markdown.RenderWiki(data, ctx.Repo.RepoLink, metas)
var rctx = &markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
IsWiki: true,
}
var buf strings.Builder
if err := markdown.Render(rctx, bytes.NewReader(data), &buf); err != nil {
ctx.ServerError("Render", err)
return nil, nil
}
ctx.Data["content"] = buf.String()
buf.Reset()
if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil {
ctx.ServerError("Render", err)
return nil, nil
}
ctx.Data["sidebarPresent"] = sidebarContent != nil
ctx.Data["sidebarContent"] = markdown.RenderWiki(sidebarContent, ctx.Repo.RepoLink, metas)
ctx.Data["sidebarContent"] = buf.String()
buf.Reset()
if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil {
ctx.ServerError("Render", err)
return nil, nil
}
ctx.Data["footerPresent"] = footerContent != nil
ctx.Data["footerContent"] = markdown.RenderWiki(footerContent, ctx.Repo.RepoLink, metas)
ctx.Data["footerContent"] = buf.String()
// get commit count - wiki revisions
commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename)

11
routers/user/home.go
View file

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/context"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -267,7 +268,15 @@ func Milestones(ctx *context.Context) {
continue
}
milestones[i].RenderedContent = string(markdown.Render([]byte(milestones[i].Content), milestones[i].Repo.Link(), milestones[i].Repo.ComposeMetas()))
milestones[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: milestones[i].Repo.Link(),
Metas: milestones[i].Repo.ComposeMetas(),
}, milestones[i].Content)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
if milestones[i].Repo.IsTimetrackerEnabled() {
err := milestones[i].LoadTotalTrackedTime()
if err != nil {

11
routers/user/profile.go
View file

@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -110,7 +111,15 @@ func Profile(ctx *context.Context) {
}
if len(ctxUser.Description) != 0 {
ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(ctxUser.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"}))
content, err := markdown.RenderString(&markup.RenderContext{
URLPrefix: ctx.Repo.RepoLink,
Metas: map[string]string{"mode": "document"},
}, ctxUser.Description)
if err != nil {
ctx.ServerError("RenderString", err)
return
}
ctx.Data["RenderedDescription"] = content
}
showPrivate := ctx.IsSigned && (ctx.User.IsAdmin || ctx.User.ID == ctxUser.ID)