From 1d3240887c519a04c13bcd7e852c6d6ad1cb00b5 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Fri, 15 Mar 2024 13:44:42 +0100 Subject: [PATCH 01/18] Render inline file permalinks --- modules/markup/html.go | 267 +++++++++++++++++++++ modules/markup/html_test.go | 57 +++++ modules/markup/renderer.go | 4 + modules/markup/sanitizer.go | 17 ++ services/markup/processorhelper.go | 81 +++++++ web_src/css/index.css | 1 + web_src/css/markup/content.css | 3 +- web_src/css/markup/filepreview.css | 35 +++ web_src/css/repo/linebutton.css | 3 +- web_src/js/features/repo-unicode-escape.js | 4 +- 10 files changed, 468 insertions(+), 4 deletions(-) create mode 100644 web_src/css/markup/filepreview.css diff --git a/modules/markup/html.go b/modules/markup/html.go index b7291823b..2501f8062 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -10,10 +10,12 @@ import ( "path" "path/filepath" "regexp" + "strconv" "strings" "sync" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -61,6 +63,9 @@ var ( validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) + // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" + filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{7,64})/(\S+)#(L\d+(?:-L\d+)?)`) + // While this email regex is definitely not perfect and I'm sure you can come up // with edge cases, it is still accepted by the CommonMark specification, as // well as the HTML5 spec: @@ -171,6 +176,7 @@ type processor func(ctx *RenderContext, node *html.Node) var defaultProcessors = []processor{ fullIssuePatternProcessor, comparePatternProcessor, + filePreviewPatternProcessor, fullHashPatternProcessor, shortLinkProcessor, linkProcessor, @@ -1054,6 +1060,267 @@ func comparePatternProcessor(ctx *RenderContext, node *html.Node) { } } +func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { + return + } + if DefaultProcessorHelper.GetRepoFileContent == nil || DefaultProcessorHelper.GetLocale == nil { + return + } + + next := node.NextSibling + for node != nil && node != next { + m := filePreviewPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + // Ensure that every group (m[0]...m[9]) has a match + for i := 0; i < 10; i++ { + if m[i] == -1 { + return + } + } + + urlFull := node.Data[m[0]:m[1]] + + // Ensure that we only use links to local repositories + if !strings.HasPrefix(urlFull, setting.AppURL+setting.AppSubURL) { + return + } + + projPath := node.Data[m[2]:m[3]] + projPath = strings.TrimSuffix(projPath, "/") + + commitSha := node.Data[m[4]:m[5]] + filePath := node.Data[m[6]:m[7]] + hash := node.Data[m[8]:m[9]] + + start := m[0] + end := m[1] + + // If url ends in '.', it's very likely that it is not part of the + // actual url but used to finish a sentence. + if strings.HasSuffix(urlFull, ".") { + end-- + urlFull = urlFull[:len(urlFull)-1] + hash = hash[:len(hash)-1] + } + + projPathSegments := strings.Split(projPath, "/") + fileContent, err := DefaultProcessorHelper.GetRepoFileContent( + ctx.Ctx, + projPathSegments[len(projPathSegments)-2], + projPathSegments[len(projPathSegments)-1], + commitSha, filePath, + ) + if err != nil { + return + } + + lineSpecs := strings.Split(hash, "-") + lineCount := len(fileContent) + + var subTitle string + var lineOffset int + + if len(lineSpecs) == 1 { + line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + if line < 1 || line > lineCount { + return + } + + fileContent = fileContent[line-1 : line] + subTitle = "Line " + strconv.Itoa(line) + + lineOffset = line - 1 + } else { + startLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) + + if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { + return + } + + fileContent = fileContent[startLine-1 : endLine] + subTitle = "Lines " + strconv.Itoa(startLine) + " to " + strconv.Itoa(endLine) + + lineOffset = startLine - 1 + } + + table := &html.Node{ + Type: html.ElementNode, + Data: atom.Table.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview"}}, + } + tbody := &html.Node{ + Type: html.ElementNode, + Data: atom.Tbody.String(), + } + + locale, err := DefaultProcessorHelper.GetLocale(ctx.Ctx) + if err != nil { + log.Error("Unable to get locale. Error: %v", err) + return + } + + status := &charset.EscapeStatus{} + statuses := make([]*charset.EscapeStatus, len(fileContent)) + for i, line := range fileContent { + statuses[i], fileContent[i] = charset.EscapeControlHTML(line, locale, charset.FileviewContext) + status = status.Or(statuses[i]) + } + + for idx, code := range fileContent { + tr := &html.Node{ + Type: html.ElementNode, + Data: atom.Tr.String(), + } + + lineNum := strconv.Itoa(lineOffset + idx + 1) + + tdLinesnum := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "id", Val: "L" + lineNum}, + {Key: "class", Val: "lines-num"}, + }, + } + spanLinesNum := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{ + {Key: "id", Val: "L" + lineNum}, + {Key: "data-line-number", Val: lineNum}, + }, + } + tdLinesnum.AppendChild(spanLinesNum) + tr.AppendChild(tdLinesnum) + + if status.Escaped { + tdLinesEscape := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "lines-escape"}, + }, + } + + if statuses[idx].Escaped { + btnTitle := "" + if statuses[idx].HasInvisible { + btnTitle += locale.TrString("repo.invisible_runes_line") + " " + } + if statuses[idx].HasAmbiguous { + btnTitle += locale.TrString("repo.ambiguous_runes_line") + } + + escapeBtn := &html.Node{ + Type: html.ElementNode, + Data: atom.Button.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "toggle-escape-button btn interact-bg"}, + {Key: "title", Val: btnTitle}, + }, + } + tdLinesEscape.AppendChild(escapeBtn) + } + + tr.AppendChild(tdLinesEscape) + } + + tdCode := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "rel", Val: "L" + lineNum}, + {Key: "class", Val: "lines-code chroma"}, + }, + } + codeInner := &html.Node{ + Type: html.ElementNode, + Data: atom.Code.String(), + Attr: []html.Attribute{{Key: "class", Val: "code-inner"}}, + } + codeText := &html.Node{ + Type: html.RawNode, + Data: string(code), + } + codeInner.AppendChild(codeText) + tdCode.AppendChild(codeInner) + tr.AppendChild(tdCode) + + tbody.AppendChild(tr) + } + + table.AppendChild(tbody) + + twrapper := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "ui table"}}, + } + twrapper.AppendChild(table) + + header := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "header"}}, + } + afilepath := &html.Node{ + Type: html.ElementNode, + Data: atom.A.String(), + Attr: []html.Attribute{ + {Key: "href", Val: urlFull}, + {Key: "class", Val: "muted"}, + }, + } + afilepath.AppendChild(&html.Node{ + Type: html.TextNode, + Data: filePath, + }) + header.AppendChild(afilepath) + + psubtitle := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{{Key: "class", Val: "text small grey"}}, + } + psubtitle.AppendChild(&html.Node{ + Type: html.TextNode, + Data: subTitle + " in ", + }) + psubtitle.AppendChild(createLink(urlFull[m[0]:m[5]], commitSha[0:7], "text black")) + header.AppendChild(psubtitle) + + preview := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, + } + preview.AppendChild(header) + preview.AppendChild(twrapper) + + // Specialized version of replaceContent, so the parent paragraph element is not destroyed from our div + before := node.Data[:start] + after := node.Data[end:] + node.Data = before + nextSibling := node.NextSibling + node.Parent.InsertBefore(&html.Node{ + Type: html.RawNode, + Data: "

", + }, nextSibling) + node.Parent.InsertBefore(preview, nextSibling) + node.Parent.InsertBefore(&html.Node{ + Type: html.RawNode, + Data: "

" + after, + }, nextSibling) + + node = node.NextSibling + } +} + // emojiShortCodeProcessor for rendering text like :smile: into emoji func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { start := 0 diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 132955c01..652db13e5 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -5,6 +5,7 @@ package markup_test import ( "context" + "html/template" "io" "os" "strings" @@ -13,10 +14,12 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -673,3 +676,57 @@ func TestIssue18471(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "783b039...da951ce", res.String()) } + +func TestRender_FilePreview(t *testing.T) { + setting.AppURL = markup.TestAppURL + markup.Init(&markup.ProcessorHelper{ + GetRepoFileContent: func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) { + buf := []byte("A\nB\nC\nD\n") + return highlight.PlainText(buf), nil + }, + GetLocale: func(ctx context.Context) (translation.Locale, error) { + return translation.NewLocale("en-US"), nil + }, + }) + + sha := "b6dd6210eaebc915fd5be5579c58cce4da2e2579" + commitFilePreview := util.URLJoin(markup.TestRepoURL, "src", "commit", sha, "path", "to", "file.go") + "#L1-L2" + + test := func(input, expected string) { + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: ".md", + Metas: localMetas, + }, input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + test( + commitFilePreview, + `

`+ + `
`+ + `
`+ + `path/to/file.go`+ + ``+ + `Lines 1 to 2 in b6dd621`+ + ``+ + `
`+ + `
`+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + `
A`+"\n"+`
B`+"\n"+`
`+ + `
`+ + `
`+ + `

`, + ) +} diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 5a7adcc55..37d3fde58 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "html/template" "io" "net/url" "path/filepath" @@ -16,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "github.com/yuin/goldmark/ast" @@ -31,6 +33,8 @@ const ( type ProcessorHelper struct { IsUsernameMentionable func(ctx context.Context, username string) bool + GetRepoFileContent func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) + GetLocale func(ctx context.Context) (translation.Locale, error) ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute } diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index ffc33c3b8..73e17060a 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -120,6 +120,23 @@ func createDefaultPolicy() *bluemonday.Policy { // Allow 'color' and 'background-color' properties for the style attribute on text elements. policy.AllowStyles("color", "background-color").OnElements("span", "p") + // Allow classes for file preview links... + policy.AllowAttrs("class").Matching(regexp.MustCompile("^(lines-num|lines-code chroma)$")).OnElements("td") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^code-inner$")).OnElements("code") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^file-preview-box$")).OnElements("div") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ui table$")).OnElements("div") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^header$")).OnElements("div") + policy.AllowAttrs("data-line-number").Matching(regexp.MustCompile("^[0-9]+$")).OnElements("span") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^text small grey$")).OnElements("span") + policy.AllowAttrs("rel").Matching(regexp.MustCompile("^L[0-9]+$")).OnElements("td") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^file-preview*")).OnElements("table") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^lines-escape$")).OnElements("td") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^toggle-escape-button btn interact-bg$")).OnElements("button") + policy.AllowAttrs("title").OnElements("button") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ambiguous-code-point$")).OnElements("span") + policy.AllowAttrs("data-tooltip-content").OnElements("span") + policy.AllowAttrs("class").Matching(regexp.MustCompile("muted|(text black)")).OnElements("a") + // Allow generally safe attributes generalSafeAttrs := []string{ "abbr", "accept", "accept-charset", diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index a4378678a..134b1b515 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -5,10 +5,21 @@ package markup import ( "context" + "fmt" + "html/template" + "io" + "code.gitea.io/gitea/models/perm/access" + "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/highlight" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/translation" gitea_context "code.gitea.io/gitea/services/context" + file_service "code.gitea.io/gitea/services/repository/files" ) func ProcessorHelper() *markup.ProcessorHelper { @@ -29,5 +40,75 @@ func ProcessorHelper() *markup.ProcessorHelper { // when using gitea context (web context), use user's visibility and user's permission to check return user.IsUserVisibleToViewer(giteaCtx, mentionedUser, giteaCtx.Doer) }, + GetRepoFileContent: func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) { + repo, err := repo.GetRepositoryByOwnerAndName(ctx, ownerName, repoName) + if err != nil { + return nil, err + } + + var user *user.User + + giteaCtx, ok := ctx.(*gitea_context.Context) + if ok { + user = giteaCtx.Doer + } + + perms, err := access.GetUserRepoPermission(ctx, repo, user) + if err != nil { + return nil, err + } + if !perms.CanRead(unit.TypeCode) { + return nil, fmt.Errorf("cannot access repository code") + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + return nil, err + } + + commit, err := gitRepo.GetCommit(commitSha) + if err != nil { + return nil, err + } + + language, err := file_service.TryGetContentLanguage(gitRepo, commitSha, filePath) + if err != nil { + log.Error("Unable to get file language for %-v:%s. Error: %v", repo, filePath, err) + } + + blob, err := commit.GetBlobByPath(filePath) + if err != nil { + return nil, err + } + + dataRc, err := blob.DataAsync() + if err != nil { + return nil, err + } + defer dataRc.Close() + + buf, _ := io.ReadAll(dataRc) + + fileContent, _, err := highlight.File(blob.Name(), language, buf) + if err != nil { + log.Error("highlight.File failed, fallback to plain text: %v", err) + fileContent = highlight.PlainText(buf) + } + + return fileContent, nil + }, + GetLocale: func(ctx context.Context) (translation.Locale, error) { + giteaCtx, ok := ctx.(*gitea_context.Context) + if ok { + return giteaCtx.Locale, nil + } + + giteaBaseCtx, ok := ctx.(*gitea_context.Base) + if ok { + return giteaBaseCtx.Locale, nil + } + + return nil, fmt.Errorf("could not retrieve locale from context") + }, } } diff --git a/web_src/css/index.css b/web_src/css/index.css index ab925a4aa..8d2780ba4 100644 --- a/web_src/css/index.css +++ b/web_src/css/index.css @@ -30,6 +30,7 @@ @import "./markup/content.css"; @import "./markup/codecopy.css"; @import "./markup/asciicast.css"; +@import "./markup/filepreview.css"; @import "./chroma/base.css"; @import "./codemirror/base.css"; diff --git a/web_src/css/markup/content.css b/web_src/css/markup/content.css index 5eeef078a..430b4802d 100644 --- a/web_src/css/markup/content.css +++ b/web_src/css/markup/content.css @@ -451,7 +451,8 @@ text-decoration: inherit; } -.markup pre > code { +.markup pre > code, +.markup .file-preview code { padding: 0; margin: 0; font-size: 100%; diff --git a/web_src/css/markup/filepreview.css b/web_src/css/markup/filepreview.css new file mode 100644 index 000000000..69360e2a7 --- /dev/null +++ b/web_src/css/markup/filepreview.css @@ -0,0 +1,35 @@ +.markup table.file-preview { + margin-bottom: 0; +} + +.markup table.file-preview td { + padding: 0 10px !important; + border: none !important; +} + +.markup table.file-preview tr { + border-top: none; + background-color: inherit !important; +} + +.markup .file-preview-box { + margin-bottom: 16px; +} + +.markup .file-preview-box .header { + padding: .5rem; + padding-left: 1rem; + border: 1px solid var(--color-secondary); + border-bottom: none; + border-radius: 0.28571429rem 0.28571429rem 0 0; + background: var(--color-box-header); +} + +.markup .file-preview-box .header > a { + display: block; +} + +.markup .file-preview-box .table { + margin-top: 0; + border-radius: 0 0 0.28571429rem 0.28571429rem; +} diff --git a/web_src/css/repo/linebutton.css b/web_src/css/repo/linebutton.css index 1e5e51eac..7780d6a26 100644 --- a/web_src/css/repo/linebutton.css +++ b/web_src/css/repo/linebutton.css @@ -1,4 +1,5 @@ -.code-view .lines-num:hover { +.code-view .lines-num:hover, +.file-preview .lines-num:hover { color: var(--color-text-dark) !important; } diff --git a/web_src/js/features/repo-unicode-escape.js b/web_src/js/features/repo-unicode-escape.js index d87853200..9f0c74522 100644 --- a/web_src/js/features/repo-unicode-escape.js +++ b/web_src/js/features/repo-unicode-escape.js @@ -7,8 +7,8 @@ export function initUnicodeEscapeButton() { e.preventDefault(); - const fileContent = btn.closest('.file-content, .non-diff-file-content'); - const fileView = fileContent?.querySelectorAll('.file-code, .file-view'); + const fileContent = btn.closest('.file-content, .non-diff-file-content, .file-preview-box'); + const fileView = fileContent?.querySelectorAll('.file-code, .file-view, .file-preview'); if (btn.matches('.escape-button')) { for (const el of fileView) el.classList.add('unicode-escaped'); hideElem(btn); From 781a37fbe18c223763f51968862f1c8f61e7e260 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Fri, 15 Mar 2024 23:49:13 +0100 Subject: [PATCH 02/18] Check error in GetRepoFileContent for io.ReadAll --- services/markup/processorhelper.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index 134b1b515..ab6a66b36 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -87,7 +87,10 @@ func ProcessorHelper() *markup.ProcessorHelper { } defer dataRc.Close() - buf, _ := io.ReadAll(dataRc) + buf, err := io.ReadAll(dataRc) + if err != nil { + log.Error("failed to completly read blob for %-v:%s. Error: %v", repo, filePath, err) + } fileContent, _, err := highlight.File(blob.Name(), language, buf) if err != nil { From 8309f008c2721e313e1949ce42ed410e844c16e7 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Fri, 15 Mar 2024 23:52:38 +0100 Subject: [PATCH 03/18] Fix some code issues --- modules/markup/html.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 2501f8062..631c93fc3 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" @@ -64,7 +65,7 @@ var ( validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" - filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{7,64})/(\S+)#(L\d+(?:-L\d+)?)`) + filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){4,5})src/commit/([0-9a-f]{7,64})/(\S+)#(L\d+(?:-L\d+)?)`) // While this email regex is definitely not perfect and I'm sure you can come up // with edge cases, it is still accepted by the CommonMark specification, as @@ -1075,11 +1076,9 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { return } - // Ensure that every group (m[0]...m[9]) has a match - for i := 0; i < 10; i++ { - if m[i] == -1 { - return - } + // Ensure that every group has a match + if slices.Contains(m, -1) { + return } urlFull := node.Data[m[0]:m[1]] @@ -1089,8 +1088,7 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { return } - projPath := node.Data[m[2]:m[3]] - projPath = strings.TrimSuffix(projPath, "/") + projPath := strings.TrimSuffix(node.Data[m[2]:m[3]], "/") commitSha := node.Data[m[4]:m[5]] filePath := node.Data[m[6]:m[7]] From fae8d9f70d31704af91cbf37bcefcc4772830695 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Fri, 15 Mar 2024 23:54:07 +0100 Subject: [PATCH 04/18] Accept at minimum 4 chars for the commit sha --- modules/markup/html.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 631c93fc3..7fe7e5fc4 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -65,7 +65,7 @@ var ( validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" - filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){4,5})src/commit/([0-9a-f]{7,64})/(\S+)#(L\d+(?:-L\d+)?)`) + filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){4,5})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) // While this email regex is definitely not perfect and I'm sure you can come up // with edge cases, it is still accepted by the CommonMark specification, as From 6721cba75b4997448b618a4b00ef25f142924de0 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Sat, 16 Mar 2024 00:35:56 +0100 Subject: [PATCH 05/18] Fix filePreviewPattern --- modules/markup/html.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 7fe7e5fc4..d0d253073 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -65,7 +65,7 @@ var ( validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" - filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){4,5})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) + filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) // While this email regex is definitely not perfect and I'm sure you can come up // with edge cases, it is still accepted by the CommonMark specification, as From 562e5cdf324597882b7e6971be1b9a148bbc7839 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Sat, 16 Mar 2024 01:17:04 +0100 Subject: [PATCH 06/18] Get locales directly from context like the other code; add translations for subtitle --- modules/markup/html.go | 34 ++++++++++++++++++------------ modules/markup/html_test.go | 4 ---- modules/markup/renderer.go | 2 -- options/locale/locale_en-US.ini | 4 ++++ services/markup/processorhelper.go | 14 ------------ 5 files changed, 25 insertions(+), 33 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index d0d253073..1e83dad70 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -5,6 +5,7 @@ package markup import ( "bytes" + "html/template" "io" "net/url" "path" @@ -1065,7 +1066,7 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { if ctx.Metas == nil { return } - if DefaultProcessorHelper.GetRepoFileContent == nil || DefaultProcessorHelper.GetLocale == nil { + if DefaultProcessorHelper.GetRepoFileContent == nil { return } @@ -1119,9 +1120,17 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { lineSpecs := strings.Split(hash, "-") lineCount := len(fileContent) - var subTitle string + commitLinkBuffer := new(bytes.Buffer) + html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) + + var subTitle template.HTML var lineOffset int + locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale) + if !ok { + locale = translation.NewLocale("en-US") + } + if len(lineSpecs) == 1 { line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) if line < 1 || line > lineCount { @@ -1129,7 +1138,10 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { } fileContent = fileContent[line-1 : line] - subTitle = "Line " + strconv.Itoa(line) + subTitle = locale.Tr( + "markup.filepreview.line", line, + template.HTML(commitLinkBuffer.String()), + ) lineOffset = line - 1 } else { @@ -1141,7 +1153,10 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { } fileContent = fileContent[startLine-1 : endLine] - subTitle = "Lines " + strconv.Itoa(startLine) + " to " + strconv.Itoa(endLine) + subTitle = locale.Tr( + "markup.filepreview.lines", startLine, endLine, + template.HTML(commitLinkBuffer.String()), + ) lineOffset = startLine - 1 } @@ -1156,12 +1171,6 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { Data: atom.Tbody.String(), } - locale, err := DefaultProcessorHelper.GetLocale(ctx.Ctx) - if err != nil { - log.Error("Unable to get locale. Error: %v", err) - return - } - status := &charset.EscapeStatus{} statuses := make([]*charset.EscapeStatus, len(fileContent)) for i, line := range fileContent { @@ -1286,10 +1295,9 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { Attr: []html.Attribute{{Key: "class", Val: "text small grey"}}, } psubtitle.AppendChild(&html.Node{ - Type: html.TextNode, - Data: subTitle + " in ", + Type: html.RawNode, + Data: string(subTitle), }) - psubtitle.AppendChild(createLink(urlFull[m[0]:m[5]], commitSha[0:7], "text black")) header.AppendChild(psubtitle) preview := &html.Node{ diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 652db13e5..c43f00626 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -19,7 +19,6 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -684,9 +683,6 @@ func TestRender_FilePreview(t *testing.T) { buf := []byte("A\nB\nC\nD\n") return highlight.PlainText(buf), nil }, - GetLocale: func(ctx context.Context) (translation.Locale, error) { - return translation.NewLocale("en-US"), nil - }, }) sha := "b6dd6210eaebc915fd5be5579c58cce4da2e2579" diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 37d3fde58..b6d742e5c 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "github.com/yuin/goldmark/ast" @@ -34,7 +33,6 @@ const ( type ProcessorHelper struct { IsUsernameMentionable func(ctx context.Context, username string) bool GetRepoFileContent func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) - GetLocale func(ctx context.Context) (translation.Locale, error) ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a7f4de48a..ebc8db24c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3707,3 +3707,7 @@ normal_file = Normal file executable_file = Executable file symbolic_link = Symbolic link submodule = Submodule + +[markup] +filepreview.line = Line %[1]d in %[3]s +filepreview.lines = Lines %[1]d to %[2]d in %[3]s diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index ab6a66b36..df96f25ce 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" - "code.gitea.io/gitea/modules/translation" gitea_context "code.gitea.io/gitea/services/context" file_service "code.gitea.io/gitea/services/repository/files" ) @@ -100,18 +99,5 @@ func ProcessorHelper() *markup.ProcessorHelper { return fileContent, nil }, - GetLocale: func(ctx context.Context) (translation.Locale, error) { - giteaCtx, ok := ctx.(*gitea_context.Context) - if ok { - return giteaCtx.Locale, nil - } - - giteaBaseCtx, ok := ctx.(*gitea_context.Base) - if ok { - return giteaBaseCtx.Locale, nil - } - - return nil, fmt.Errorf("could not retrieve locale from context") - }, } } From d789d33229b3998bb33f1505d122504c8039f23d Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Sat, 16 Mar 2024 08:09:49 +0100 Subject: [PATCH 07/18] Split filePreviewPatternProcessor into a new type FilePreview and some functions to make code more maintainable --- modules/markup/file_preview.go | 269 +++++++++++++++++++++++++++++++++ modules/markup/html.go | 245 +----------------------------- 2 files changed, 276 insertions(+), 238 deletions(-) create mode 100644 modules/markup/file_preview.go diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go new file mode 100644 index 000000000..646bf8363 --- /dev/null +++ b/modules/markup/file_preview.go @@ -0,0 +1,269 @@ +package markup + +import ( + "bytes" + "html/template" + "regexp" + "slices" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +var ( + // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" + filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) +) + +type FilePreview struct { + fileContent []template.HTML + subTitle template.HTML + lineOffset int + urlFull string + filePath string + start int + end int +} + +func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Locale) *FilePreview { + preview := &FilePreview{} + + m := filePreviewPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return nil + } + + // Ensure that every group has a match + if slices.Contains(m, -1) { + return nil + } + + preview.urlFull = node.Data[m[0]:m[1]] + + // Ensure that we only use links to local repositories + if !strings.HasPrefix(preview.urlFull, setting.AppURL+setting.AppSubURL) { + return nil + } + + projPath := strings.TrimSuffix(node.Data[m[2]:m[3]], "/") + + commitSha := node.Data[m[4]:m[5]] + preview.filePath = node.Data[m[6]:m[7]] + hash := node.Data[m[8]:m[9]] + + preview.start = m[0] + preview.end = m[1] + + // If url ends in '.', it's very likely that it is not part of the + // actual url but used to finish a sentence. + if strings.HasSuffix(preview.urlFull, ".") { + preview.end-- + preview.urlFull = preview.urlFull[:len(preview.urlFull)-1] + hash = hash[:len(hash)-1] + } + + projPathSegments := strings.Split(projPath, "/") + fileContent, err := DefaultProcessorHelper.GetRepoFileContent( + ctx.Ctx, + projPathSegments[len(projPathSegments)-2], + projPathSegments[len(projPathSegments)-1], + commitSha, preview.filePath, + ) + if err != nil { + return nil + } + + lineSpecs := strings.Split(hash, "-") + lineCount := len(fileContent) + + commitLinkBuffer := new(bytes.Buffer) + html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) + + if len(lineSpecs) == 1 { + line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + if line < 1 || line > lineCount { + return nil + } + + preview.fileContent = fileContent[line-1 : line] + preview.subTitle = locale.Tr( + "markup.filepreview.line", line, + template.HTML(commitLinkBuffer.String()), + ) + + preview.lineOffset = line - 1 + } else { + startLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) + + if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { + return nil + } + + preview.fileContent = fileContent[startLine-1 : endLine] + preview.subTitle = locale.Tr( + "markup.filepreview.lines", startLine, endLine, + template.HTML(commitLinkBuffer.String()), + ) + + preview.lineOffset = startLine - 1 + } + + return preview +} + +func (p *FilePreview) CreateHtml(locale translation.Locale) *html.Node { + table := &html.Node{ + Type: html.ElementNode, + Data: atom.Table.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview"}}, + } + tbody := &html.Node{ + Type: html.ElementNode, + Data: atom.Tbody.String(), + } + + status := &charset.EscapeStatus{} + statuses := make([]*charset.EscapeStatus, len(p.fileContent)) + for i, line := range p.fileContent { + statuses[i], p.fileContent[i] = charset.EscapeControlHTML(line, locale, charset.FileviewContext) + status = status.Or(statuses[i]) + } + + for idx, code := range p.fileContent { + tr := &html.Node{ + Type: html.ElementNode, + Data: atom.Tr.String(), + } + + lineNum := strconv.Itoa(p.lineOffset + idx + 1) + + tdLinesnum := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "id", Val: "L" + lineNum}, + {Key: "class", Val: "lines-num"}, + }, + } + spanLinesNum := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{ + {Key: "id", Val: "L" + lineNum}, + {Key: "data-line-number", Val: lineNum}, + }, + } + tdLinesnum.AppendChild(spanLinesNum) + tr.AppendChild(tdLinesnum) + + if status.Escaped { + tdLinesEscape := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "lines-escape"}, + }, + } + + if statuses[idx].Escaped { + btnTitle := "" + if statuses[idx].HasInvisible { + btnTitle += locale.TrString("repo.invisible_runes_line") + " " + } + if statuses[idx].HasAmbiguous { + btnTitle += locale.TrString("repo.ambiguous_runes_line") + } + + escapeBtn := &html.Node{ + Type: html.ElementNode, + Data: atom.Button.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "toggle-escape-button btn interact-bg"}, + {Key: "title", Val: btnTitle}, + }, + } + tdLinesEscape.AppendChild(escapeBtn) + } + + tr.AppendChild(tdLinesEscape) + } + + tdCode := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "rel", Val: "L" + lineNum}, + {Key: "class", Val: "lines-code chroma"}, + }, + } + codeInner := &html.Node{ + Type: html.ElementNode, + Data: atom.Code.String(), + Attr: []html.Attribute{{Key: "class", Val: "code-inner"}}, + } + codeText := &html.Node{ + Type: html.RawNode, + Data: string(code), + } + codeInner.AppendChild(codeText) + tdCode.AppendChild(codeInner) + tr.AppendChild(tdCode) + + tbody.AppendChild(tr) + } + + table.AppendChild(tbody) + + twrapper := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "ui table"}}, + } + twrapper.AppendChild(table) + + header := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "header"}}, + } + afilepath := &html.Node{ + Type: html.ElementNode, + Data: atom.A.String(), + Attr: []html.Attribute{ + {Key: "href", Val: p.urlFull}, + {Key: "class", Val: "muted"}, + }, + } + afilepath.AppendChild(&html.Node{ + Type: html.TextNode, + Data: p.filePath, + }) + header.AppendChild(afilepath) + + psubtitle := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{{Key: "class", Val: "text small grey"}}, + } + psubtitle.AppendChild(&html.Node{ + Type: html.RawNode, + Data: string(p.subTitle), + }) + header.AppendChild(psubtitle) + + preview_node := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, + } + preview_node.AppendChild(header) + preview_node.AppendChild(twrapper) + + return preview_node +} diff --git a/modules/markup/html.go b/modules/markup/html.go index 1e83dad70..2e38c05f5 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -5,19 +5,15 @@ package markup import ( "bytes" - "html/template" "io" "net/url" "path" "path/filepath" "regexp" - "slices" - "strconv" "strings" "sync" "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -65,9 +61,6 @@ var ( validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) - // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" - filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) - // While this email regex is definitely not perfect and I'm sure you can come up // with edge cases, it is still accepted by the CommonMark specification, as // well as the HTML5 spec: @@ -1072,252 +1065,28 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { next := node.NextSibling for node != nil && node != next { - m := filePreviewPattern.FindStringSubmatchIndex(node.Data) - if m == nil { - return - } - - // Ensure that every group has a match - if slices.Contains(m, -1) { - return - } - - urlFull := node.Data[m[0]:m[1]] - - // Ensure that we only use links to local repositories - if !strings.HasPrefix(urlFull, setting.AppURL+setting.AppSubURL) { - return - } - - projPath := strings.TrimSuffix(node.Data[m[2]:m[3]], "/") - - commitSha := node.Data[m[4]:m[5]] - filePath := node.Data[m[6]:m[7]] - hash := node.Data[m[8]:m[9]] - - start := m[0] - end := m[1] - - // If url ends in '.', it's very likely that it is not part of the - // actual url but used to finish a sentence. - if strings.HasSuffix(urlFull, ".") { - end-- - urlFull = urlFull[:len(urlFull)-1] - hash = hash[:len(hash)-1] - } - - projPathSegments := strings.Split(projPath, "/") - fileContent, err := DefaultProcessorHelper.GetRepoFileContent( - ctx.Ctx, - projPathSegments[len(projPathSegments)-2], - projPathSegments[len(projPathSegments)-1], - commitSha, filePath, - ) - if err != nil { - return - } - - lineSpecs := strings.Split(hash, "-") - lineCount := len(fileContent) - - commitLinkBuffer := new(bytes.Buffer) - html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) - - var subTitle template.HTML - var lineOffset int - locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale) if !ok { locale = translation.NewLocale("en-US") } - if len(lineSpecs) == 1 { - line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) - if line < 1 || line > lineCount { - return - } - - fileContent = fileContent[line-1 : line] - subTitle = locale.Tr( - "markup.filepreview.line", line, - template.HTML(commitLinkBuffer.String()), - ) - - lineOffset = line - 1 - } else { - startLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) - endLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) - - if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { - return - } - - fileContent = fileContent[startLine-1 : endLine] - subTitle = locale.Tr( - "markup.filepreview.lines", startLine, endLine, - template.HTML(commitLinkBuffer.String()), - ) - - lineOffset = startLine - 1 + preview := NewFilePreview(ctx, node, locale) + if preview == nil { + return } - table := &html.Node{ - Type: html.ElementNode, - Data: atom.Table.String(), - Attr: []html.Attribute{{Key: "class", Val: "file-preview"}}, - } - tbody := &html.Node{ - Type: html.ElementNode, - Data: atom.Tbody.String(), - } - - status := &charset.EscapeStatus{} - statuses := make([]*charset.EscapeStatus, len(fileContent)) - for i, line := range fileContent { - statuses[i], fileContent[i] = charset.EscapeControlHTML(line, locale, charset.FileviewContext) - status = status.Or(statuses[i]) - } - - for idx, code := range fileContent { - tr := &html.Node{ - Type: html.ElementNode, - Data: atom.Tr.String(), - } - - lineNum := strconv.Itoa(lineOffset + idx + 1) - - tdLinesnum := &html.Node{ - Type: html.ElementNode, - Data: atom.Td.String(), - Attr: []html.Attribute{ - {Key: "id", Val: "L" + lineNum}, - {Key: "class", Val: "lines-num"}, - }, - } - spanLinesNum := &html.Node{ - Type: html.ElementNode, - Data: atom.Span.String(), - Attr: []html.Attribute{ - {Key: "id", Val: "L" + lineNum}, - {Key: "data-line-number", Val: lineNum}, - }, - } - tdLinesnum.AppendChild(spanLinesNum) - tr.AppendChild(tdLinesnum) - - if status.Escaped { - tdLinesEscape := &html.Node{ - Type: html.ElementNode, - Data: atom.Td.String(), - Attr: []html.Attribute{ - {Key: "class", Val: "lines-escape"}, - }, - } - - if statuses[idx].Escaped { - btnTitle := "" - if statuses[idx].HasInvisible { - btnTitle += locale.TrString("repo.invisible_runes_line") + " " - } - if statuses[idx].HasAmbiguous { - btnTitle += locale.TrString("repo.ambiguous_runes_line") - } - - escapeBtn := &html.Node{ - Type: html.ElementNode, - Data: atom.Button.String(), - Attr: []html.Attribute{ - {Key: "class", Val: "toggle-escape-button btn interact-bg"}, - {Key: "title", Val: btnTitle}, - }, - } - tdLinesEscape.AppendChild(escapeBtn) - } - - tr.AppendChild(tdLinesEscape) - } - - tdCode := &html.Node{ - Type: html.ElementNode, - Data: atom.Td.String(), - Attr: []html.Attribute{ - {Key: "rel", Val: "L" + lineNum}, - {Key: "class", Val: "lines-code chroma"}, - }, - } - codeInner := &html.Node{ - Type: html.ElementNode, - Data: atom.Code.String(), - Attr: []html.Attribute{{Key: "class", Val: "code-inner"}}, - } - codeText := &html.Node{ - Type: html.RawNode, - Data: string(code), - } - codeInner.AppendChild(codeText) - tdCode.AppendChild(codeInner) - tr.AppendChild(tdCode) - - tbody.AppendChild(tr) - } - - table.AppendChild(tbody) - - twrapper := &html.Node{ - Type: html.ElementNode, - Data: atom.Div.String(), - Attr: []html.Attribute{{Key: "class", Val: "ui table"}}, - } - twrapper.AppendChild(table) - - header := &html.Node{ - Type: html.ElementNode, - Data: atom.Div.String(), - Attr: []html.Attribute{{Key: "class", Val: "header"}}, - } - afilepath := &html.Node{ - Type: html.ElementNode, - Data: atom.A.String(), - Attr: []html.Attribute{ - {Key: "href", Val: urlFull}, - {Key: "class", Val: "muted"}, - }, - } - afilepath.AppendChild(&html.Node{ - Type: html.TextNode, - Data: filePath, - }) - header.AppendChild(afilepath) - - psubtitle := &html.Node{ - Type: html.ElementNode, - Data: atom.Span.String(), - Attr: []html.Attribute{{Key: "class", Val: "text small grey"}}, - } - psubtitle.AppendChild(&html.Node{ - Type: html.RawNode, - Data: string(subTitle), - }) - header.AppendChild(psubtitle) - - preview := &html.Node{ - Type: html.ElementNode, - Data: atom.Div.String(), - Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, - } - preview.AppendChild(header) - preview.AppendChild(twrapper) + preview_node := preview.CreateHtml(locale) // Specialized version of replaceContent, so the parent paragraph element is not destroyed from our div - before := node.Data[:start] - after := node.Data[end:] + before := node.Data[:preview.start] + after := node.Data[preview.end:] node.Data = before nextSibling := node.NextSibling node.Parent.InsertBefore(&html.Node{ Type: html.RawNode, Data: "

", }, nextSibling) - node.Parent.InsertBefore(preview, nextSibling) + node.Parent.InsertBefore(preview_node, nextSibling) node.Parent.InsertBefore(&html.Node{ Type: html.RawNode, Data: "

" + after, From 8218e80bfc3a1f9ba02ce60f1acafdc0e57c5ae0 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Sat, 16 Mar 2024 08:18:47 +0100 Subject: [PATCH 08/18] Fix linting issues --- modules/markup/file_preview.go | 22 ++++++++++++---------- modules/markup/html.go | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 646bf8363..be788aae4 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -9,16 +9,15 @@ import ( "strings" "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" "golang.org/x/net/html" "golang.org/x/net/html/atom" ) -var ( - // filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" - filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) -) +// filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" +var filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) type FilePreview struct { fileContent []template.HTML @@ -82,7 +81,10 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca lineCount := len(fileContent) commitLinkBuffer := new(bytes.Buffer) - html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) + err = html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) + if err != nil { + log.Error("failed to render commitLink: %v", err) + } if len(lineSpecs) == 1 { line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) @@ -117,7 +119,7 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca return preview } -func (p *FilePreview) CreateHtml(locale translation.Locale) *html.Node { +func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { table := &html.Node{ Type: html.ElementNode, Data: atom.Table.String(), @@ -257,13 +259,13 @@ func (p *FilePreview) CreateHtml(locale translation.Locale) *html.Node { }) header.AppendChild(psubtitle) - preview_node := &html.Node{ + node := &html.Node{ Type: html.ElementNode, Data: atom.Div.String(), Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, } - preview_node.AppendChild(header) - preview_node.AppendChild(twrapper) + node.AppendChild(header) + node.AppendChild(twrapper) - return preview_node + return node } diff --git a/modules/markup/html.go b/modules/markup/html.go index 2e38c05f5..9a04e02fb 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -1075,7 +1075,7 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { return } - preview_node := preview.CreateHtml(locale) + previewNode := preview.CreateHTML(locale) // Specialized version of replaceContent, so the parent paragraph element is not destroyed from our div before := node.Data[:preview.start] @@ -1086,7 +1086,7 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { Type: html.RawNode, Data: "

", }, nextSibling) - node.Parent.InsertBefore(preview_node, nextSibling) + node.Parent.InsertBefore(previewNode, nextSibling) node.Parent.InsertBefore(&html.Node{ Type: html.RawNode, Data: "

" + after, From 10bca456a9140519e95559aa7bac2221e1156c5b Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 18 Mar 2024 06:19:27 +0100 Subject: [PATCH 09/18] Remove `rel` and `id` attributes that only add the linenumber to elements --- modules/markup/file_preview.go | 3 --- modules/markup/sanitizer.go | 1 - 2 files changed, 4 deletions(-) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index be788aae4..167bbd199 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -149,7 +149,6 @@ func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { Type: html.ElementNode, Data: atom.Td.String(), Attr: []html.Attribute{ - {Key: "id", Val: "L" + lineNum}, {Key: "class", Val: "lines-num"}, }, } @@ -157,7 +156,6 @@ func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { Type: html.ElementNode, Data: atom.Span.String(), Attr: []html.Attribute{ - {Key: "id", Val: "L" + lineNum}, {Key: "data-line-number", Val: lineNum}, }, } @@ -200,7 +198,6 @@ func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { Type: html.ElementNode, Data: atom.Td.String(), Attr: []html.Attribute{ - {Key: "rel", Val: "L" + lineNum}, {Key: "class", Val: "lines-code chroma"}, }, } diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index 73e17060a..c37027b84 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -128,7 +128,6 @@ func createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("class").Matching(regexp.MustCompile("^header$")).OnElements("div") policy.AllowAttrs("data-line-number").Matching(regexp.MustCompile("^[0-9]+$")).OnElements("span") policy.AllowAttrs("class").Matching(regexp.MustCompile("^text small grey$")).OnElements("span") - policy.AllowAttrs("rel").Matching(regexp.MustCompile("^L[0-9]+$")).OnElements("td") policy.AllowAttrs("class").Matching(regexp.MustCompile("^file-preview*")).OnElements("table") policy.AllowAttrs("class").Matching(regexp.MustCompile("^lines-escape$")).OnElements("td") policy.AllowAttrs("class").Matching(regexp.MustCompile("^toggle-escape-button btn interact-bg$")).OnElements("button") From db6f6281fcf568ae8e35330a4a93c9be1cb46efd Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 18 Mar 2024 06:21:35 +0100 Subject: [PATCH 10/18] Add copyright & license header to file_preview.go --- modules/markup/file_preview.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 167bbd199..377809529 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -1,3 +1,6 @@ +// Copyright The Forgejo Authors. +// SPDX-License-Identifier: MIT + package markup import ( From ed8e8a792e75b930074cd3cf1bab580a09ff8485 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 18 Mar 2024 06:23:12 +0100 Subject: [PATCH 11/18] Run make fmt --- modules/markup/file_preview.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 377809529..2702cb7ce 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "golang.org/x/net/html" "golang.org/x/net/html/atom" ) From d6428f92ce7ce67d127cbd5bb4977aa92abf071c Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 25 Mar 2024 14:33:30 +0100 Subject: [PATCH 12/18] Fix typo in language files --- options/locale/locale_en-US.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ebc8db24c..efaf8b72c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3709,5 +3709,5 @@ symbolic_link = Symbolic link submodule = Submodule [markup] -filepreview.line = Line %[1]d in %[3]s +filepreview.line = Line %[1]d in %[2]s filepreview.lines = Lines %[1]d to %[2]d in %[3]s From 069d87b80f909e91626249afbb240a1df339a8fd Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 25 Mar 2024 14:33:54 +0100 Subject: [PATCH 13/18] Remove unneeded case for a trailing dot --- modules/markup/file_preview.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 2702cb7ce..3e76dcb8a 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -62,14 +62,6 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca preview.start = m[0] preview.end = m[1] - // If url ends in '.', it's very likely that it is not part of the - // actual url but used to finish a sentence. - if strings.HasSuffix(preview.urlFull, ".") { - preview.end-- - preview.urlFull = preview.urlFull[:len(preview.urlFull)-1] - hash = hash[:len(hash)-1] - } - projPathSegments := strings.Split(projPath, "/") fileContent, err := DefaultProcessorHelper.GetRepoFileContent( ctx.Ctx, From 2b6546adc954d450a9c6befccd407ce2ca1636a0 Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Mon, 25 Mar 2024 16:05:01 +0100 Subject: [PATCH 14/18] Add setting to restrict count of lines being displayed & only highlight those lines --- custom/conf/app.example.ini | 2 + modules/markup/file_preview.go | 103 ++++++++++++++++++++++++----- modules/markup/html.go | 2 +- modules/markup/renderer.go | 4 +- modules/markup/sanitizer.go | 1 + modules/setting/markup.go | 2 + options/locale/locale_en-US.ini | 1 + services/markup/processorhelper.go | 24 ++++--- web_src/css/markup/filepreview.css | 6 ++ 9 files changed, 117 insertions(+), 28 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b3896bc31..91f86da5f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2338,6 +2338,8 @@ LEVEL = Info ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Set the maximum number of characters in a mermaid source. (Set to -1 to disable limits) ;MERMAID_MAX_SOURCE_CHARACTERS = 5000 +;; Set the maximum number of lines allowed for a filepreview. (Set to -1 to disable limits; set to 0 to disable the feature) +;FILEPREVIEW_MAX_LINES = 50 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 3e76dcb8a..32683c317 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -4,6 +4,7 @@ package markup import ( + "bufio" "bytes" "html/template" "regexp" @@ -12,6 +13,7 @@ import ( "strings" "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -31,9 +33,15 @@ type FilePreview struct { filePath string start int end int + isTruncated bool } func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Locale) *FilePreview { + if (setting.FilePreviewMaxLines == 0) { + // Feature is disabled + return nil + } + preview := &FilePreview{} m := filePreviewPattern.FindStringSubmatchIndex(node.Data) @@ -63,18 +71,20 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca preview.end = m[1] projPathSegments := strings.Split(projPath, "/") - fileContent, err := DefaultProcessorHelper.GetRepoFileContent( + var language string + fileBlob, err := DefaultProcessorHelper.GetRepoFileBlob( ctx.Ctx, projPathSegments[len(projPathSegments)-2], projPathSegments[len(projPathSegments)-1], commitSha, preview.filePath, + &language, ) if err != nil { return nil } lineSpecs := strings.Split(hash, "-") - lineCount := len(fileContent) + // lineCount := len(fileContent) commitLinkBuffer := new(bytes.Buffer) err = html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) @@ -82,28 +92,31 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca log.Error("failed to render commitLink: %v", err) } - if len(lineSpecs) == 1 { - line, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) - if line < 1 || line > lineCount { - return nil - } + var startLine, endLine int - preview.fileContent = fileContent[line-1 : line] + if len(lineSpecs) == 1 { + startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine = startLine + // if line < 1 || line > lineCount { + // return nil + // } + + // preview.fileContent = fileContent[line-1 : line] preview.subTitle = locale.Tr( - "markup.filepreview.line", line, + "markup.filepreview.line", startLine, template.HTML(commitLinkBuffer.String()), ) - preview.lineOffset = line - 1 + preview.lineOffset = startLine - 1 } else { - startLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) - endLine, _ := strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) + startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) - if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { - return nil - } + // if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { + // return nil + // } - preview.fileContent = fileContent[startLine-1 : endLine] + // preview.fileContent = fileContent[startLine-1 : endLine] preview.subTitle = locale.Tr( "markup.filepreview.lines", startLine, endLine, template.HTML(commitLinkBuffer.String()), @@ -112,6 +125,50 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca preview.lineOffset = startLine - 1 } + lineCount := endLine - (startLine-1) + if startLine < 1 || endLine < 1 || lineCount < 1 { + return nil + } + + if setting.FilePreviewMaxLines > 0 && lineCount > setting.FilePreviewMaxLines { + preview.isTruncated = true + lineCount = setting.FilePreviewMaxLines + } + + dataRc, err := fileBlob.DataAsync() + if err != nil { + return nil + } + defer dataRc.Close() + + reader := bufio.NewReader(dataRc) + + // skip all lines until we find our startLine + for i := 1; i < startLine; i++ { + _, err := reader.ReadBytes('\n') + if err != nil { + return nil + } + } + + // capture the lines we're interested in + lineBuffer := new(bytes.Buffer) + for i := 0; i < lineCount; i++ { + buf, err := reader.ReadBytes('\n') + if err != nil { + break; + } + lineBuffer.Write(buf) + } + + // highlight the file... + fileContent, _, err := highlight.File(fileBlob.Name(), language, lineBuffer.Bytes()) + if err != nil { + log.Error("highlight.File failed, fallback to plain text: %v", err) + fileContent = highlight.PlainText(lineBuffer.Bytes()) + } + preview.fileContent = fileContent + return preview } @@ -258,6 +315,20 @@ func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, } node.AppendChild(header) + + if (p.isTruncated) { + warning := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "ui warning message tw-text-left"}}, + } + warning.AppendChild(&html.Node{ + Type: html.TextNode, + Data: locale.TrString("markup.filepreview.truncated"), + }) + node.AppendChild(warning) + } + node.AppendChild(twrapper) return node diff --git a/modules/markup/html.go b/modules/markup/html.go index 9a04e02fb..4c74a81ba 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -1059,7 +1059,7 @@ func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { if ctx.Metas == nil { return } - if DefaultProcessorHelper.GetRepoFileContent == nil { + if DefaultProcessorHelper.GetRepoFileBlob == nil { return } diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index b6d742e5c..b08c9eb23 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -8,7 +8,7 @@ import ( "context" "errors" "fmt" - "html/template" + // "html/template" "io" "net/url" "path/filepath" @@ -32,7 +32,7 @@ const ( type ProcessorHelper struct { IsUsernameMentionable func(ctx context.Context, username string) bool - GetRepoFileContent func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) + GetRepoFileBlob func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute } diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index c37027b84..1048f0e37 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -135,6 +135,7 @@ func createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("class").Matching(regexp.MustCompile("^ambiguous-code-point$")).OnElements("span") policy.AllowAttrs("data-tooltip-content").OnElements("span") policy.AllowAttrs("class").Matching(regexp.MustCompile("muted|(text black)")).OnElements("a") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ui warning message tw-text-left$")).OnElements("div") // Allow generally safe attributes generalSafeAttrs := []string{ diff --git a/modules/setting/markup.go b/modules/setting/markup.go index 6c2246342..e893c1c2f 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -15,6 +15,7 @@ var ( ExternalMarkupRenderers []*MarkupRenderer ExternalSanitizerRules []MarkupSanitizerRule MermaidMaxSourceCharacters int + FilePreviewMaxLines int ) const ( @@ -62,6 +63,7 @@ func loadMarkupFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "markdown", &Markdown) MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000) + FilePreviewMaxLines = rootCfg.Section("markup").Key("FILEPREVIEW_MAX_LINES").MustInt(50) ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10) ExternalSanitizerRules = make([]MarkupSanitizerRule, 0, 10) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index efaf8b72c..8533cf065 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3711,3 +3711,4 @@ submodule = Submodule [markup] filepreview.line = Line %[1]d in %[2]s filepreview.lines = Lines %[1]d to %[2]d in %[3]s +filepreview.truncated = Preview has been truncated diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index df96f25ce..98a7824a6 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -6,15 +6,17 @@ package markup import ( "context" "fmt" - "html/template" - "io" + + // "html/template" + // "io" "code.gitea.io/gitea/models/perm/access" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/highlight" + // "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" gitea_context "code.gitea.io/gitea/services/context" @@ -39,7 +41,7 @@ func ProcessorHelper() *markup.ProcessorHelper { // when using gitea context (web context), use user's visibility and user's permission to check return user.IsUserVisibleToViewer(giteaCtx, mentionedUser, giteaCtx.Doer) }, - GetRepoFileContent: func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) { + GetRepoFileBlob: func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) { repo, err := repo.GetRepositoryByOwnerAndName(ctx, ownerName, repoName) if err != nil { return nil, err @@ -70,9 +72,11 @@ func ProcessorHelper() *markup.ProcessorHelper { return nil, err } - language, err := file_service.TryGetContentLanguage(gitRepo, commitSha, filePath) - if err != nil { - log.Error("Unable to get file language for %-v:%s. Error: %v", repo, filePath, err) + if language != nil { + *language, err = file_service.TryGetContentLanguage(gitRepo, commitSha, filePath) + if err != nil { + log.Error("Unable to get file language for %-v:%s. Error: %v", repo, filePath, err) + } } blob, err := commit.GetBlobByPath(filePath) @@ -80,7 +84,9 @@ func ProcessorHelper() *markup.ProcessorHelper { return nil, err } - dataRc, err := blob.DataAsync() + return blob, nil + + /*dataRc, err := blob.DataAsync() if err != nil { return nil, err } @@ -97,7 +103,7 @@ func ProcessorHelper() *markup.ProcessorHelper { fileContent = highlight.PlainText(buf) } - return fileContent, nil + return fileContent, nil*/ }, } } diff --git a/web_src/css/markup/filepreview.css b/web_src/css/markup/filepreview.css index 69360e2a7..d2ec16ea8 100644 --- a/web_src/css/markup/filepreview.css +++ b/web_src/css/markup/filepreview.css @@ -25,6 +25,12 @@ background: var(--color-box-header); } +.markup .file-preview-box .warning { + border-radius: 0; + margin: 0; + padding: .5rem .5rem .5rem 1rem; +} + .markup .file-preview-box .header > a { display: block; } From 4c7cb0a5d20e8973b03e35d91119cf917eed125e Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Wed, 27 Mar 2024 18:25:37 +0100 Subject: [PATCH 15/18] Close git.Repository when GetRepoFileBlob returns --- services/markup/processorhelper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index 98a7824a6..7466e962d 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -66,6 +66,7 @@ func ProcessorHelper() *markup.ProcessorHelper { if err != nil { return nil, err } + defer gitRepo.Close() commit, err := gitRepo.GetCommit(commitSha) if err != nil { From 7e0014dd1391e123d95f2537c3b2165fef7122ef Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Wed, 27 Mar 2024 18:36:12 +0100 Subject: [PATCH 16/18] Fix formating & remove commented out code --- modules/markup/file_preview.go | 20 ++++---------------- modules/markup/renderer.go | 2 +- services/markup/processorhelper.go | 19 ------------------- 3 files changed, 5 insertions(+), 36 deletions(-) diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 32683c317..95c94e0c1 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -37,7 +37,7 @@ type FilePreview struct { } func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Locale) *FilePreview { - if (setting.FilePreviewMaxLines == 0) { + if setting.FilePreviewMaxLines == 0 { // Feature is disabled return nil } @@ -84,7 +84,6 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca } lineSpecs := strings.Split(hash, "-") - // lineCount := len(fileContent) commitLinkBuffer := new(bytes.Buffer) err = html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitSha[0:7], "text black")) @@ -97,11 +96,6 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca if len(lineSpecs) == 1 { startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) endLine = startLine - // if line < 1 || line > lineCount { - // return nil - // } - - // preview.fileContent = fileContent[line-1 : line] preview.subTitle = locale.Tr( "markup.filepreview.line", startLine, template.HTML(commitLinkBuffer.String()), @@ -111,12 +105,6 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca } else { startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) endLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) - - // if startLine < 1 || endLine < 1 || startLine > lineCount || endLine > lineCount || endLine < startLine { - // return nil - // } - - // preview.fileContent = fileContent[startLine-1 : endLine] preview.subTitle = locale.Tr( "markup.filepreview.lines", startLine, endLine, template.HTML(commitLinkBuffer.String()), @@ -125,7 +113,7 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca preview.lineOffset = startLine - 1 } - lineCount := endLine - (startLine-1) + lineCount := endLine - (startLine - 1) if startLine < 1 || endLine < 1 || lineCount < 1 { return nil } @@ -156,7 +144,7 @@ func NewFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca for i := 0; i < lineCount; i++ { buf, err := reader.ReadBytes('\n') if err != nil { - break; + break } lineBuffer.Write(buf) } @@ -316,7 +304,7 @@ func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { } node.AppendChild(header) - if (p.isTruncated) { + if p.isTruncated { warning := &html.Node{ Type: html.ElementNode, Data: atom.Div.String(), diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index b08c9eb23..163cd5d68 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -32,7 +32,7 @@ const ( type ProcessorHelper struct { IsUsernameMentionable func(ctx context.Context, username string) bool - GetRepoFileBlob func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) + GetRepoFileBlob func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute } diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index 7466e962d..ac751d0e6 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -86,25 +86,6 @@ func ProcessorHelper() *markup.ProcessorHelper { } return blob, nil - - /*dataRc, err := blob.DataAsync() - if err != nil { - return nil, err - } - defer dataRc.Close() - - buf, err := io.ReadAll(dataRc) - if err != nil { - log.Error("failed to completly read blob for %-v:%s. Error: %v", repo, filePath, err) - } - - fileContent, _, err := highlight.File(blob.Name(), language, buf) - if err != nil { - log.Error("highlight.File failed, fallback to plain text: %v", err) - fileContent = highlight.PlainText(buf) - } - - return fileContent, nil*/ }, } } From 16a8658878a2656cb131453b728b65a89271f11f Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Thu, 28 Mar 2024 04:20:13 +0100 Subject: [PATCH 17/18] Update test --- modules/markup/html_test.go | 40 ++++++++++++------ .../markup/tests/repo/repo1_filepreview/HEAD | 1 + .../tests/repo/repo1_filepreview/config | 6 +++ .../tests/repo/repo1_filepreview/description | 1 + .../tests/repo/repo1_filepreview/info/exclude | 6 +++ .../19/0d9492934af498c3f669d6a2431dc5459e5b20 | Bin 0 -> 120 bytes .../4b/825dc642cb6eb9a060e54bf8d69288fbee4904 | Bin 0 -> 15 bytes .../83/57a737d04385bb7f2ab59ff184be94756e7972 | Bin 0 -> 44 bytes .../84/22d40f12717e1ebd5cef2449f6c09d1f775969 | Bin 0 -> 23 bytes .../d4/490327def9658be036d6a52c4417d84e74dd4c | Bin 0 -> 46 bytes .../ee/2b1253d9cf407796e2e724926cbe3a974b214d | 1 + .../repo/repo1_filepreview/refs/heads/master | 1 + 12 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 modules/markup/tests/repo/repo1_filepreview/HEAD create mode 100644 modules/markup/tests/repo/repo1_filepreview/config create mode 100644 modules/markup/tests/repo/repo1_filepreview/description create mode 100644 modules/markup/tests/repo/repo1_filepreview/info/exclude create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/83/57a737d04385bb7f2ab59ff184be94756e7972 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/84/22d40f12717e1ebd5cef2449f6c09d1f775969 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/d4/490327def9658be036d6a52c4417d84e74dd4c create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d create mode 100644 modules/markup/tests/repo/repo1_filepreview/refs/heads/master diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index c43f00626..3583894ba 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -5,7 +5,6 @@ package markup_test import ( "context" - "html/template" "io" "os" "strings" @@ -14,14 +13,15 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var localMetas = map[string]string{ @@ -677,16 +677,30 @@ func TestIssue18471(t *testing.T) { } func TestRender_FilePreview(t *testing.T) { + setting.StaticRootPath = "../../" + setting.Names = []string{"english"} + setting.Langs = []string{"en-US"} + translation.InitLocales(context.Background()) + setting.AppURL = markup.TestAppURL markup.Init(&markup.ProcessorHelper{ - GetRepoFileContent: func(ctx context.Context, ownerName, repoName, commitSha, filePath string) ([]template.HTML, error) { - buf := []byte("A\nB\nC\nD\n") - return highlight.PlainText(buf), nil + GetRepoFileBlob: func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) { + gitRepo, err := git.OpenRepository(git.DefaultContext, "./tests/repo/repo1_filepreview") + require.NoError(t, err) + defer gitRepo.Close() + + commit, err := gitRepo.GetCommit("HEAD") + require.NoError(t, err) + + blob, err := commit.GetBlobByPath("path/to/file.go") + require.NoError(t, err) + + return blob, nil }, }) - sha := "b6dd6210eaebc915fd5be5579c58cce4da2e2579" - commitFilePreview := util.URLJoin(markup.TestRepoURL, "src", "commit", sha, "path", "to", "file.go") + "#L1-L2" + sha := "190d9492934af498c3f669d6a2431dc5459e5b20" + commitFilePreview := util.URLJoin(markup.TestRepoURL, "src", "commit", sha, "path", "to", "file.go") + "#L2-L3" test := func(input, expected string) { buffer, err := markup.RenderString(&markup.RenderContext{ @@ -703,21 +717,21 @@ func TestRender_FilePreview(t *testing.T) { `

`+ `
`+ `
`+ - `path/to/file.go`+ + `path/to/file.go`+ ``+ - `Lines 1 to 2 in b6dd621`+ + `Lines 2 to 3 in 190d949`+ ``+ `
`+ `
`+ ``+ ``+ ``+ - ``+ - ``+ + ``+ + ``+ ``+ ``+ - ``+ - ``+ + ``+ + ``+ ``+ ``+ `
A`+"\n"+`B`+"\n"+`
B`+"\n"+`C`+"\n"+`
`+ diff --git a/modules/markup/tests/repo/repo1_filepreview/HEAD b/modules/markup/tests/repo/repo1_filepreview/HEAD new file mode 100644 index 000000000..cb089cd89 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/modules/markup/tests/repo/repo1_filepreview/config b/modules/markup/tests/repo/repo1_filepreview/config new file mode 100644 index 000000000..42cc799c8 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/config @@ -0,0 +1,6 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true +[remote "origin"] + url = /home/mai/projects/codeark/forgejo/forgejo/modules/markup/tests/repo/repo1_filepreview/../../__test_repo diff --git a/modules/markup/tests/repo/repo1_filepreview/description b/modules/markup/tests/repo/repo1_filepreview/description new file mode 100644 index 000000000..498b267a8 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/modules/markup/tests/repo/repo1_filepreview/info/exclude b/modules/markup/tests/repo/repo1_filepreview/info/exclude new file mode 100644 index 000000000..a5196d1be --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 b/modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 new file mode 100644 index 0000000000000000000000000000000000000000..161d0bafc6731f5fe0b3b3c29ffe5463b056e840 GIT binary patch literal 120 zcmV-;0Ehp00hNtO4#F@D06FIsz9S(!b&^)95RZTdgxH8kluC)q`&oX#X-+d!)@7*% z6w=O`DhTt0gHNKjDTeW?I7Ep#_`*y{M%Kh4TwLDlzBagYZ3Of7#i>`*Lw&yTqskE| a(I?AD9`;CxuKZr6|5@&=-P{|T!ZCX0g*&(a literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 new file mode 100644 index 0000000000000000000000000000000000000000..adf64119a33d7621aeeaa505d30adb58afaa5559 GIT binary patch literal 15 Wcmb)%hIiUR!8gx4luvu~TxC+uKC9{8ioO8p9WH2!R0)>Lak_?9C@a5(goLhI-Yi*tXv1Q+s(!9zd00ff{ EldH%Sg8%>k literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d b/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d new file mode 100644 index 000000000..e13ca647d --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d @@ -0,0 +1 @@ +x+)JMU06e040031QHËÌIÕKÏghQºÂ/TX'·7潊ç·såË#3‹ô \ No newline at end of file diff --git a/modules/markup/tests/repo/repo1_filepreview/refs/heads/master b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master new file mode 100644 index 000000000..49c348b41 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master @@ -0,0 +1 @@ +190d9492934af498c3f669d6a2431dc5459e5b20 From 6e98bacbbd3c089b2ccfa725c58184f4dfe5e7fe Mon Sep 17 00:00:00 2001 From: Mai-Lapyst Date: Thu, 28 Mar 2024 05:42:25 +0100 Subject: [PATCH 18/18] Format code --- modules/markup/html_test.go | 2 +- modules/markup/renderer.go | 1 - services/markup/processorhelper.go | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 3583894ba..1ecf519f4 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -688,7 +688,7 @@ func TestRender_FilePreview(t *testing.T) { gitRepo, err := git.OpenRepository(git.DefaultContext, "./tests/repo/repo1_filepreview") require.NoError(t, err) defer gitRepo.Close() - + commit, err := gitRepo.GetCommit("HEAD") require.NoError(t, err) diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 163cd5d68..6781d2e55 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - // "html/template" "io" "net/url" "path/filepath" diff --git a/services/markup/processorhelper.go b/services/markup/processorhelper.go index ac751d0e6..40bf1d65d 100644 --- a/services/markup/processorhelper.go +++ b/services/markup/processorhelper.go @@ -7,16 +7,12 @@ import ( "context" "fmt" - // "html/template" - // "io" - "code.gitea.io/gitea/models/perm/access" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" - // "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" gitea_context "code.gitea.io/gitea/services/context"