finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor "Content" for file uploading (#25851)

Browse source 
Before: the concept "Content string" is used everywhere. It has some
problems:

1. Sometimes it means "base64 encoded content", sometimes it means "raw
binary content"
2. It doesn't work with large files, eg: uploading a 1G LFS file would
make Gitea process OOM

This PR does the refactoring: use "ContentReader" / "ContentBase64"
instead of "Content"

This PR is not breaking because the key in API JSON is still "content":
`` ContentBase64 string `json:"content"` ``
This commit is contained in:
wxiaoguang 2023-07-19 02:14:47 +08:00 • committed by GitHub
parent 265a28802a
commit 236c645bf1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 103 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

61
routers/api/v1/repo/file.go
View file

@ -408,6 +408,14 @@ func canReadFiles(r *context.Repository) bool {
return r.Permission.CanRead(unit.TypeCode)
}
func base64Reader(s string) (io.Reader, error) {
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return nil, err
}
return bytes.NewReader(b), nil
}
// ChangeFiles handles API call for modifying multiple files
func ChangeFiles(ctx *context.APIContext) {
// swagger:operation POST /repos/{owner}/{repo}/contents repository repoChangeFiles
@ -449,14 +457,19 @@ func ChangeFiles(ctx *context.APIContext) {
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
}
files := []*files_service.ChangeRepoFile{}
var files []*files_service.ChangeRepoFile
for _, file := range apiOpts.Files {
contentReader, err := base64Reader(file.ContentBase64)
if err != nil {
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
return
}
changeRepoFile := &files_service.ChangeRepoFile{
Operation: file.Operation,
TreePath: file.Path,
FromTreePath: file.FromPath,
Content: file.Content,
SHA: file.SHA,
Operation: file.Operation,
TreePath: file.Path,
FromTreePath: file.FromPath,
ContentReader: contentReader,
SHA: file.SHA,
}
files = append(files, changeRepoFile)
}
@ -544,12 +557,18 @@ func CreateFile(ctx *context.APIContext) {
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
}
contentReader, err := base64Reader(apiOpts.ContentBase64)
if err != nil {
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
return
}
opts := &files_service.ChangeRepoFilesOptions{
Files: []*files_service.ChangeRepoFile{
{
Operation: "create",
TreePath: ctx.Params("*"),
Content: apiOpts.Content,
Operation: "create",
TreePath: ctx.Params("*"),
ContentReader: contentReader,
},
},
Message: apiOpts.Message,
@ -636,14 +655,20 @@ func UpdateFile(ctx *context.APIContext) {
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
}
contentReader, err := base64Reader(apiOpts.ContentBase64)
if err != nil {
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
return
}
opts := &files_service.ChangeRepoFilesOptions{
Files: []*files_service.ChangeRepoFile{
{
Operation: "update",
Content: apiOpts.Content,
SHA: apiOpts.SHA,
FromTreePath: apiOpts.FromPath,
TreePath: ctx.Params("*"),
Operation: "update",
ContentReader: contentReader,
SHA: apiOpts.SHA,
FromTreePath: apiOpts.FromPath,
TreePath: ctx.Params("*"),
},
},
Message: apiOpts.Message,
@ -709,14 +734,6 @@ func createOrUpdateFiles(ctx *context.APIContext, opts *files_service.ChangeRepo
}
}
for _, file := range opts.Files {
content, err := base64.StdEncoding.DecodeString(file.Content)
if err != nil {
return nil, err
}
file.Content = string(content)
}
return files_service.ChangeRepoFiles(ctx, ctx.Repo.Repository, ctx.Doer, opts)
}

8
routers/web/repo/editor.go
View file

@ -284,10 +284,10 @@ func editFilePost(ctx *context.Context, form forms.EditRepoFileForm, isNewFile b
Message: message,
Files: []*files_service.ChangeRepoFile{
{
Operation: operation,
FromTreePath: ctx.Repo.TreePath,
TreePath: form.TreePath,
Content: strings.ReplaceAll(form.Content, "\r", ""),
Operation: operation,
FromTreePath: ctx.Repo.TreePath,
TreePath: form.TreePath,
ContentReader: strings.NewReader(strings.ReplaceAll(form.Content, "\r", "")),
},
},
Signoff: form.Signoff,