finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Avatars and Repo avatars support storing in minio (#12516)

Browse source 
* Avatar support minio

* Support repo avatar minio storage

* Add missing migration

* Fix bug

* Fix test

* Add test for minio store type on avatars and repo avatars; Add documents

* Fix bug

* Fix bug

* Add back missed avatar link method

* refactor codes

* Simplify the codes

* Code improvements

* Fix lint

* Fix test mysql

* Fix test mysql

* Fix test mysql

* Fix settings

* Fix test

* fix test

* Fix bug
This commit is contained in:
Lunny Xiao 2020-10-14 21:07:51 +08:00 • committed by GitHub
parent 93f7525061
commit 80a6b0f5bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 705 additions and 477 deletions
Download patch file Download diff file Expand all files Collapse all files

9
modules/avatar/avatar.go
View file

@ -9,6 +9,7 @@ import (
"fmt"
"image"
"image/color/palette"
// Enable PNG support:
_ "image/png"
"math/rand"
@ -57,11 +58,11 @@ func Prepare(data []byte) (*image.Image, error) {
if err != nil {
return nil, fmt.Errorf("DecodeConfig: %v", err)
}
if imgCfg.Width > setting.AvatarMaxWidth {
return nil, fmt.Errorf("Image width is too large: %d > %d", imgCfg.Width, setting.AvatarMaxWidth)
if imgCfg.Width > setting.Avatar.MaxWidth {
return nil, fmt.Errorf("Image width is too large: %d > %d", imgCfg.Width, setting.Avatar.MaxWidth)
}
if imgCfg.Height > setting.AvatarMaxHeight {
return nil, fmt.Errorf("Image height is too large: %d > %d", imgCfg.Height, setting.AvatarMaxHeight)
if imgCfg.Height > setting.Avatar.MaxHeight {
return nil, fmt.Errorf("Image height is too large: %d > %d", imgCfg.Height, setting.Avatar.MaxHeight)
}
img, _, err := image.Decode(bytes.NewReader(data))

16
modules/avatar/avatar_test.go
View file

@ -22,8 +22,8 @@ func Test_RandomImage(t *testing.T) {
}
func Test_PrepareWithPNG(t *testing.T) {
setting.AvatarMaxWidth = 4096
setting.AvatarMaxHeight = 4096
setting.Avatar.MaxWidth = 4096
setting.Avatar.MaxHeight = 4096
data, err := ioutil.ReadFile("testdata/avatar.png")
assert.NoError(t, err)
@ -36,8 +36,8 @@ func Test_PrepareWithPNG(t *testing.T) {
}
func Test_PrepareWithJPEG(t *testing.T) {
setting.AvatarMaxWidth = 4096
setting.AvatarMaxHeight = 4096
setting.Avatar.MaxWidth = 4096
setting.Avatar.MaxHeight = 4096
data, err := ioutil.ReadFile("testdata/avatar.jpeg")
assert.NoError(t, err)
@ -50,15 +50,15 @@ func Test_PrepareWithJPEG(t *testing.T) {
}
func Test_PrepareWithInvalidImage(t *testing.T) {
setting.AvatarMaxWidth = 5
setting.AvatarMaxHeight = 5
setting.Avatar.MaxWidth = 5
setting.Avatar.MaxHeight = 5
_, err := Prepare([]byte{})
assert.EqualError(t, err, "DecodeConfig: image: unknown format")
}
func Test_PrepareWithInvalidImageSize(t *testing.T) {
setting.AvatarMaxWidth = 5
setting.AvatarMaxHeight = 5
setting.Avatar.MaxWidth = 5
setting.Avatar.MaxHeight = 5
data, err := ioutil.ReadFile("testdata/avatar.png")
assert.NoError(t, err)

3
modules/setting/database.go
View file

@ -47,7 +47,8 @@ var (
ConnMaxLifetime time.Duration
IterateBufferSize int
}{
Timeout: 500,
Timeout: 500,
IterateBufferSize: 50,
}
)

114
modules/setting/picture.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"net/url"
"code.gitea.io/gitea/modules/log"
"strk.kbt.io/projects/go/libravatar"
)
// settings
var (
// Picture settings
Avatar = struct {
Storage
MaxWidth int
MaxHeight int
MaxFileSize int64
}{
MaxWidth: 4096,
MaxHeight: 3072,
MaxFileSize: 1048576,
}
GravatarSource string
GravatarSourceURL *url.URL
DisableGravatar bool
EnableFederatedAvatar bool
LibravatarService *libravatar.Libravatar
RepoAvatar = struct {
Storage
Fallback string
FallbackImage string
}{}
)
func newPictureService() {
sec := Cfg.Section("picture")
avatarSec := Cfg.Section("avatar")
storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH
avatarSec.Key("PATH").MustString(
sec.Key("AVATAR_UPLOAD_PATH").String())
Avatar.Storage = getStorage("avatars", storageType, avatarSec)
Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096)
Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(3072)
Avatar.MaxFileSize = sec.Key("AVATAR_MAX_FILE_SIZE").MustInt64(1048576)
switch source := sec.Key("GRAVATAR_SOURCE").MustString("gravatar"); source {
case "duoshuo":
GravatarSource = "http://gravatar.duoshuo.com/avatar/"
case "gravatar":
GravatarSource = "https://secure.gravatar.com/avatar/"
case "libravatar":
GravatarSource = "https://seccdn.libravatar.org/avatar/"
default:
GravatarSource = source
}
DisableGravatar = sec.Key("DISABLE_GRAVATAR").MustBool()
EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(!InstallLock)
if OfflineMode {
DisableGravatar = true
EnableFederatedAvatar = false
}
if DisableGravatar {
EnableFederatedAvatar = false
}
if EnableFederatedAvatar || !DisableGravatar {
var err error
GravatarSourceURL, err = url.Parse(GravatarSource)
if err != nil {
log.Fatal("Failed to parse Gravatar URL(%s): %v",
GravatarSource, err)
}
}
if EnableFederatedAvatar {
LibravatarService = libravatar.New()
if GravatarSourceURL.Scheme == "https" {
LibravatarService.SetUseHTTPS(true)
LibravatarService.SetSecureFallbackHost(GravatarSourceURL.Host)
} else {
LibravatarService.SetUseHTTPS(false)
LibravatarService.SetFallbackHost(GravatarSourceURL.Host)
}
}
newRepoAvatarService()
}
func newRepoAvatarService() {
sec := Cfg.Section("picture")
repoAvatarSec := Cfg.Section("repo-avatar")
storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH
repoAvatarSec.Key("PATH").MustString(
sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String())
RepoAvatar.Storage = getStorage("repo-avatars", storageType, repoAvatarSec)
RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none")
RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString("/img/repo_default.png")
}

69
modules/setting/setting.go
View file

@ -30,7 +30,6 @@ import (
"github.com/unknwon/com"
gossh "golang.org/x/crypto/ssh"
ini "gopkg.in/ini.v1"
"strk.kbt.io/projects/go/libravatar"
)
// Scheme describes protocol types
@ -272,20 +271,6 @@ var (
DefaultEmailNotification string
}
// Picture settings
AvatarUploadPath string
AvatarMaxWidth int
AvatarMaxHeight int
GravatarSource string
GravatarSourceURL *url.URL
DisableGravatar bool
EnableFederatedAvatar bool
LibravatarService *libravatar.Libravatar
AvatarMaxFileSize int64
RepositoryAvatarUploadPath string
RepositoryAvatarFallback string
RepositoryAvatarFallbackImage string
// Log settings
LogLevel string
StacktraceLogLevel string
@ -864,59 +849,7 @@ func NewContext() {
newRepository()
sec = Cfg.Section("picture")
AvatarUploadPath = sec.Key("AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "avatars"))
forcePathSeparator(AvatarUploadPath)
if !filepath.IsAbs(AvatarUploadPath) {
AvatarUploadPath = path.Join(AppWorkPath, AvatarUploadPath)
}
RepositoryAvatarUploadPath = sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "repo-avatars"))
forcePathSeparator(RepositoryAvatarUploadPath)
if !filepath.IsAbs(RepositoryAvatarUploadPath) {
RepositoryAvatarUploadPath = path.Join(AppWorkPath, RepositoryAvatarUploadPath)
}
RepositoryAvatarFallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none")
RepositoryAvatarFallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString("/img/repo_default.png")
AvatarMaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096)
AvatarMaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(3072)
AvatarMaxFileSize = sec.Key("AVATAR_MAX_FILE_SIZE").MustInt64(1048576)
switch source := sec.Key("GRAVATAR_SOURCE").MustString("gravatar"); source {
case "duoshuo":
GravatarSource = "http://gravatar.duoshuo.com/avatar/"
case "gravatar":
GravatarSource = "https://secure.gravatar.com/avatar/"
case "libravatar":
GravatarSource = "https://seccdn.libravatar.org/avatar/"
default:
GravatarSource = source
}
DisableGravatar = sec.Key("DISABLE_GRAVATAR").MustBool()
EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(!InstallLock)
if OfflineMode {
DisableGravatar = true
EnableFederatedAvatar = false
}
if DisableGravatar {
EnableFederatedAvatar = false
}
if EnableFederatedAvatar || !DisableGravatar {
GravatarSourceURL, err = url.Parse(GravatarSource)
if err != nil {
log.Fatal("Failed to parse Gravatar URL(%s): %v",
GravatarSource, err)
}
}
if EnableFederatedAvatar {
LibravatarService = libravatar.New()
if GravatarSourceURL.Scheme == "https" {
LibravatarService.SetUseHTTPS(true)
LibravatarService.SetSecureFallbackHost(GravatarSourceURL.Host)
} else {
LibravatarService.SetUseHTTPS(false)
LibravatarService.SetFallbackHost(GravatarSourceURL.Host)
}
}
newPictureService()
if err = Cfg.Section("ui").MapTo(&UI); err != nil {
log.Fatal("Failed to map UI settings: %v", err)

38
modules/storage/storage.go
View file

@ -82,12 +82,32 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr
return dstStorage.Save(dstPath, f)
}
// SaveFrom saves data to the ObjectStorage with path p from the callback
func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error {
pr, pw := io.Pipe()
defer pr.Close()
go func() {
defer pw.Close()
if err := callback(pw); err != nil {
_ = pw.CloseWithError(err)
}
}()
_, err := objStorage.Save(p, pr)
return err
}
var (
// Attachments represents attachments storage
Attachments ObjectStorage
// LFS represents lfs storage
LFS ObjectStorage
// Avatars represents user avatars storage
Avatars ObjectStorage
// RepoAvatars represents repository avatars storage
RepoAvatars ObjectStorage
)
// Init init the stoarge
@ -96,6 +116,14 @@ func Init() error {
return err
}
if err := initAvatars(); err != nil {
return err
}
if err := initRepoAvatars(); err != nil {
return err
}
return initLFS()
}
@ -112,6 +140,11 @@ func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
return fn(context.Background(), cfg)
}
func initAvatars() (err error) {
Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage)
return
}
func initAttachments() (err error) {
Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage)
return
@ -121,3 +154,8 @@ func initLFS() (err error) {
LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage)
return
}
func initRepoAvatars() (err error) {
RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage)
return
}