finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add default storage configurations (#12813)

Browse source 
Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
Lunny Xiao 2020-09-29 17:05:13 +08:00 • committed by GitHub
parent 4c6ac08182
commit 3878e985b6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 459 additions and 185 deletions
Download patch file Download diff file Expand all files Collapse all files

73
modules/setting/attachment.go
View file

@ -5,42 +5,25 @@
package setting
import (
"path"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/log"
)
var (
// Attachment settings
Attachment = struct {
StoreType string
Path string
ServeDirect bool
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}
Storage
AllowedTypes string
MaxSize int64
MaxFiles int
Enabled bool
}{
StoreType: "local",
ServeDirect: false,
Minio: struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}{},
Storage: Storage{
Type: LocalStorageType,
ServeDirect: false,
},
AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
MaxSize: 4,
MaxFiles: 5,
@ -50,22 +33,36 @@ var (
func newAttachmentService() {
sec := Cfg.Section("attachment")
Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
switch Attachment.StoreType {
case "local":
Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
if !filepath.IsAbs(Attachment.Path) {
Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("")
if Attachment.Storage.Type == "" {
Attachment.Storage.Type = "default"
}
if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType {
storage, ok := storages[Attachment.Storage.Type]
if !ok {
log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type)
}
case "minio":
Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
Attachment.Storage = storage
}
// Override
Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect)
switch Attachment.Storage.Type {
case LocalStorageType:
Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
if !filepath.IsAbs(Attachment.Path) {
Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path)
}
case MinioStorageType:
Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint)
Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID)
Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey)
Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket)
Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location)
Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL)
Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
}
Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)

62
modules/setting/lfs.go
View file

@ -21,27 +21,14 @@ import (
// LFS represents the configuration for Git LFS
var LFS = struct {
StartServer bool `ini:"LFS_START_SERVER"`
ContentPath string `ini:"LFS_CONTENT_PATH"`
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
StoreType string
ServeDirect bool
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}
}{
StoreType: "local",
}
Storage
}{}
func newLFSService() {
sec := Cfg.Section("server")
@ -49,10 +36,41 @@ func newLFSService() {
log.Fatal("Failed to map LFS settings: %v", err)
}
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
if !filepath.IsAbs(LFS.ContentPath) {
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
lfsSec := Cfg.Section("lfs")
LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("")
if LFS.Storage.Type == "" {
LFS.Storage.Type = "default"
}
if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType {
storage, ok := storages[LFS.Storage.Type]
if !ok {
log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type)
}
LFS.Storage = storage
}
// Override
LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect)
switch LFS.Storage.Type {
case LocalStorageType:
// keep compatible
LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path)
if !filepath.IsAbs(LFS.Path) {
LFS.Path = filepath.Join(AppWorkPath, LFS.Path)
}
case MinioStorageType:
LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint)
LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID)
LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey)
LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket)
LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location)
LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL)
LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
}
if LFS.LocksPagingNum == 0 {
LFS.LocksPagingNum = 50
}
@ -92,14 +110,6 @@ func newLFSService() {
}
}
func ensureLFSDirectory() {
if LFS.StartServer {
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
}
}
}
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
func CheckLFSVersion() {
if LFS.StartServer {

5
modules/setting/setting.go
View file

@ -691,8 +691,6 @@ func NewContext() {
SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
newLFSService()
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
log.Fatal("Failed to OAuth2 settings: %v", err)
return
@ -761,7 +759,9 @@ func NewContext() {
}
}
newStorageService()
newAttachmentService()
newLFSService()
timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
if timeFormatKey != "" {
@ -1017,7 +1017,6 @@ func NewServices() {
InitDBConfig()
newService()
NewLogServices(false)
ensureLFSDirectory()
newCacheService()
newSessionService()
newCORSService()

69
modules/setting/storage.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"strings"
"code.gitea.io/gitea/modules/log"
ini "gopkg.in/ini.v1"
)
// enumerate all storage types
const (
LocalStorageType = "local"
MinioStorageType = "minio"
)
// Storage represents configuration of storages
type Storage struct {
Type string
Path string
ServeDirect bool
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}
}
var (
storages = make(map[string]Storage)
)
func getStorage(sec *ini.Section) Storage {
var storage Storage
storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType)
storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
switch storage.Type {
case LocalStorageType:
case MinioStorageType:
storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
}
return storage
}
func newStorageService() {
sec := Cfg.Section("storage")
storages["default"] = getStorage(sec)
for _, sec := range Cfg.Section("storage").ChildSections() {
name := strings.TrimPrefix(sec.Name(), "storage.")
if name == "default" || name == LocalStorageType || name == MinioStorageType {
log.Error("storage name %s is system reserved!", name)
continue
}
storages[name] = getStorage(sec)
}
}

27
modules/storage/local.go
View file

@ -59,7 +59,7 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
}
// Stat returns the info of the file
func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
func (l *LocalStorage) Stat(path string) (os.FileInfo, error) {
return os.Stat(filepath.Join(l.dir, path))
}
@ -73,3 +73,28 @@ func (l *LocalStorage) Delete(path string) error {
func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
return nil, ErrURLNotSupported
}
// IterateObjects iterates across the objects in the local storage
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == l.dir {
return nil
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(l.dir, path)
if err != nil {
return err
}
obj, err := os.Open(path)
if err != nil {
return err
}
defer obj.Close()
return fn(relPath, obj)
})
}

52
modules/storage/minio.go
View file

@ -22,6 +22,19 @@ var (
quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
)
type minioObject struct {
*minio.Object
}
func (m *minioObject) Stat() (os.FileInfo, error) {
oi, err := m.Object.Stat()
if err != nil {
return nil, err
}
return &minioFileInfo{oi}, nil
}
// MinioStorage returns a minio bucket storage
type MinioStorage struct {
ctx context.Context
@ -69,7 +82,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
if err != nil {
return nil, err
}
return object, nil
return &minioObject{object}, nil
}
// Save save a file to minio
@ -104,8 +117,20 @@ func (m minioFileInfo) ModTime() time.Time {
return m.LastModified
}
func (m minioFileInfo) IsDir() bool {
return strings.HasSuffix(m.ObjectInfo.Key, "/")
}
func (m minioFileInfo) Mode() os.FileMode {
return os.ModePerm
}
func (m minioFileInfo) Sys() interface{} {
return nil
}
// Stat returns the stat information of the object
func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
info, err := m.client.StatObject(
m.ctx,
m.bucket,
@ -135,3 +160,26 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
}
// IterateObjects iterates across the objects in the miniostorage
func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) error {
var opts = minio.GetObjectOptions{}
lobjectCtx, cancel := context.WithCancel(m.ctx)
defer cancel()
for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
Prefix: m.basePath,
Recursive: true,
}) {
object, err := m.client.GetObject(lobjectCtx, m.bucket, mObjInfo.Key, opts)
if err != nil {
return err
}
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
defer object.Close()
return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object})
}(object, fn); err != nil {
return err
}
}
return nil
}

70
modules/storage/storage.go
View file

@ -10,7 +10,7 @@ import (
"fmt"
"io"
"net/url"
"time"
"os"
"code.gitea.io/gitea/modules/setting"
)
@ -18,28 +18,25 @@ import (
var (
// ErrURLNotSupported represents url is not supported
ErrURLNotSupported = errors.New("url method not supported")
// ErrIterateObjectsNotSupported represents IterateObjects not supported
ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported")
)
// Object represents the object on the storage
type Object interface {
io.ReadCloser
io.Seeker
}
// ObjectInfo represents the object info on the storage
type ObjectInfo interface {
Name() string
Size() int64
ModTime() time.Time
Stat() (os.FileInfo, error)
}
// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
Open(path string) (Object, error)
Save(path string, r io.Reader) (int64, error)
Stat(path string) (ObjectInfo, error)
Stat(path string) (os.FileInfo, error)
Delete(path string) error
URL(path, name string) (*url.URL, error)
IterateObjects(func(path string, obj Object) error) error
}
// Copy copys a file from source ObjectStorage to dest ObjectStorage
@ -70,14 +67,15 @@ func Init() error {
return initLFS()
}
func initAttachments() error {
func initStorage(storageCfg setting.Storage) (ObjectStorage, error) {
var err error
switch setting.Attachment.StoreType {
case "local":
Attachments, err = NewLocalStorage(setting.Attachment.Path)
case "minio":
minio := setting.Attachment.Minio
Attachments, err = NewMinioStorage(
var s ObjectStorage
switch storageCfg.Type {
case setting.LocalStorageType:
s, err = NewLocalStorage(storageCfg.Path)
case setting.MinioStorageType:
minio := storageCfg.Minio
s, err = NewMinioStorage(
context.Background(),
minio.Endpoint,
minio.AccessKeyID,
@ -88,40 +86,22 @@ func initAttachments() error {
minio.UseSSL,
)
default:
return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type)
}
if err != nil {
return err
return nil, err
}
return nil
return s, nil
}
func initLFS() error {
var err error
switch setting.LFS.StoreType {
case "local":
LFS, err = NewLocalStorage(setting.LFS.ContentPath)
case "minio":
minio := setting.LFS.Minio
LFS, err = NewMinioStorage(
context.Background(),
minio.Endpoint,
minio.AccessKeyID,
minio.SecretAccessKey,
minio.Bucket,
minio.Location,
minio.BasePath,
minio.UseSSL,
)
default:
return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
}
if err != nil {
return err
}
return nil
func initAttachments() (err error) {
Attachments, err = initStorage(setting.Attachment.Storage)
return
}
func initLFS() (err error) {
LFS, err = initStorage(setting.LFS.Storage)
return
}