finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Dump: add output format tar and output to stdout (#10376)

Browse source 
* Dump: Use mholt/archive/v3 to support tar including many compressions

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Dump: Allow dump output to stdout

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Dump: Fixed bug present since #6677 where SessionConfig.Provider is never "file"

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Dump: never pack RepoRootPath, LFS.ContentPath and LogRootPath when they are below AppDataPath

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Dump: also dump LFS (fixes #10058)

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Dump: never dump CustomPath if CustomPath is a subdir of or equal to AppDataPath (fixes #10365)

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* Use log.Info instead of fmt.Fprintf

Signed-off-by: Philipp Homann <homann.philipp@googlemail.com>

* import ordering

* make fmt

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Matti R <matti@mdranta.net>
This commit is contained in:
PhilippHomann 2020-06-05 22:47:39 +02:00 • committed by GitHub
parent 209b17c4e2
commit 684b7a999f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
303 changed files with 301317 additions and 1183 deletions
Download patch file Download diff file Expand all files Collapse all files

273
cmd/dump.go
View file

@ -6,22 +6,120 @@
package cmd
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/unknwon/cae/zip"
"gitea.com/macaron/session"
archiver "github.com/mholt/archiver/v3"
"github.com/unknwon/com"
"github.com/urfave/cli"
)
func addFile(w archiver.Writer, filePath string, absPath string, verbose bool) error {
if verbose {
log.Info("Adding file %s\n", filePath)
}
file, err := os.Open(absPath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
return w.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: fileInfo,
CustomName: filePath,
},
ReadCloser: file,
})
}
func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose bool) error {
if verbose {
log.Info("Adding dir %s\n", dirPath)
}
dir, err := os.Open(absPath)
if err != nil {
return fmt.Errorf("Could not open directory %s: %s", absPath, err)
}
files, err := dir.Readdir(0)
if err != nil {
return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
}
if err := addFile(w, dirPath, absPath, false); err != nil {
return err
}
for _, fileInfo := range files {
if fileInfo.IsDir() {
err = addRecursive(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
} else {
err = addFile(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
}
if err != nil {
return err
}
}
return nil
}
func isSubdir(upper string, lower string) (bool, error) {
if relPath, err := filepath.Rel(upper, lower); err != nil {
return false, err
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
return true, nil
}
return false, nil
}
type outputType struct {
Enum []string
Default string
selected string
}
func (o outputType) Join() string {
return strings.Join(o.Enum, ", ")
}
func (o *outputType) Set(value string) error {
for _, enum := range o.Enum {
if enum == value {
o.selected = value
return nil
}
}
return fmt.Errorf("allowed values are %s", o.Join())
}
func (o outputType) String() string {
if o.selected == "" {
return o.Default
}
return o.selected
}
var outputTypeEnum = &outputType{
Enum: []string{"zip", "tar", "tar.gz", "tar.xz", "tar.bz2"},
Default: "zip",
}
// CmdDump represents the available dump sub-command.
var CmdDump = cli.Command{
Name: "dump",
@ -33,7 +131,7 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
cli.StringFlag{
Name: "file, f",
Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
Usage: "Name of the dump file which will be created.",
Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
},
cli.BoolFlag{
Name: "verbose, V",
@ -56,6 +154,11 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
Name: "skip-log, L",
Usage: "Skip the log dumping",
},
cli.GenericFlag{
Name: "type",
Value: outputTypeEnum,
Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
},
},
}
@ -65,7 +168,23 @@ func fatal(format string, args ...interface{}) {
}
func runDump(ctx *cli.Context) error {
var file *os.File
fileName := ctx.String("file")
if fileName == "-" {
file = os.Stdout
err := log.DelLogger("console")
if err != nil {
fatal("Deleting default logger failed. Can not write to stdout: %v", err)
}
}
setting.NewContext()
// make sure we are logging to the console no matter what the configuration tells us do to
if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
fatal("Setting logging mode to console failed: %v", err)
}
if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil {
fatal("Setting console logger to stderr failed: %v", err)
}
setting.NewServices() // cannot access session settings otherwise
err := models.SetEngine()
@ -73,45 +192,59 @@ func runDump(ctx *cli.Context) error {
return err
}
tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
fatal("Path does not exist: %s", tmpDir)
if file == nil {
file, err = os.Create(fileName)
if err != nil {
fatal("Unable to open %s: %v", fileName, err)
}
}
defer file.Close()
verbose := ctx.Bool("verbose")
outType := ctx.String("type")
var iface interface{}
if fileName == "-" {
iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
} else {
iface, err = archiver.ByExtension(fileName)
}
tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-")
if err != nil {
fatal("Failed to create tmp work directory: %v", err)
}
log.Info("Creating tmp work dir: %s", tmpWorkDir)
// work-around #1103
if os.Getenv("TMPDIR") == "" {
os.Setenv("TMPDIR", tmpWorkDir)
fatal("Unable to get archiver for extension: %v", err)
}
dbDump := path.Join(tmpWorkDir, "gitea-db.sql")
fileName := ctx.String("file")
log.Info("Packing dump files...")
z, err := zip.Create(fileName)
if err != nil {
fatal("Failed to create %s: %v", fileName, err)
w, _ := iface.(archiver.Writer)
if err := w.Create(file); err != nil {
fatal("Creating archiver.Writer failed: %v", err)
}
zip.Verbose = ctx.Bool("verbose")
defer w.Close()
if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
log.Info("Skip dumping local repositories")
} else {
log.Info("Dumping local repositories...%s", setting.RepoRootPath)
reposDump := path.Join(tmpWorkDir, "gitea-repo.zip")
if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
fatal("Failed to dump local repositories: %v", err)
log.Info("Dumping local repositories... %s", setting.RepoRootPath)
if err := addRecursive(w, "repos", setting.RepoRootPath, verbose); err != nil {
fatal("Failed to include repositories: %v", err)
}
if err := z.AddFile("gitea-repo.zip", reposDump); err != nil {
fatal("Failed to include gitea-repo.zip: %v", err)
if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
fatal("Failed to include lfs: %v", err)
}
}
}
tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
fatal("Path does not exist: %s", tmpDir)
}
dbDump, err := ioutil.TempFile(tmpDir, "gitea-db.sql")
if err != nil {
fatal("Failed to create tmp file: %v", err)
}
defer os.Remove(dbDump.Name())
targetDBType := ctx.String("database")
if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
@ -119,25 +252,29 @@ func runDump(ctx *cli.Context) error {
log.Info("Dumping database...")
}
if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
if err := models.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
fatal("Failed to dump database: %v", err)
}
if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
fatal("Failed to include gitea-db.sql: %v", err)
}
if len(setting.CustomConf) > 0 {
log.Info("Adding custom configuration file from %s", setting.CustomConf)
if err := z.AddFile("app.ini", setting.CustomConf); err != nil {
if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
fatal("Failed to include specified app.ini: %v", err)
}
}
customDir, err := os.Stat(setting.CustomPath)
if err == nil && customDir.IsDir() {
if err := z.AddDir("custom", setting.CustomPath); err != nil {
fatal("Failed to include custom: %v", err)
if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
if err := addRecursive(w, "custom", setting.CustomPath, verbose); err != nil {
fatal("Failed to include custom: %v", err)
}
} else {
log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
}
} else {
log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
@ -146,11 +283,19 @@ func runDump(ctx *cli.Context) error {
if com.IsExist(setting.AppDataPath) {
log.Info("Packing data directory...%s", setting.AppDataPath)
var sessionAbsPath string
if setting.SessionConfig.Provider == "file" {
sessionAbsPath = setting.SessionConfig.ProviderConfig
var excludes []string
if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" {
var opts session.Options
if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
return err
}
excludes = append(excludes, opts.ProviderConfig)
}
if err := zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil {
excludes = append(excludes, setting.RepoRootPath)
excludes = append(excludes, setting.LFS.ContentPath)
excludes = append(excludes, setting.LogRootPath)
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
fatal("Failed to include data directory: %v", err)
}
}
@ -161,32 +306,42 @@ func runDump(ctx *cli.Context) error {
if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
log.Info("Skip dumping log files")
} else if com.IsExist(setting.LogRootPath) {
if err := z.AddDir("log", setting.LogRootPath); err != nil {
if err := addRecursive(w, "log", setting.LogRootPath, verbose); err != nil {
fatal("Failed to include log: %v", err)
}
}
if err = z.Close(); err != nil {
_ = os.Remove(fileName)
fatal("Failed to save %s: %v", fileName, err)
if fileName != "-" {
if err = w.Close(); err != nil {
_ = os.Remove(fileName)
fatal("Failed to save %s: %v", fileName, err)
}
if err := os.Chmod(fileName, 0600); err != nil {
log.Info("Can't change file access permissions mask to 0600: %v", err)
}
}
if err := os.Chmod(fileName, 0600); err != nil {
log.Info("Can't change file access permissions mask to 0600: %v", err)
if fileName != "-" {
log.Info("Finish dumping in file %s", fileName)
} else {
log.Info("Finish dumping to stdout")
}
log.Info("Removing tmp work dir: %s", tmpWorkDir)
if err := os.RemoveAll(tmpWorkDir); err != nil {
fatal("Failed to remove %s: %v", tmpWorkDir, err)
}
log.Info("Finish dumping in file %s", fileName)
return nil
}
// zipAddDirectoryExclude zips absPath to specified zipPath inside z excluding excludeAbsPath
func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, excludeAbsPath string) error {
func contains(slice []string, s string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}
// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
absPath, err := filepath.Abs(absPath)
if err != nil {
return err
@ -197,24 +352,24 @@ func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, exclud
}
defer dir.Close()
zip.AddEmptyDir(zipPath)
files, err := dir.Readdir(0)
if err != nil {
return err
}
for _, file := range files {
currentAbsPath := path.Join(absPath, file.Name())
currentZipPath := path.Join(zipPath, file.Name())
currentInsidePath := path.Join(insidePath, file.Name())
if file.IsDir() {
if currentAbsPath != excludeAbsPath {
if err = zipAddDirectoryExclude(zip, currentZipPath, currentAbsPath, excludeAbsPath); err != nil {
if !contains(excludeAbsPath, currentAbsPath) {
if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
return err
}
if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
return err
}
}
} else {
if err = zip.AddFile(currentZipPath, currentAbsPath); err != nil {
if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
return err
}
}