Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

routers/api/v1/repo/migrate.go

@@ -231,7 +231,7 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA
 	case base.IsErrNotSupported(err):
 		ctx.Error(http.StatusUnprocessableEntity, "", err)
 	default:
-		err = util.URLSanitizedError(err, remoteAddr)
+		err = util.NewStringURLSanitizedError(err, remoteAddr, true)
 		if strings.Contains(err.Error(), "Authentication failed") ||
 			strings.Contains(err.Error(), "Bad credentials") ||
 			strings.Contains(err.Error(), "could not read Username") {

routers/web/repo/migrate.go

@@ -101,7 +101,7 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam
 		ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
 	default:
 		remoteAddr, _ := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
-		err = util.URLSanitizedError(err, remoteAddr)
+		err = util.NewStringURLSanitizedError(err, remoteAddr, true)
 		if strings.Contains(err.Error(), "Authentication failed") ||
 			strings.Contains(err.Error(), "Bad credentials") ||
 			strings.Contains(err.Error(), "could not read Username") {

routers/web/repo/setting.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -25,6 +26,7 @@ import (
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/validation"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/routers/utils"
@@ -49,6 +51,8 @@ func Settings(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("repo.settings")
 	ctx.Data["PageIsSettingsOptions"] = true
 	ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate
+	ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
+	ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval
 
 	signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath())
 	ctx.Data["SigningKeyAvailable"] = len(signing) > 0
@@ -167,10 +171,9 @@ func SettingsPost(ctx *context.Context) {
 			}
 		}
 
-		oldUsername := mirror_service.Username(ctx.Repo.Mirror)
-		oldPassword := mirror_service.Password(ctx.Repo.Mirror)
-		if form.MirrorPassword == "" && form.MirrorUsername == oldUsername {
-			form.MirrorPassword = oldPassword
+		u, _ := git.GetRemoteAddress(ctx.Repo.Repository.RepoPath(), ctx.Repo.Mirror.GetRemoteName())
+		if u.User != nil && form.MirrorPassword == "" && form.MirrorUsername == u.User.Username() {
+			form.MirrorPassword, _ = u.User.Password()
 		}
 
 		address, err := forms.ParseRemoteAddr(form.MirrorAddress, form.MirrorUsername, form.MirrorPassword)
@@ -226,6 +229,92 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
 		ctx.Redirect(repo.Link() + "/settings")
 
+	case "push-mirror-sync":
+		m, err := selectPushMirrorByForm(form, repo)
+		if err != nil {
+			ctx.NotFound("", nil)
+			return
+		}
+
+		mirror_service.AddPushMirrorToQueue(m.ID)
+
+		ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
+		ctx.Redirect(repo.Link() + "/settings")
+
+	case "push-mirror-remove":
+		// This section doesn't require repo_name/RepoName to be set in the form, don't show it
+		// as an error on the UI for this action
+		ctx.Data["Err_RepoName"] = nil
+
+		m, err := selectPushMirrorByForm(form, repo)
+		if err != nil {
+			ctx.NotFound("", nil)
+			return
+		}
+
+		if err = mirror_service.RemovePushMirrorRemote(m); err != nil {
+			ctx.ServerError("RemovePushMirrorRemote", err)
+			return
+		}
+
+		if err = models.DeletePushMirrorByID(m.ID); err != nil {
+			ctx.ServerError("DeletePushMirrorByID", err)
+			return
+		}
+
+		ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
+		ctx.Redirect(repo.Link() + "/settings")
+
+	case "push-mirror-add":
+		// This section doesn't require repo_name/RepoName to be set in the form, don't show it
+		// as an error on the UI for this action
+		ctx.Data["Err_RepoName"] = nil
+
+		interval, err := time.ParseDuration(form.PushMirrorInterval)
+		if err != nil || (interval != 0 && interval < setting.Mirror.MinInterval) {
+			ctx.Data["Err_PushMirrorInterval"] = true
+			ctx.RenderWithErr(ctx.Tr("repo.mirror_interval_invalid"), tplSettingsOptions, &form)
+			return
+		}
+
+		address, err := forms.ParseRemoteAddr(form.PushMirrorAddress, form.PushMirrorUsername, form.PushMirrorPassword)
+		if err == nil {
+			err = migrations.IsMigrateURLAllowed(address, ctx.User)
+		}
+		if err != nil {
+			ctx.Data["Err_PushMirrorAddress"] = true
+			handleSettingRemoteAddrError(ctx, err, form)
+			return
+		}
+
+		remoteSuffix, err := util.RandomString(10)
+		if err != nil {
+			ctx.ServerError("RandomString", err)
+			return
+		}
+
+		m := &models.PushMirror{
+			RepoID:     repo.ID,
+			Repo:       repo,
+			RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
+			Interval:   interval,
+		}
+		if err := models.InsertPushMirror(m); err != nil {
+			ctx.ServerError("InsertPushMirror", err)
+			return
+		}
+
+		if err := mirror_service.AddPushMirrorRemote(m, address); err != nil {
+			if err := models.DeletePushMirrorByID(m.ID); err != nil {
+				log.Error("DeletePushMirrorByID %v", err)
+			}
+			ctx.ServerError("AddPushMirrorRemote", err)
+			return
+		}
+
+		ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
+		ctx.Redirect(repo.Link() + "/settings")
+
 	case "advanced":
 		var repoChanged bool
 		var units []models.RepoUnit
@@ -1051,3 +1140,22 @@ func SettingsDeleteAvatar(ctx *context.Context) {
 	}
 	ctx.Redirect(ctx.Repo.RepoLink + "/settings")
 }
+
+func selectPushMirrorByForm(form *forms.RepoSettingForm, repo *models.Repository) (*models.PushMirror, error) {
+	id, err := strconv.ParseInt(form.PushMirrorID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = repo.LoadPushMirrors(); err != nil {
+		return nil, err
+	}
+
+	for _, m := range repo.PushMirrors {
+		if m.ID == id {
+			return m, nil
+		}
+	}
+
+	return nil, fmt.Errorf("PushMirror[%v] not associated to repository %v", id, repo)
+}