From 03508b33a8e890b05d845bbd8ead8672b8f03578 Mon Sep 17 00:00:00 2001 From: Philip Peterson Date: Sun, 4 Aug 2024 14:46:05 -0400 Subject: [PATCH] [FEAT] Allow pushmirror to use publickey authentication - Continuation of https://github.com/go-gitea/gitea/pull/18835 (by @Gusted, so it's fine to change copyright holder to Forgejo). - Add the option to use SSH for push mirrors, this would allow for the deploy keys feature to be used and not require tokens to be used which cannot be limited to a specific repository. The private key is stored encrypted (via the `keying` module) on the database and NEVER given to the user, to avoid accidental exposure and misuse. - CAVEAT: This does require the `ssh` binary to be present, which may not be available in containerized environments, this could be solved by adding a SSH client into forgejo itself and use the forgejo binary as SSH command, but should be done in another PR. - CAVEAT: Mirroring of LFS content is not supported, this would require the previous stated problem to be solved due to LFS authentication (an attempt was made at forgejo/forgejo#2544). - Integration test added. - Resolves #4416 --- .deadcode-out | 5 - models/forgejo_migrations/migrate.go | 2 + models/forgejo_migrations/v21.go | 16 +++ models/repo/pushmirror.go | 28 +++++ models/repo/pushmirror_test.go | 27 ++++ modules/git/repo.go | 36 +++++- modules/keying/keying.go | 14 +++ modules/keying/keying_test.go | 15 +++ modules/lfs/endpoint.go | 4 + modules/structs/mirror.go | 2 + modules/util/util.go | 24 ++++ modules/util/util_test.go | 118 +++++++++++------- options/locale/locale_en-US.ini | 6 + release-notes/4819.md | 1 + routers/api/v1/repo/mirror.go | 25 +++- routers/web/repo/setting/setting.go | 30 ++++- services/convert/mirror.go | 1 + services/forms/repo_form.go | 16 ++- services/migrations/migrate.go | 2 +- services/mirror/mirror_push.go | 41 ++++++- templates/repo/settings/options.tmpl | 14 ++- templates/swagger/v1_json.tmpl | 8 ++ tests/integration/api_push_mirror_test.go | 136 ++++++++++++++++++++ tests/integration/mirror_push_test.go | 143 +++++++++++++++++++++- 24 files changed, 648 insertions(+), 66 deletions(-) create mode 100644 models/forgejo_migrations/v21.go create mode 100644 release-notes/4819.md diff --git a/.deadcode-out b/.deadcode-out index 539056a42..72d5df86d 100644 --- a/.deadcode-out +++ b/.deadcode-out @@ -170,11 +170,6 @@ code.gitea.io/gitea/modules/json StdJSON.NewDecoder StdJSON.Indent -code.gitea.io/gitea/modules/keying - DeriveKey - Key.Encrypt - Key.Decrypt - code.gitea.io/gitea/modules/markup GetRendererByType RenderString diff --git a/models/forgejo_migrations/migrate.go b/models/forgejo_migrations/migrate.go index e9db250e7..598ec8bba 100644 --- a/models/forgejo_migrations/migrate.go +++ b/models/forgejo_migrations/migrate.go @@ -78,6 +78,8 @@ var migrations = []*Migration{ NewMigration("Add external_url to attachment table", AddExternalURLColumnToAttachmentTable), // v20 -> v21 NewMigration("Creating Quota-related tables", CreateQuotaTables), + // v21 -> v22 + NewMigration("Add SSH keypair to `pull_mirror` table", AddSSHKeypairToPushMirror), } // GetCurrentDBVersion returns the current Forgejo database version. diff --git a/models/forgejo_migrations/v21.go b/models/forgejo_migrations/v21.go new file mode 100644 index 000000000..53f141b2a --- /dev/null +++ b/models/forgejo_migrations/v21.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package forgejo_migrations //nolint:revive + +import "xorm.io/xorm" + +func AddSSHKeypairToPushMirror(x *xorm.Engine) error { + type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + PublicKey string `xorm:"VARCHAR(100)"` + PrivateKey []byte `xorm:"BLOB"` + } + + return x.Sync(&PushMirror{}) +} diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 3cf54faca..68fb504fd 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/git" giturl "code.gitea.io/gitea/modules/git/url" + "code.gitea.io/gitea/modules/keying" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -32,6 +33,10 @@ type PushMirror struct { RemoteName string RemoteAddress string `xorm:"VARCHAR(2048)"` + // A keypair formatted in OpenSSH format. + PublicKey string `xorm:"VARCHAR(100)"` + PrivateKey []byte `xorm:"BLOB"` + SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"` Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -82,6 +87,29 @@ func (m *PushMirror) GetRemoteName() string { return m.RemoteName } +// GetPublicKey returns a sanitized version of the public key. +// This should only be used when displaying the public key to the user, not for actual code. +func (m *PushMirror) GetPublicKey() string { + return strings.TrimSuffix(m.PublicKey, "\n") +} + +// SetPrivatekey encrypts the given private key and store it in the database. +// The ID of the push mirror must be known, so this should be done after the +// push mirror is inserted. +func (m *PushMirror) SetPrivatekey(ctx context.Context, privateKey []byte) error { + key := keying.DeriveKey(keying.ContextPushMirror) + m.PrivateKey = key.Encrypt(privateKey, keying.ColumnAndID("private_key", m.ID)) + + _, err := db.GetEngine(ctx).ID(m.ID).Cols("private_key").Update(m) + return err +} + +// Privatekey retrieves the encrypted private key and decrypts it. +func (m *PushMirror) Privatekey() ([]byte, error) { + key := keying.DeriveKey(keying.ContextPushMirror) + return key.Decrypt(m.PrivateKey, keying.ColumnAndID("private_key", m.ID)) +} + // UpdatePushMirror updates the push-mirror func UpdatePushMirror(ctx context.Context, m *PushMirror) error { _, err := db.GetEngine(ctx).ID(m.ID).AllCols().Update(m) diff --git a/models/repo/pushmirror_test.go b/models/repo/pushmirror_test.go index ebaa6e53b..c3368ccaf 100644 --- a/models/repo/pushmirror_test.go +++ b/models/repo/pushmirror_test.go @@ -50,3 +50,30 @@ func TestPushMirrorsIterate(t *testing.T) { return nil }) } + +func TestPushMirrorPrivatekey(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + m := &repo_model.PushMirror{ + RemoteName: "test-privatekey", + } + require.NoError(t, db.Insert(db.DefaultContext, m)) + + privateKey := []byte{0x00, 0x01, 0x02, 0x04, 0x08, 0x10} + t.Run("Set privatekey", func(t *testing.T) { + require.NoError(t, m.SetPrivatekey(db.DefaultContext, privateKey)) + }) + + t.Run("Normal retrieval", func(t *testing.T) { + actualPrivateKey, err := m.Privatekey() + require.NoError(t, err) + assert.EqualValues(t, privateKey, actualPrivateKey) + }) + + t.Run("Incorrect retrieval", func(t *testing.T) { + m.ID++ + actualPrivateKey, err := m.Privatekey() + require.Error(t, err) + assert.Empty(t, actualPrivateKey) + }) +} diff --git a/modules/git/repo.go b/modules/git/repo.go index 857424fcd..84db08d70 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -1,5 +1,6 @@ // Copyright 2015 The Gogs Authors. All rights reserved. // Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors. All rights reserved. // SPDX-License-Identifier: MIT package git @@ -18,6 +19,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -190,17 +192,39 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op // PushOptions options when push to remote type PushOptions struct { - Remote string - Branch string - Force bool - Mirror bool - Env []string - Timeout time.Duration + Remote string + Branch string + Force bool + Mirror bool + Env []string + Timeout time.Duration + PrivateKeyPath string } // Push pushs local commits to given remote branch. func Push(ctx context.Context, repoPath string, opts PushOptions) error { cmd := NewCommand(ctx, "push") + + if opts.PrivateKeyPath != "" { + // Preserve the behavior that existing environments are used if no + // environments are passed. + if len(opts.Env) == 0 { + opts.Env = os.Environ() + } + + // Use environment because it takes precedence over using -c core.sshcommand + // and it's possible that a system might have an existing GIT_SSH_COMMAND + // environment set. + opts.Env = append(opts.Env, "GIT_SSH_COMMAND=ssh"+ + fmt.Sprintf(` -i %s`, opts.PrivateKeyPath)+ + " -o IdentitiesOnly=yes"+ + // This will store new SSH host keys and verify connections to existing + // host keys, but it doesn't allow replacement of existing host keys. This + // means TOFU is used for Git over SSH pushes. + " -o StrictHostKeyChecking=accept-new"+ + " -o UserKnownHostsFile="+filepath.Join(setting.SSH.RootPath, "known_hosts")) + } + if opts.Force { cmd.AddArguments("-f") } diff --git a/modules/keying/keying.go b/modules/keying/keying.go index 7cf5f28a4..7c595c7f9 100644 --- a/modules/keying/keying.go +++ b/modules/keying/keying.go @@ -18,6 +18,7 @@ package keying import ( "crypto/rand" "crypto/sha256" + "encoding/binary" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/hkdf" @@ -44,6 +45,9 @@ func Init(ikm []byte) { // This must be a hardcoded string and must not be arbitrarily constructed. type Context string +// Used for the `push_mirror` table. +var ContextPushMirror Context = "pushmirror" + // Derive *the* key for a given context, this is a determistic function. The // same key will be provided for the same context. func DeriveKey(context Context) *Key { @@ -109,3 +113,13 @@ func (k *Key) Decrypt(ciphertext, additionalData []byte) ([]byte, error) { return e.Open(nil, nonce, ciphertext, additionalData) } + +// ColumnAndID generates a context that can be used as additional context for +// encrypting and decrypting data. It requires the column name and the row ID +// (this requires to be known beforehand). Be careful when using this, as the +// table name isn't part of this context. This means it's not bound to a +// particular table. The table should be part of the context that the key was +// derived for, in which case it binds through that. +func ColumnAndID(column string, id int64) []byte { + return binary.BigEndian.AppendUint64(append([]byte(column), ':'), uint64(id)) +} diff --git a/modules/keying/keying_test.go b/modules/keying/keying_test.go index 16a6781af..8a6e8d5ab 100644 --- a/modules/keying/keying_test.go +++ b/modules/keying/keying_test.go @@ -4,6 +4,7 @@ package keying_test import ( + "math" "testing" "code.gitea.io/gitea/modules/keying" @@ -94,3 +95,17 @@ func TestKeying(t *testing.T) { }) }) } + +func TestKeyingColumnAndID(t *testing.T) { + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, keying.ColumnAndID("table", math.MinInt64)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, keying.ColumnAndID("table", -1)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, keying.ColumnAndID("table", 0)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, keying.ColumnAndID("table", 1)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, keying.ColumnAndID("table", math.MaxInt64)) + + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x32, 0x3a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, keying.ColumnAndID("table2", math.MinInt64)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x32, 0x3a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, keying.ColumnAndID("table2", -1)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x32, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, keying.ColumnAndID("table2", 0)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x32, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, keying.ColumnAndID("table2", 1)) + assert.EqualValues(t, []byte{0x74, 0x61, 0x62, 0x6c, 0x65, 0x32, 0x3a, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, keying.ColumnAndID("table2", math.MaxInt64)) +} diff --git a/modules/lfs/endpoint.go b/modules/lfs/endpoint.go index 2931defcd..97bd7d444 100644 --- a/modules/lfs/endpoint.go +++ b/modules/lfs/endpoint.go @@ -60,6 +60,10 @@ func endpointFromURL(rawurl string) *url.URL { case "git": u.Scheme = "https" return u + case "ssh": + u.Scheme = "https" + u.User = nil + return u case "file": return u default: diff --git a/modules/structs/mirror.go b/modules/structs/mirror.go index 8259583cd..1b6566803 100644 --- a/modules/structs/mirror.go +++ b/modules/structs/mirror.go @@ -12,6 +12,7 @@ type CreatePushMirrorOption struct { RemotePassword string `json:"remote_password"` Interval string `json:"interval"` SyncOnCommit bool `json:"sync_on_commit"` + UseSSH bool `json:"use_ssh"` } // PushMirror represents information of a push mirror @@ -27,4 +28,5 @@ type PushMirror struct { LastError string `json:"last_error"` Interval string `json:"interval"` SyncOnCommit bool `json:"sync_on_commit"` + PublicKey string `json:"public_key"` } diff --git a/modules/util/util.go b/modules/util/util.go index b6ea28355..044468022 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -1,11 +1,14 @@ // Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors. All rights reserved. // SPDX-License-Identifier: MIT package util import ( "bytes" + "crypto/ed25519" "crypto/rand" + "encoding/pem" "fmt" "math/big" "strconv" @@ -13,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/optional" + "golang.org/x/crypto/ssh" "golang.org/x/text/cases" "golang.org/x/text/language" ) @@ -229,3 +233,23 @@ func ReserveLineBreakForTextarea(input string) string { // Other than this, we should respect the original content, even leading or trailing spaces. return strings.ReplaceAll(input, "\r\n", "\n") } + +// GenerateSSHKeypair generates a ed25519 SSH-compatible keypair. +func GenerateSSHKeypair() (publicKey, privateKey []byte, err error) { + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + return nil, nil, fmt.Errorf("ed25519.GenerateKey: %w", err) + } + + privPEM, err := ssh.MarshalPrivateKey(private, "") + if err != nil { + return nil, nil, fmt.Errorf("ssh.MarshalPrivateKey: %w", err) + } + + sshPublicKey, err := ssh.NewPublicKey(public) + if err != nil { + return nil, nil, fmt.Errorf("ssh.NewPublicKey: %w", err) + } + + return ssh.MarshalAuthorizedKey(sshPublicKey), pem.EncodeToMemory(privPEM), nil +} diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 8ed1e3207..549b53f5a 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -1,14 +1,19 @@ // Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors. All rights reserved. // SPDX-License-Identifier: MIT -package util +package util_test import ( + "bytes" + "crypto/rand" "regexp" "strings" "testing" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -43,7 +48,7 @@ func TestURLJoin(t *testing.T) { newTest("/a/b/c#hash", "/a", "b/c#hash"), } { - assert.Equal(t, test.Expected, URLJoin(test.Base, test.Elements...)) + assert.Equal(t, test.Expected, util.URLJoin(test.Base, test.Elements...)) } } @@ -59,7 +64,7 @@ func TestIsEmptyString(t *testing.T) { } for _, v := range cases { - assert.Equal(t, v.expected, IsEmptyString(v.s)) + assert.Equal(t, v.expected, util.IsEmptyString(v.s)) } } @@ -100,42 +105,42 @@ func Test_NormalizeEOL(t *testing.T) { unix := buildEOLData(data1, "\n") mac := buildEOLData(data1, "\r") - assert.Equal(t, unix, NormalizeEOL(dos)) - assert.Equal(t, unix, NormalizeEOL(mac)) - assert.Equal(t, unix, NormalizeEOL(unix)) + assert.Equal(t, unix, util.NormalizeEOL(dos)) + assert.Equal(t, unix, util.NormalizeEOL(mac)) + assert.Equal(t, unix, util.NormalizeEOL(unix)) dos = buildEOLData(data2, "\r\n") unix = buildEOLData(data2, "\n") mac = buildEOLData(data2, "\r") - assert.Equal(t, unix, NormalizeEOL(dos)) - assert.Equal(t, unix, NormalizeEOL(mac)) - assert.Equal(t, unix, NormalizeEOL(unix)) + assert.Equal(t, unix, util.NormalizeEOL(dos)) + assert.Equal(t, unix, util.NormalizeEOL(mac)) + assert.Equal(t, unix, util.NormalizeEOL(unix)) - assert.Equal(t, []byte("one liner"), NormalizeEOL([]byte("one liner"))) - assert.Equal(t, []byte("\n"), NormalizeEOL([]byte("\n"))) - assert.Equal(t, []byte("\ntwo liner"), NormalizeEOL([]byte("\ntwo liner"))) - assert.Equal(t, []byte("two liner\n"), NormalizeEOL([]byte("two liner\n"))) - assert.Equal(t, []byte{}, NormalizeEOL([]byte{})) + assert.Equal(t, []byte("one liner"), util.NormalizeEOL([]byte("one liner"))) + assert.Equal(t, []byte("\n"), util.NormalizeEOL([]byte("\n"))) + assert.Equal(t, []byte("\ntwo liner"), util.NormalizeEOL([]byte("\ntwo liner"))) + assert.Equal(t, []byte("two liner\n"), util.NormalizeEOL([]byte("two liner\n"))) + assert.Equal(t, []byte{}, util.NormalizeEOL([]byte{})) - assert.Equal(t, []byte("mix\nand\nmatch\n."), NormalizeEOL([]byte("mix\r\nand\rmatch\n."))) + assert.Equal(t, []byte("mix\nand\nmatch\n."), util.NormalizeEOL([]byte("mix\r\nand\rmatch\n."))) } func Test_RandomInt(t *testing.T) { - randInt, err := CryptoRandomInt(255) + randInt, err := util.CryptoRandomInt(255) assert.GreaterOrEqual(t, randInt, int64(0)) assert.LessOrEqual(t, randInt, int64(255)) require.NoError(t, err) } func Test_RandomString(t *testing.T) { - str1, err := CryptoRandomString(32) + str1, err := util.CryptoRandomString(32) require.NoError(t, err) matches, err := regexp.MatchString(`^[a-zA-Z0-9]{32}$`, str1) require.NoError(t, err) assert.True(t, matches) - str2, err := CryptoRandomString(32) + str2, err := util.CryptoRandomString(32) require.NoError(t, err) matches, err = regexp.MatchString(`^[a-zA-Z0-9]{32}$`, str1) require.NoError(t, err) @@ -143,13 +148,13 @@ func Test_RandomString(t *testing.T) { assert.NotEqual(t, str1, str2) - str3, err := CryptoRandomString(256) + str3, err := util.CryptoRandomString(256) require.NoError(t, err) matches, err = regexp.MatchString(`^[a-zA-Z0-9]{256}$`, str3) require.NoError(t, err) assert.True(t, matches) - str4, err := CryptoRandomString(256) + str4, err := util.CryptoRandomString(256) require.NoError(t, err) matches, err = regexp.MatchString(`^[a-zA-Z0-9]{256}$`, str4) require.NoError(t, err) @@ -159,34 +164,34 @@ func Test_RandomString(t *testing.T) { } func Test_RandomBytes(t *testing.T) { - bytes1, err := CryptoRandomBytes(32) + bytes1, err := util.CryptoRandomBytes(32) require.NoError(t, err) - bytes2, err := CryptoRandomBytes(32) + bytes2, err := util.CryptoRandomBytes(32) require.NoError(t, err) assert.NotEqual(t, bytes1, bytes2) - bytes3, err := CryptoRandomBytes(256) + bytes3, err := util.CryptoRandomBytes(256) require.NoError(t, err) - bytes4, err := CryptoRandomBytes(256) + bytes4, err := util.CryptoRandomBytes(256) require.NoError(t, err) assert.NotEqual(t, bytes3, bytes4) } func TestOptionalBoolParse(t *testing.T) { - assert.Equal(t, optional.None[bool](), OptionalBoolParse("")) - assert.Equal(t, optional.None[bool](), OptionalBoolParse("x")) + assert.Equal(t, optional.None[bool](), util.OptionalBoolParse("")) + assert.Equal(t, optional.None[bool](), util.OptionalBoolParse("x")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("0")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("f")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("False")) + assert.Equal(t, optional.Some(false), util.OptionalBoolParse("0")) + assert.Equal(t, optional.Some(false), util.OptionalBoolParse("f")) + assert.Equal(t, optional.Some(false), util.OptionalBoolParse("False")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("1")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("t")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("True")) + assert.Equal(t, optional.Some(true), util.OptionalBoolParse("1")) + assert.Equal(t, optional.Some(true), util.OptionalBoolParse("t")) + assert.Equal(t, optional.Some(true), util.OptionalBoolParse("True")) } // Test case for any function which accepts and returns a single string. @@ -209,7 +214,7 @@ var upperTests = []StringTest{ func TestToUpperASCII(t *testing.T) { for _, tc := range upperTests { - assert.Equal(t, ToUpperASCII(tc.in), tc.out) + assert.Equal(t, util.ToUpperASCII(tc.in), tc.out) } } @@ -217,27 +222,56 @@ func BenchmarkToUpper(b *testing.B) { for _, tc := range upperTests { b.Run(tc.in, func(b *testing.B) { for i := 0; i < b.N; i++ { - ToUpperASCII(tc.in) + util.ToUpperASCII(tc.in) } }) } } func TestToTitleCase(t *testing.T) { - assert.Equal(t, `Foo Bar Baz`, ToTitleCase(`foo bar baz`)) - assert.Equal(t, `Foo Bar Baz`, ToTitleCase(`FOO BAR BAZ`)) + assert.Equal(t, `Foo Bar Baz`, util.ToTitleCase(`foo bar baz`)) + assert.Equal(t, `Foo Bar Baz`, util.ToTitleCase(`FOO BAR BAZ`)) } func TestToPointer(t *testing.T) { - assert.Equal(t, "abc", *ToPointer("abc")) - assert.Equal(t, 123, *ToPointer(123)) + assert.Equal(t, "abc", *util.ToPointer("abc")) + assert.Equal(t, 123, *util.ToPointer(123)) abc := "abc" - assert.NotSame(t, &abc, ToPointer(abc)) + assert.NotSame(t, &abc, util.ToPointer(abc)) val123 := 123 - assert.NotSame(t, &val123, ToPointer(val123)) + assert.NotSame(t, &val123, util.ToPointer(val123)) } func TestReserveLineBreakForTextarea(t *testing.T) { - assert.Equal(t, "test\ndata", ReserveLineBreakForTextarea("test\r\ndata")) - assert.Equal(t, "test\ndata\n", ReserveLineBreakForTextarea("test\r\ndata\r\n")) + assert.Equal(t, "test\ndata", util.ReserveLineBreakForTextarea("test\r\ndata")) + assert.Equal(t, "test\ndata\n", util.ReserveLineBreakForTextarea("test\r\ndata\r\n")) +} + +const ( + testPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAOhB7/zzhC+HXDdGOdLwJln5NYwm6UNXx3chmQSVTG4\n" + testPrivateKey = `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtz +c2gtZWQyNTUxOQAAACADoQe/884Qvh1w3RjnS8CZZ+TWMJulDV8d3IZkElUxuAAA +AIggISIjICEiIwAAAAtzc2gtZWQyNTUxOQAAACADoQe/884Qvh1w3RjnS8CZZ+TW +MJulDV8d3IZkElUxuAAAAEAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0e +HwOhB7/zzhC+HXDdGOdLwJln5NYwm6UNXx3chmQSVTG4AAAAAAECAwQF +-----END OPENSSH PRIVATE KEY-----` + "\n" +) + +func TestGeneratingEd25519Keypair(t *testing.T) { + defer test.MockProtect(&rand.Reader)() + + // Only 32 bytes needs to be provided to generate a ed25519 keypair. + // And another 32 bytes are required, which is included as random value + // in the OpenSSH format. + b := make([]byte, 64) + for i := 0; i < 64; i++ { + b[i] = byte(i) + } + rand.Reader = bytes.NewReader(b) + + publicKey, privateKey, err := util.GenerateSSHKeypair() + require.NoError(t, err) + assert.EqualValues(t, testPublicKey, string(publicKey)) + assert.EqualValues(t, testPrivateKey, string(privateKey)) } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 148c1eeb9..c6c30b515 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1102,6 +1102,10 @@ mirror_prune = Prune mirror_prune_desc = Remove obsolete remote-tracking references mirror_interval = Mirror interval (valid time units are "h", "m", "s"). 0 to disable periodic sync. (Minimum interval: %s) mirror_interval_invalid = The mirror interval is not valid. +mirror_public_key = Public SSH key +mirror_use_ssh.text = Use SSH authentication +mirror_use_ssh.helper = Forgejo will mirror the repository via Git over SSH and create a keypair for you when you select this option. You must ensure that the generated public key is authorized to push to the destination repository. You cannot use password-based authorization when selecting this. +mirror_denied_combination = Cannot use public key and password based authentication in combination. mirror_sync = synced mirror_sync_on_commit = Sync when commits are pushed mirror_address = Clone from URL @@ -2177,12 +2181,14 @@ settings.mirror_settings.push_mirror.none = No push mirrors configured settings.mirror_settings.push_mirror.remote_url = Git remote repository URL settings.mirror_settings.push_mirror.add = Add push mirror settings.mirror_settings.push_mirror.edit_sync_time = Edit mirror sync interval +settings.mirror_settings.push_mirror.none = None settings.units.units = Repository units settings.units.overview = Overview settings.units.add_more = Add more... settings.sync_mirror = Synchronize now +settings.mirror_settings.push_mirror.copy_public_key = Copy public key settings.pull_mirror_sync_in_progress = Pulling changes from the remote %s at the moment. settings.pull_mirror_sync_quota_exceeded = Quota exceeded, not pulling changes. settings.push_mirror_sync_in_progress = Pushing changes to the remote %s at the moment. diff --git a/release-notes/4819.md b/release-notes/4819.md new file mode 100644 index 000000000..88c3f7732 --- /dev/null +++ b/release-notes/4819.md @@ -0,0 +1 @@ +Allow push mirrors to use a SSH key as the authentication method for the mirroring action instead of using user:password authentication. The SSH keypair is created by Forgejo and the destination repository must be configured with the public key to allow for push over SSH. diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index c0297d77a..9ccf6f05a 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -350,6 +350,11 @@ func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirro return } + if mirrorOption.UseSSH && (mirrorOption.RemoteUsername != "" || mirrorOption.RemotePassword != "") { + ctx.Error(http.StatusBadRequest, "CreatePushMirror", "'use_ssh' is mutually exclusive with 'remote_username' and 'remote_passoword'") + return + } + address, err := forms.ParseRemoteAddr(mirrorOption.RemoteAddress, mirrorOption.RemoteUsername, mirrorOption.RemotePassword) if err == nil { err = migrations.IsMigrateURLAllowed(address, ctx.ContextUser) @@ -365,7 +370,7 @@ func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirro return } - remoteAddress, err := util.SanitizeURL(mirrorOption.RemoteAddress) + remoteAddress, err := util.SanitizeURL(address) if err != nil { ctx.ServerError("SanitizeURL", err) return @@ -380,11 +385,29 @@ func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirro RemoteAddress: remoteAddress, } + var plainPrivateKey []byte + if mirrorOption.UseSSH { + publicKey, privateKey, err := util.GenerateSSHKeypair() + if err != nil { + ctx.ServerError("GenerateSSHKeypair", err) + return + } + plainPrivateKey = privateKey + pushMirror.PublicKey = string(publicKey) + } + if err = db.Insert(ctx, pushMirror); err != nil { ctx.ServerError("InsertPushMirror", err) return } + if mirrorOption.UseSSH { + if err = pushMirror.SetPrivatekey(ctx, plainPrivateKey); err != nil { + ctx.ServerError("SetPrivatekey", err) + return + } + } + // if the registration of the push mirrorOption fails remove it from the database if err = mirror_service.AddPushMirrorRemote(ctx, pushMirror, address); err != nil { if err := repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{ID: pushMirror.ID, RepoID: pushMirror.RepoID}); err != nil { diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index 7da622101..76539b9fa 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -478,8 +478,7 @@ func SettingsPost(ctx *context.Context) { ctx.ServerError("UpdateAddress", err) return } - - remoteAddress, err := util.SanitizeURL(form.MirrorAddress) + remoteAddress, err := util.SanitizeURL(address) if err != nil { ctx.ServerError("SanitizeURL", err) return @@ -638,6 +637,12 @@ func SettingsPost(ctx *context.Context) { return } + if form.PushMirrorUseSSH && (form.PushMirrorUsername != "" || form.PushMirrorPassword != "") { + ctx.Data["Err_PushMirrorUseSSH"] = true + ctx.RenderWithErr(ctx.Tr("repo.mirror_denied_combination"), tplSettingsOptions, &form) + return + } + address, err := forms.ParseRemoteAddr(form.PushMirrorAddress, form.PushMirrorUsername, form.PushMirrorPassword) if err == nil { err = migrations.IsMigrateURLAllowed(address, ctx.Doer) @@ -654,7 +659,7 @@ func SettingsPost(ctx *context.Context) { return } - remoteAddress, err := util.SanitizeURL(form.PushMirrorAddress) + remoteAddress, err := util.SanitizeURL(address) if err != nil { ctx.ServerError("SanitizeURL", err) return @@ -668,11 +673,30 @@ func SettingsPost(ctx *context.Context) { Interval: interval, RemoteAddress: remoteAddress, } + + var plainPrivateKey []byte + if form.PushMirrorUseSSH { + publicKey, privateKey, err := util.GenerateSSHKeypair() + if err != nil { + ctx.ServerError("GenerateSSHKeypair", err) + return + } + plainPrivateKey = privateKey + m.PublicKey = string(publicKey) + } + if err := db.Insert(ctx, m); err != nil { ctx.ServerError("InsertPushMirror", err) return } + if form.PushMirrorUseSSH { + if err := m.SetPrivatekey(ctx, plainPrivateKey); err != nil { + ctx.ServerError("SetPrivatekey", err) + return + } + } + if err := mirror_service.AddPushMirrorRemote(ctx, m, address); err != nil { if err := repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{ID: m.ID, RepoID: m.RepoID}); err != nil { log.Error("DeletePushMirrors %v", err) diff --git a/services/convert/mirror.go b/services/convert/mirror.go index 249ce2f96..85e0d1c85 100644 --- a/services/convert/mirror.go +++ b/services/convert/mirror.go @@ -22,5 +22,6 @@ func ToPushMirror(ctx context.Context, pm *repo_model.PushMirror) (*api.PushMirr LastError: pm.LastError, Interval: pm.Interval.String(), SyncOnCommit: pm.SyncOnCommit, + PublicKey: pm.GetPublicKey(), }, nil } diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index e18bcfdd8..c3d9c3edc 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -6,8 +6,10 @@ package forms import ( + "fmt" "net/http" "net/url" + "regexp" "strings" "code.gitea.io/gitea/models" @@ -88,6 +90,9 @@ func (f *MigrateRepoForm) Validate(req *http.Request, errs binding.Errors) bindi return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } +// scpRegex matches the SCP-like addresses used by Git to access repositories over SSH. +var scpRegex = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) + // ParseRemoteAddr checks if given remote address is valid, // and returns composed URL with needed username and password. func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, error) { @@ -103,7 +108,15 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, err if len(authUsername)+len(authPassword) > 0 { u.User = url.UserPassword(authUsername, authPassword) } - remoteAddr = u.String() + return u.String(), nil + } + + // Detect SCP-like remote addresses and return host. + if m := scpRegex.FindStringSubmatch(remoteAddr); m != nil { + // Match SCP-like syntax and convert it to a URL. + // Eg, "git@forgejo.org:user/repo" becomes + // "ssh://git@forgejo.org/user/repo". + return fmt.Sprintf("ssh://%s@%s/%s", url.User(m[1]), m[2], m[3]), nil } return remoteAddr, nil @@ -127,6 +140,7 @@ type RepoSettingForm struct { PushMirrorPassword string PushMirrorSyncOnCommit bool PushMirrorInterval string + PushMirrorUseSSH bool Private bool Template bool EnablePrune bool diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index de90c5e98..6854a5628 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -71,7 +71,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true} } - if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" { + if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" && u.Scheme != "ssh" { return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 8303c9fb0..3a9644c3a 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "os" "regexp" "strings" "time" @@ -169,11 +170,43 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + // OpenSSH isn't very intuitive when you want to specify a specific keypair. + // Therefore, we need to create a temporary file that stores the private key, so that OpenSSH can use it. + // We delete the the temporary file afterwards. + privateKeyPath := "" + if m.PublicKey != "" { + f, err := os.CreateTemp(os.TempDir(), m.RemoteName) + if err != nil { + log.Error("os.CreateTemp: %v", err) + return errors.New("unexpected error") + } + + defer func() { + f.Close() + if err := os.Remove(f.Name()); err != nil { + log.Error("os.Remove: %v", err) + } + }() + + privateKey, err := m.Privatekey() + if err != nil { + log.Error("Privatekey: %v", err) + return errors.New("unexpected error") + } + + if _, err := f.Write(privateKey); err != nil { + log.Error("f.Write: %v", err) + return errors.New("unexpected error") + } + + privateKeyPath = f.Name() + } if err := git.Push(ctx, path, git.PushOptions{ - Remote: m.RemoteName, - Force: true, - Mirror: true, - Timeout: timeout, + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + PrivateKeyPath: privateKeyPath, }); err != nil { log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index affb7dad4..d37169c07 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -136,6 +136,7 @@ {{ctx.Locale.Tr "repo.settings.mirror_settings.mirrored_repository"}} {{ctx.Locale.Tr "repo.settings.mirror_settings.direction"}} {{ctx.Locale.Tr "repo.settings.mirror_settings.last_update"}} + {{ctx.Locale.Tr "repo.mirror_public_key"}} @@ -233,6 +234,7 @@ {{ctx.Locale.Tr "repo.settings.mirror_settings.pushed_repository"}} {{ctx.Locale.Tr "repo.settings.mirror_settings.direction"}} {{ctx.Locale.Tr "repo.settings.mirror_settings.last_update"}} + {{ctx.Locale.Tr "repo.mirror_public_key"}} @@ -242,7 +244,8 @@ {{.RemoteAddress}} {{ctx.Locale.Tr "repo.settings.mirror_settings.direction.push"}} {{if .LastUpdateUnix}}{{DateTime "full" .LastUpdateUnix}}{{else}}{{ctx.Locale.Tr "never"}}{{end}} {{if .LastError}}
{{ctx.Locale.Tr "error"}}
{{end}} - + {{if not (eq (len .GetPublicKey) 0)}}{{ctx.Locale.Tr "repo.settings.mirror_settings.push_mirror.copy_public_key"}}{{else}}{{ctx.Locale.Tr "repo.settings.mirror_settings.push_mirror.none"}}{{end}} +