Add push to remote mirror repository (#15157)
* Added push mirror model. * Integrated push mirror into queue. * Moved methods into own file. * Added basic implementation. * Mirror wiki too. * Removed duplicated method. * Get url for different remotes. * Added migration. * Unified remote url access. * Add/Remove push mirror remotes. * Prevent hangs with missing credentials. * Moved code between files. * Changed sanitizer interface. * Added push mirror backend methods. * Only update the mirror remote. * Limit refs on push. * Added UI part. * Added missing table. * Delete mirror if repository gets removed. * Changed signature. Handle object errors. * Added upload method. * Added "upload" unit tests. * Added transfer adapter unit tests. * Send correct headers. * Added pushing of LFS objects. * Added more logging. * Simpler body handling. * Process files in batches to reduce HTTP calls. * Added created timestamp. * Fixed invalid column name. * Changed name to prevent xorm auto setting. * Remove table header im empty. * Strip exit code from error message. * Added docs page about mirroring. * Fixed date. * Fixed merge errors. * Moved test to integrations. * Added push mirror test. * Added test.
This commit is contained in:
KN4CK3R
GitHub
parent
5d113bdd19
commit
440039c0cc
39 changed files with 2468 additions and 885 deletions
|
|
@ -7,13 +7,13 @@ package lfs
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
|
|
@ -30,69 +30,146 @@ func (a *DummyTransferAdapter) Name() string {
|
|||
return "dummy"
|
||||
}
|
||||
|
||||
func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
|
||||
func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) {
|
||||
return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil
|
||||
}
|
||||
|
||||
func TestHTTPClientDownload(t *testing.T) {
|
||||
oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041"
|
||||
size := int64(6)
|
||||
func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
roundTripHandler := func(req *http.Request) *http.Response {
|
||||
url := req.URL.String()
|
||||
if strings.Contains(url, "status-not-ok") {
|
||||
return &http.Response{StatusCode: http.StatusBadRequest}
|
||||
func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func lfsTestRoundtripHandler(req *http.Request) *http.Response {
|
||||
var batchResponse *BatchResponse
|
||||
url := req.URL.String()
|
||||
|
||||
if strings.Contains(url, "status-not-ok") {
|
||||
return &http.Response{StatusCode: http.StatusBadRequest}
|
||||
} else if strings.Contains(url, "invalid-json-response") {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
|
||||
} else if strings.Contains(url, "valid-batch-request-download") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"download": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.Contains(url, "invalid-json-response") {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
|
||||
} else if strings.Contains(url, "valid-batch-request-upload") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"upload": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.Contains(url, "valid-batch-request-download") {
|
||||
assert.Equal(t, "POST", req.Method)
|
||||
assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url)
|
||||
assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url)
|
||||
|
||||
var batchRequest BatchRequest
|
||||
err := json.NewDecoder(req.Body).Decode(&batchRequest)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "download", batchRequest.Operation)
|
||||
assert.Len(t, batchRequest.Objects, 1)
|
||||
assert.Equal(t, oid, batchRequest.Objects[0].Oid)
|
||||
assert.Equal(t, size, batchRequest.Objects[0].Size)
|
||||
|
||||
batchResponse := &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: make([]*ObjectResponse, 1),
|
||||
}
|
||||
|
||||
payload := new(bytes.Buffer)
|
||||
json.NewEncoder(payload).Encode(batchResponse)
|
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
|
||||
} else if strings.Contains(url, "response-no-objects") {
|
||||
batchResponse = &BatchResponse{Transfer: "dummy"}
|
||||
} else if strings.Contains(url, "unknown-transfer-adapter") {
|
||||
batchResponse = &BatchResponse{Transfer: "unknown_adapter"}
|
||||
} else if strings.Contains(url, "error-in-response-objects") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Error: &ObjectError{
|
||||
Code: 404,
|
||||
Message: "Object not found",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.Contains(url, "invalid-response-no-objects") {
|
||||
batchResponse := &BatchResponse{Transfer: "dummy"}
|
||||
|
||||
payload := new(bytes.Buffer)
|
||||
json.NewEncoder(payload).Encode(batchResponse)
|
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
|
||||
} else if strings.Contains(url, "empty-actions-map") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.Contains(url, "unknown-transfer-adapter") {
|
||||
batchResponse := &BatchResponse{Transfer: "unknown_adapter"}
|
||||
|
||||
payload := new(bytes.Buffer)
|
||||
json.NewEncoder(payload).Encode(batchResponse)
|
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
|
||||
} else if strings.Contains(url, "download-actions-map") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"download": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
t.Errorf("Unknown test case: %s", url)
|
||||
|
||||
} else if strings.Contains(url, "upload-actions-map") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"upload": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
} else if strings.Contains(url, "verify-actions-map") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"verify": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
} else if strings.Contains(url, "unknown-actions-map") {
|
||||
batchResponse = &BatchResponse{
|
||||
Transfer: "dummy",
|
||||
Objects: []*ObjectResponse{
|
||||
{
|
||||
Actions: map[string]*Link{
|
||||
"unknown": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
|
||||
payload := new(bytes.Buffer)
|
||||
jsoniter.NewEncoder(payload).Encode(batchResponse)
|
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
|
||||
}
|
||||
|
||||
func TestHTTPClientDownload(t *testing.T) {
|
||||
p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
|
||||
|
||||
hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
|
||||
assert.Equal(t, "POST", req.Method)
|
||||
assert.Equal(t, MediaType, req.Header.Get("Content-type"))
|
||||
assert.Equal(t, MediaType, req.Header.Get("Accept"))
|
||||
|
||||
var batchRequest BatchRequest
|
||||
err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "download", batchRequest.Operation)
|
||||
assert.Equal(t, 1, len(batchRequest.Objects))
|
||||
assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
|
||||
assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
|
||||
|
||||
return lfsTestRoundtripHandler(req)
|
||||
})}
|
||||
dummy := &DummyTransferAdapter{}
|
||||
|
||||
var cases = []struct {
|
||||
|
|
@ -102,12 +179,12 @@ func TestHTTPClientDownload(t *testing.T) {
|
|||
// case 0
|
||||
{
|
||||
endpoint: "https://status-not-ok.io",
|
||||
expectederror: "Unexpected servers response: ",
|
||||
expectederror: "Unexpected server response: ",
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
endpoint: "https://invalid-json-response.io",
|
||||
expectederror: "json.Decode: ",
|
||||
expectederror: "invalid json",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
|
|
@ -116,13 +193,43 @@ func TestHTTPClientDownload(t *testing.T) {
|
|||
},
|
||||
// case 3
|
||||
{
|
||||
endpoint: "https://invalid-response-no-objects.io",
|
||||
expectederror: "No objects in result",
|
||||
endpoint: "https://response-no-objects.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
endpoint: "https://unknown-transfer-adapter.io",
|
||||
expectederror: "Transferadapter not found: ",
|
||||
expectederror: "TransferAdapter not found: ",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
endpoint: "https://error-in-response-objects.io",
|
||||
expectederror: "Object not found",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
endpoint: "https://empty-actions-map.io",
|
||||
expectederror: "Missing action 'download'",
|
||||
},
|
||||
// case 7
|
||||
{
|
||||
endpoint: "https://download-actions-map.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 8
|
||||
{
|
||||
endpoint: "https://upload-actions-map.io",
|
||||
expectederror: "Missing action 'download'",
|
||||
},
|
||||
// case 9
|
||||
{
|
||||
endpoint: "https://verify-actions-map.io",
|
||||
expectederror: "Missing action 'download'",
|
||||
},
|
||||
// case 10
|
||||
{
|
||||
endpoint: "https://unknown-actions-map.io",
|
||||
expectederror: "Missing action 'download'",
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -134,7 +241,116 @@ func TestHTTPClientDownload(t *testing.T) {
|
|||
}
|
||||
client.transfers["dummy"] = dummy
|
||||
|
||||
_, err := client.Download(context.Background(), oid, size)
|
||||
err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
|
||||
if objectError != nil {
|
||||
return objectError
|
||||
}
|
||||
b, err := io.ReadAll(content)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("dummy"), b)
|
||||
return nil
|
||||
})
|
||||
if len(c.expectederror) > 0 {
|
||||
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
|
||||
} else {
|
||||
assert.NoError(t, err, "case %d", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientUpload(t *testing.T) {
|
||||
p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
|
||||
|
||||
hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
|
||||
assert.Equal(t, "POST", req.Method)
|
||||
assert.Equal(t, MediaType, req.Header.Get("Content-type"))
|
||||
assert.Equal(t, MediaType, req.Header.Get("Accept"))
|
||||
|
||||
var batchRequest BatchRequest
|
||||
err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "upload", batchRequest.Operation)
|
||||
assert.Equal(t, 1, len(batchRequest.Objects))
|
||||
assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
|
||||
assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
|
||||
|
||||
return lfsTestRoundtripHandler(req)
|
||||
})}
|
||||
dummy := &DummyTransferAdapter{}
|
||||
|
||||
var cases = []struct {
|
||||
endpoint string
|
||||
expectederror string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
endpoint: "https://status-not-ok.io",
|
||||
expectederror: "Unexpected server response: ",
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
endpoint: "https://invalid-json-response.io",
|
||||
expectederror: "invalid json",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
endpoint: "https://valid-batch-request-upload.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
endpoint: "https://response-no-objects.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
endpoint: "https://unknown-transfer-adapter.io",
|
||||
expectederror: "TransferAdapter not found: ",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
endpoint: "https://error-in-response-objects.io",
|
||||
expectederror: "Object not found",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
endpoint: "https://empty-actions-map.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 7
|
||||
{
|
||||
endpoint: "https://download-actions-map.io",
|
||||
expectederror: "Missing action 'upload'",
|
||||
},
|
||||
// case 8
|
||||
{
|
||||
endpoint: "https://upload-actions-map.io",
|
||||
expectederror: "",
|
||||
},
|
||||
// case 9
|
||||
{
|
||||
endpoint: "https://verify-actions-map.io",
|
||||
expectederror: "Missing action 'upload'",
|
||||
},
|
||||
// case 10
|
||||
{
|
||||
endpoint: "https://unknown-actions-map.io",
|
||||
expectederror: "Missing action 'upload'",
|
||||
},
|
||||
}
|
||||
|
||||
for n, c := range cases {
|
||||
client := &HTTPClient{
|
||||
client: hc,
|
||||
endpoint: c.endpoint,
|
||||
transfers: make(map[string]TransferAdapter),
|
||||
}
|
||||
client.transfers["dummy"] = dummy
|
||||
|
||||
err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
|
||||
return ioutil.NopCloser(new(bytes.Buffer)), objectError
|
||||
})
|
||||
if len(c.expectederror) > 0 {
|
||||
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue