finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add push webhook support for mirrored repositories (#4127)

Browse source
This commit is contained in:
Lauris BH 2018-09-07 05:06:09 +03:00 • committed by techknowlogick
parent bf55276189
commit fa4663e61e
8 changed files with 257 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

68
models/action.go
View file

@ -47,6 +47,9 @@ const (
ActionReopenPullRequest // 15
ActionDeleteTag // 16
ActionDeleteBranch // 17
ActionMirrorSyncPush // 18
ActionMirrorSyncCreate // 19
ActionMirrorSyncDelete // 20
)
var (
@ -736,6 +739,71 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error
return mergePullRequestAction(x, actUser, repo, pull)
}
func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName string, data []byte) error {
if err := notifyWatchers(e, &Action{
ActUserID: repo.OwnerID,
ActUser: repo.MustOwner(),
OpType: opType,
RepoID: repo.ID,
Repo: repo,
IsPrivate: repo.IsPrivate,
RefName: refName,
Content: string(data),
}); err != nil {
return fmt.Errorf("notifyWatchers: %v", err)
}
return nil
}
// MirrorSyncPushActionOptions mirror synchronization action options.
type MirrorSyncPushActionOptions struct {
RefName string
OldCommitID string
NewCommitID string
Commits *PushCommits
}
// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits.
func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error {
if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
}
apiCommits := opts.Commits.ToAPIPayloadCommits(repo.HTMLURL())
opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
apiPusher := repo.MustOwner().APIFormat()
if err := PrepareWebhooks(repo, HookEventPush, &api.PushPayload{
Ref: opts.RefName,
Before: opts.OldCommitID,
After: opts.NewCommitID,
CompareURL: setting.AppURL + opts.Commits.CompareURL,
Commits: apiCommits,
Repo: repo.APIFormat(AccessModeOwner),
Pusher: apiPusher,
Sender: apiPusher,
}); err != nil {
return fmt.Errorf("PrepareWebhooks: %v", err)
}
data, err := json.Marshal(opts.Commits)
if err != nil {
return err
}
return mirrorSyncAction(x, ActionMirrorSyncPush, repo, opts.RefName, data)
}
// MirrorSyncCreateAction adds new action for mirror synchronization of new reference.
func MirrorSyncCreateAction(repo *Repository, refName string) error {
return mirrorSyncAction(x, ActionMirrorSyncCreate, repo, refName, nil)
}
// MirrorSyncDeleteAction adds new action for mirror synchronization of delete reference.
func MirrorSyncDeleteAction(repo *Repository, refName string) error {
return mirrorSyncAction(x, ActionMirrorSyncDelete, repo, refName, nil)
}
// GetFeedsOptions options for retrieving feeds
type GetFeedsOptions struct {
RequestedUser *User

147
models/repo_mirror.go
View file

@ -1,4 +1,5 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@ -6,6 +7,7 @@ package models
import (
"fmt"
"strings"
"time"
"code.gitea.io/git"
@ -119,8 +121,68 @@ func (m *Mirror) SaveAddress(addr string) error {
return cfg.SaveToIndent(configPath, "\t")
}
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
refName string
oldCommitID string
newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
results := make([]*mirrorSyncResult, 0, 3)
lines := strings.Split(output, "\n")
for i := range lines {
// Make sure reference name is presented before continue
idx := strings.Index(lines[i], "-> ")
if idx == -1 {
continue
}
refName := lines[i][idx+3:]
switch {
case strings.HasPrefix(lines[i], " * "): // New reference
results = append(results, &mirrorSyncResult{
refName: refName,
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " - "): // Delete reference
results = append(results, &mirrorSyncResult{
refName: refName,
newCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " "): // New commits of a reference
delimIdx := strings.Index(lines[i][3:], " ")
if delimIdx == -1 {
log.Error(2, "SHA delimiter not found: %q", lines[i])
continue
}
shas := strings.Split(lines[i][3:delimIdx+3], "..")
if len(shas) != 2 {
log.Error(2, "Expect two SHAs but not what found: %q", lines[i])
continue
}
results = append(results, &mirrorSyncResult{
refName: refName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
default:
log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
}
}
return results
}
// runSync returns true if sync finished without error.
func (m *Mirror) runSync() bool {
func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
repoPath := m.Repo.RepoPath()
wikiPath := m.Repo.WikiPath()
timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@ -130,28 +192,30 @@ func (m *Mirror) runSync() bool {
gitArgs = append(gitArgs, "--prune")
}
if _, stderr, err := process.GetManager().ExecDir(
_, stderr, err := process.GetManager().ExecDir(
timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
"git", gitArgs...); err != nil {
"git", gitArgs...)
if err != nil {
// sanitize the output, since it may contain the remote address, which may
// contain a password
message, err := sanitizeOutput(stderr, repoPath)
if err != nil {
log.Error(4, "sanitizeOutput: %v", err)
return false
return nil, false
}
desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
log.Error(4, desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(4, "CreateRepositoryNotice: %v", err)
}
return false
return nil, false
}
output := stderr
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return false
return nil, false
}
if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
@ -170,21 +234,21 @@ func (m *Mirror) runSync() bool {
message, err := sanitizeOutput(stderr, wikiPath)
if err != nil {
log.Error(4, "sanitizeOutput: %v", err)
return false
return nil, false
}
desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
log.Error(4, desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(4, "CreateRepositoryNotice: %v", err)
}
return false
return nil, false
}
}
branches, err := m.Repo.GetBranches()
if err != nil {
log.Error(4, "GetBranches: %v", err)
return false
return nil, false
}
for i := range branches {
@ -192,7 +256,7 @@ func (m *Mirror) runSync() bool {
}
m.UpdatedUnix = util.TimeStampNow()
return true
return parseRemoteUpdateOutput(output), true
}
func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
@ -268,7 +332,8 @@ func SyncMirrors() {
continue
}
if !m.runSync() {
results, ok := m.runSync()
if !ok {
continue
}
@ -278,6 +343,66 @@ func SyncMirrors() {
continue
}
var gitRepo *git.Repository
if len(results) == 0 {
log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
} else {
gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
if err != nil {
log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err)
continue
}
}
for _, result := range results {
// Discard GitHub pull requests, i.e. refs/pull/*
if strings.HasPrefix(result.refName, "refs/pull/") {
continue
}
// Create reference
if result.oldCommitID == gitShortEmptySha {
if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
}
continue
}
// Delete reference
if result.newCommitID == gitShortEmptySha {
if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
}
continue
}
// Push commits
oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
if err != nil {
log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
continue
}
newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
if err != nil {
log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
continue
}
commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
if err != nil {
log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
continue
}
if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{
RefName: result.refName,
OldCommitID: oldCommitID,
NewCommitID: newCommitID,
Commits: ListToPushCommits(commits),
}); err != nil {
log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
continue
}
}
// Get latest commit date and update to current repository updated time
commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
if err != nil {