finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Migrate reviews when migrating repository from github (#9463)

Browse source 
* fix typo

* Migrate reviews when migrating repository from github

* fix lint

* Added test and migration when external user login

* fix test

* fix commented state

* Some improvements

* fix bug when get pull request and ref original author on code comments

* Fix migrated line; Added comment for review

* Don't load all pull requests attributes

* Fix typo

* wrong change copy head

* fix tests

* fix reactions

* Fix test

* fix fmt

* fix review comment reactions
This commit is contained in:
Lunny Xiao 2020-01-24 01:28:15 +08:00 • committed by Lauris BH
parent bfdfa9a8b3
commit f6067a8465
18 changed files with 567 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/migrations/base/downloader.go
View file

@ -23,6 +23,7 @@ type Downloader interface {
GetIssues(page, perPage int) ([]*Issue, bool, error)
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(page, perPage int) ([]*PullRequest, error)
GetReviews(pullRequestNumber int64) ([]*Review, error)
}
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader

44
modules/migrations/base/review.go Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "time"
// enumerate all review states
const (
ReviewStatePending = "PENDING"
ReviewStateApproved = "APPROVED"
ReviewStateChangesRequested = "CHANGES_REQUESTED"
ReviewStateCommented = "COMMENTED"
)
// Review is a standard review information
type Review struct {
ID int64
IssueIndex int64
ReviewerID int64
ReviewerName string
Official bool
CommitID string
Content string
CreatedAt time.Time
State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
Comments []*ReviewComment
}
// ReviewComment represents a review comment
type ReviewComment struct {
ID int64
InReplyTo int64
Content string
TreePath string
DiffHunk string
Position int
CommitID string
PosterID int64
Reactions []*Reaction
CreatedAt time.Time
UpdatedAt time.Time
}

1
modules/migrations/base/uploader.go
View file

@ -17,6 +17,7 @@ type Uploader interface {
CreateIssues(issues ...*Issue) error
CreateComments(comments ...*Comment) error
CreatePullRequests(prs ...*PullRequest) error
CreateReviews(reviews ...*Review) error
Rollback() error
Close()
}

5
modules/migrations/git.go
View file

@ -78,3 +78,8 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported
}
// GetReviews returns reviews according issue number
func (g *PlainGitDownloader) GetReviews(issueNumber int64) ([]*base.Review, error) {
return nil, ErrNotSupported
}

120
modules/migrations/gitea.go
View file

@ -6,6 +6,7 @@
package migrations
import (
"bytes"
"context"
"fmt"
"io"
@ -27,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/services/gitdiff"
gouuid "github.com/satori/go.uuid"
)
@ -48,6 +50,7 @@ type GiteaLocalUploader struct {
gitRepo *git.Repository
prHeadCache map[string]struct{}
userMap map[int64]int64 // external user id mapping to user id
prCache map[int64]*models.PullRequest
gitServiceType structs.GitServiceType
}
@ -60,6 +63,7 @@ func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, re
repoName: repoName,
prHeadCache: make(map[string]struct{}),
userMap: make(map[int64]int64),
prCache: make(map[int64]*models.PullRequest),
}
}
@ -706,6 +710,122 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
return &pullRequest, nil
}
func convertReviewState(state string) models.ReviewType {
switch state {
case base.ReviewStatePending:
return models.ReviewTypePending
case base.ReviewStateApproved:
return models.ReviewTypeApprove
case base.ReviewStateChangesRequested:
return models.ReviewTypeReject
case base.ReviewStateCommented:
return models.ReviewTypeComment
default:
return models.ReviewTypePending
}
}
// CreateReviews create pull request reviews
func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
var cms = make([]*models.Review, 0, len(reviews))
for _, review := range reviews {
var issueID int64
if issueIDStr, ok := g.issues.Load(review.IssueIndex); !ok {
issue, err := models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
if err != nil {
return err
}
issueID = issue.ID
g.issues.Store(review.IssueIndex, issueID)
} else {
issueID = issueIDStr.(int64)
}
userid, ok := g.userMap[review.ReviewerID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[review.ReviewerID] = userid
}
}
var cm = models.Review{
Type: convertReviewState(review.State),
IssueID: issueID,
Content: review.Content,
Official: review.Official,
CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
}
if userid > 0 {
cm.ReviewerID = userid
} else {
cm.ReviewerID = g.doer.ID
cm.OriginalAuthor = review.ReviewerName
cm.OriginalAuthorID = review.ReviewerID
}
// get pr
pr, ok := g.prCache[issueID]
if !ok {
var err error
pr, err = models.GetPullRequestByIssueIDWithNoAttributes(issueID)
if err != nil {
return err
}
g.prCache[issueID] = pr
}
for _, comment := range review.Comments {
headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
if err != nil {
return fmt.Errorf("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err)
}
patchBuf := new(bytes.Buffer)
if err := gitdiff.GetRawDiffForFile(g.gitRepo.Path, pr.MergeBase, headCommitID, gitdiff.RawDiffNormal, comment.TreePath, patchBuf); err != nil {
return fmt.Errorf("GetRawDiffForLine[%s, %s, %s, %s]: %v", g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath, err)
}
_, _, line, _ := gitdiff.ParseDiffHunkString(comment.DiffHunk)
patch := gitdiff.CutDiffAroundLine(patchBuf, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
var c = models.Comment{
Type: models.CommentTypeCode,
PosterID: comment.PosterID,
IssueID: issueID,
Content: comment.Content,
Line: int64(line + comment.Position - 1),
TreePath: comment.TreePath,
CommitSHA: comment.CommitID,
Patch: patch,
CreatedUnix: timeutil.TimeStamp(comment.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
}
if userid > 0 {
c.PosterID = userid
} else {
c.PosterID = g.doer.ID
c.OriginalAuthor = review.ReviewerName
c.OriginalAuthorID = review.ReviewerID
}
cm.Comments = append(cm.Comments, &c)
}
cms = append(cms, &cm)
}
return models.InsertReviews(cms)
}
// Rollback when migrating failed, this will rollback all the changes.
func (g *GiteaLocalUploader) Rollback() error {
if g.repo != nil && g.repo.ID > 0 {

114
modules/migrations/github.go
View file

@ -418,10 +418,14 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
var (
allComments = make([]*base.Comment, 0, 100)
created = "created"
asc = "asc"
)
opt := &github.IssueListCommentsOptions{
Sort: "created",
Direction: "asc",
Sort: created,
Direction: asc,
ListOptions: github.ListOptions{
PerPage: 100,
},
@ -614,3 +618,107 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
return allPRs, nil
}
func convertGithubReview(r *github.PullRequestReview) *base.Review {
return &base.Review{
ID: r.GetID(),
ReviewerID: r.GetUser().GetID(),
ReviewerName: r.GetUser().GetLogin(),
CommitID: r.GetCommitID(),
Content: r.GetBody(),
CreatedAt: r.GetSubmittedAt(),
State: r.GetState(),
}
}
func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
var rcs = make([]*base.ReviewComment, 0, len(cs))
for _, c := range cs {
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
g.sleep()
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i,
PerPage: 100,
})
if err != nil {
return nil, err
}
g.rate = &resp.Rate
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
rcs = append(rcs, &base.ReviewComment{
ID: c.GetID(),
InReplyTo: c.GetInReplyTo(),
Content: c.GetBody(),
TreePath: c.GetPath(),
DiffHunk: c.GetDiffHunk(),
Position: c.GetPosition(),
CommitID: c.GetCommitID(),
PosterID: c.GetUser().GetID(),
Reactions: reactions,
CreatedAt: c.GetCreatedAt(),
UpdatedAt: c.GetUpdatedAt(),
})
}
return rcs, nil
}
// GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
var allReviews = make([]*base.Review, 0, 100)
opt := &github.ListOptions{
PerPage: 100,
}
for {
g.sleep()
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, review := range reviews {
r := convertGithubReview(review)
r.IssueIndex = pullRequestNumber
// retrieve all review comments
opt2 := &github.ListOptions{
PerPage: 100,
}
for {
g.sleep()
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
cs, err := g.convertGithubReviewComments(reviewComments)
if err != nil {
return nil, err
}
r.Comments = append(r.Comments, cs...)
if resp.NextPage == 0 {
break
}
opt2.Page = resp.NextPage
}
allReviews = append(allReviews, r)
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allReviews, nil
}

91
modules/migrations/github_test.go
View file

@ -361,4 +361,95 @@ func TestGitHubDownloadRepo(t *testing.T) {
},
},
}, prs)
reviews, err := downloader.GetReviews(3)
assert.NoError(t, err)
assert.EqualValues(t, []*base.Review{
{
ID: 315859956,
IssueIndex: 3,
ReviewerID: 42128690,
ReviewerName: "jolheiser",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 35, 24, 0, time.UTC),
State: base.ReviewStateApproved,
},
{
ID: 315860062,
IssueIndex: 3,
ReviewerID: 1824502,
ReviewerName: "zeripath",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 35, 36, 0, time.UTC),
State: base.ReviewStateApproved,
},
{
ID: 315861440,
IssueIndex: 3,
ReviewerID: 165205,
ReviewerName: "lafriks",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 38, 00, 0, time.UTC),
State: base.ReviewStateApproved,
},
}, reviews)
reviews, err = downloader.GetReviews(4)
assert.NoError(t, err)
assert.EqualValues(t, []*base.Review{
{
ID: 338338740,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
State: base.ReviewStateApproved,
Comments: []*base.ReviewComment{
{
ID: 363017488,
Content: "This is a good pull request.",
TreePath: "README.md",
DiffHunk: "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+",
Position: 3,
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
PosterID: 81045,
CreatedAt: time.Date(2020, 01, 04, 05, 33, 06, 0, time.UTC),
UpdatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
},
},
},
{
ID: 338339651,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 06, 07, 06, 0, time.UTC),
State: base.ReviewStateChangesRequested,
Content: "Don't add more reviews",
},
{
ID: 338349019,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
State: base.ReviewStateCommented,
Comments: []*base.ReviewComment{
{
ID: 363029944,
Content: "test a single comment.",
TreePath: "LICENSE",
DiffHunk: "@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n+",
Position: 4,
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
PosterID: 81045,
CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
UpdatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
},
},
},
}, reviews)
}

29
modules/migrations/migrate.go
View file

@ -181,7 +181,10 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
var commentBatchSize = uploader.MaxBatchInsertSize("comment")
var (
commentBatchSize = uploader.MaxBatchInsertSize("comment")
reviewBatchSize = uploader.MaxBatchInsertSize("review")
)
if opts.Issues {
log.Trace("migrating issues and comments")
@ -248,6 +251,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
continue
}
// plain comments
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, pr := range prs {
comments, err := downloader.GetComments(pr.Number)
@ -270,6 +274,29 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
// migrate reviews
var allReviews = make([]*base.Review, 0, reviewBatchSize)
for _, pr := range prs {
reviews, err := downloader.GetReviews(pr.Number)
if err != nil {
return err
}
allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize {
if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
return err
}
allReviews = allReviews[reviewBatchSize:]
}
}
if len(allReviews) > 0 {
if err := uploader.CreateReviews(allReviews...); err != nil {
return err
}
}
if len(prs) < prBatchSize {
break
}