Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)

* add migrations

* fix package dependency

* fix lints

* implements migrations except pull requests

* add releases

* migrating releases

* fix bug

* fix lint

* fix migrate releases

* fix tests

* add rollback

* pull request migtations

* fix import

* fix go module vendor

* add tests for upload to gitea

* more migrate options

* fix swagger-check

* fix misspell

* add options on migration UI

* fix log error

* improve UI options on migrating

* add support for username password when migrating from github

* fix tests

* remove comments and fix migrate limitation

* improve error handles

* migrate API will also support migrate milestones/labels/issues/pulls/releases

* fix tests and remove unused codes

* add DownloaderFactory and docs about how to create a new Downloader

* fix misspell

* fix migration docs

* Add hints about migrate options on migration page

* fix tests

modules/auth/repo_form.go

@@ -51,10 +51,16 @@ type MigrateRepoForm struct {
 	// required: true
 	UID int64 `json:"uid" binding:"Required"`
 	// required: true
-	RepoName    string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
-	Mirror      bool   `json:"mirror"`
-	Private     bool   `json:"private"`
-	Description string `json:"description" binding:"MaxSize(255)"`
+	RepoName     string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
+	Mirror       bool   `json:"mirror"`
+	Private      bool   `json:"private"`
+	Description  string `json:"description" binding:"MaxSize(255)"`
+	Wiki         bool   `json:"wiki"`
+	Milestones   bool   `json:"milestones"`
+	Labels       bool   `json:"labels"`
+	Issues       bool   `json:"issues"`
+	PullRequests bool   `json:"pull_requests"`
+	Releases     bool   `json:"releases"`
 }
 
 // Validate validates the fields

modules/migrations/base/comment.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Comment is a standard comment information
+type Comment struct {
+	PosterName  string
+	PosterEmail string
+	Created     time.Time
+	Content     string
+	Reactions   *Reactions
+}

modules/migrations/base/downloader.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Downloader downloads the site repo informations
+type Downloader interface {
+	GetRepoInfo() (*Repository, error)
+	GetMilestones() ([]*Milestone, error)
+	GetReleases() ([]*Release, error)
+	GetLabels() ([]*Label, error)
+	GetIssues(start, limit int) ([]*Issue, error)
+	GetComments(issueNumber int64) ([]*Comment, error)
+	GetPullRequests(start, limit int) ([]*PullRequest, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+	Match(opts MigrateOptions) (bool, error)
+	New(opts MigrateOptions) (Downloader, error)
+}

modules/migrations/base/issue.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Issue is a standard issue information
+type Issue struct {
+	Number      int64
+	PosterName  string
+	PosterEmail string
+	Title       string
+	Content     string
+	Milestone   string
+	State       string // closed, open
+	IsLocked    bool
+	Created     time.Time
+	Closed      *time.Time
+	Labels      []*Label
+	Reactions   *Reactions
+}

modules/migrations/base/label.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Label defines a standard label informations
+type Label struct {
+	Name        string
+	Color       string
+	Description string
+}

modules/migrations/base/milestone.go

@@ -0,0 +1,19 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+	Title       string
+	Description string
+	Deadline    *time.Time
+	Created     time.Time
+	Updated     *time.Time
+	Closed      *time.Time
+	State       string
+}

modules/migrations/base/options.go

@@ -0,0 +1,26 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// MigrateOptions defines the way a repository gets migrated
+type MigrateOptions struct {
+	RemoteURL    string
+	AuthUsername string
+	AuthPassword string
+	Name         string
+	Description  string
+
+	Wiki              bool
+	Issues            bool
+	Milestones        bool
+	Labels            bool
+	Releases          bool
+	Comments          bool
+	PullRequests      bool
+	Private           bool
+	Mirror            bool
+	IgnoreIssueAuthor bool // if true will not add original author information before issues or comments content.
+}

modules/migrations/base/pullrequest.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+	"fmt"
+	"time"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+	Number         int64
+	Title          string
+	PosterName     string
+	PosterEmail    string
+	Content        string
+	Milestone      string
+	State          string
+	Created        time.Time
+	Closed         *time.Time
+	Labels         []*Label
+	PatchURL       string
+	Merged         bool
+	MergedTime     *time.Time
+	MergeCommitSHA string
+	Head           PullRequestBranch
+	Base           PullRequestBranch
+	Assignee       string
+	Assignees      []string
+	IsLocked       bool
+}
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+	return p.Head.RepoPath() != p.Base.RepoPath()
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+	CloneURL  string
+	Ref       string
+	SHA       string
+	RepoName  string
+	OwnerName string
+}
+
+// RepoPath returns pull request repo path
+func (p PullRequestBranch) RepoPath() string {
+	return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}

modules/migrations/base/reaction.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Reactions represents a summary of reactions.
+type Reactions struct {
+	TotalCount int
+	PlusOne    int
+	MinusOne   int
+	Laugh      int
+	Confused   int
+	Heart      int
+	Hooray     int
+}

modules/migrations/base/release.go

@@ -0,0 +1,31 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+	URL           string
+	Name          string
+	ContentType   *string
+	Size          *int
+	DownloadCount *int
+	Created       time.Time
+	Updated       time.Time
+}
+
+// Release represents a release
+type Release struct {
+	TagName         string
+	TargetCommitish string
+	Name            string
+	Body            string
+	Draft           bool
+	Prerelease      bool
+	Assets          []ReleaseAsset
+	Created         time.Time
+	Published       time.Time
+}

modules/migrations/base/repo.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Repository defines a standard repository information
+type Repository struct {
+	Name         string
+	Owner        string
+	IsPrivate    bool
+	IsMirror     bool
+	Description  string
+	AuthUsername string
+	AuthPassword string
+	CloneURL     string
+}

modules/migrations/base/uploader.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Uploader uploads all the informations
+type Uploader interface {
+	CreateRepo(repo *Repository, includeWiki bool) error
+	CreateMilestone(milestone *Milestone) error
+	CreateRelease(release *Release) error
+	CreateLabel(label *Label) error
+	CreateIssue(issue *Issue) error
+	CreateComment(issueNumber int64, comment *Comment) error
+	CreatePullRequest(pr *PullRequest) error
+	Rollback() error
+}

modules/migrations/error.go

@@ -0,0 +1,29 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"errors"
+
+	"github.com/google/go-github/v24/github"
+)
+
+var (
+	// ErrNotSupported returns the error not supported
+	ErrNotSupported = errors.New("not supported")
+)
+
+// IsRateLimitError returns true if the err is github.RateLimitError
+func IsRateLimitError(err error) bool {
+	_, ok := err.(*github.RateLimitError)
+	return ok
+}
+
+// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
+func IsTwoFactorAuthError(err error) bool {
+	_, ok := err.(*github.TwoFactorAuthError)
+	return ok
+}

modules/migrations/git.go

@@ -0,0 +1,69 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"code.gitea.io/gitea/modules/migrations/base"
+)
+
+var (
+	_ base.Downloader = &PlainGitDownloader{}
+)
+
+// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
+type PlainGitDownloader struct {
+	ownerName string
+	repoName  string
+	remoteURL string
+}
+
+// NewPlainGitDownloader creates a git Downloader
+func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
+	return &PlainGitDownloader{
+		ownerName: ownerName,
+		repoName:  repoName,
+		remoteURL: remoteURL,
+	}
+}
+
+// GetRepoInfo returns a repository information
+func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
+	// convert github repo to stand Repo
+	return &base.Repository{
+		Owner:    g.ownerName,
+		Name:     g.repoName,
+		CloneURL: g.remoteURL,
+	}, nil
+}
+
+// GetMilestones returns milestones
+func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) {
+	return nil, ErrNotSupported
+}
+
+// GetLabels returns labels
+func (g *PlainGitDownloader) GetLabels() ([]*base.Label, error) {
+	return nil, ErrNotSupported
+}
+
+// GetReleases returns releases
+func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
+	return nil, ErrNotSupported
+}
+
+// GetIssues returns issues according start and limit
+func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
+	return nil, ErrNotSupported
+}
+
+// GetComments returns comments according issueNumber
+func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
+	return nil, ErrNotSupported
+}
+
+// GetPullRequests returns pull requests according start and limit
+func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
+	return nil, ErrNotSupported
+}

modules/migrations/gitea.go

@@ -0,0 +1,403 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	gouuid "github.com/satori/go.uuid"
+)
+
+var (
+	_ base.Uploader = &GiteaLocalUploader{}
+)
+
+// GiteaLocalUploader implements an Uploader to gitea sites
+type GiteaLocalUploader struct {
+	doer        *models.User
+	repoOwner   string
+	repoName    string
+	repo        *models.Repository
+	labels      sync.Map
+	milestones  sync.Map
+	issues      sync.Map
+	gitRepo     *git.Repository
+	prHeadCache map[string]struct{}
+}
+
+// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
+func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
+	return &GiteaLocalUploader{
+		doer:        doer,
+		repoOwner:   repoOwner,
+		repoName:    repoName,
+		prHeadCache: make(map[string]struct{}),
+	}
+}
+
+// CreateRepo creates a repository
+func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool) error {
+	owner, err := models.GetUserByName(g.repoOwner)
+	if err != nil {
+		return err
+	}
+
+	r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{
+		Name:        g.repoName,
+		Description: repo.Description,
+		IsMirror:    repo.IsMirror,
+		RemoteAddr:  repo.CloneURL,
+		IsPrivate:   repo.IsPrivate,
+		Wiki:        includeWiki,
+	})
+	if err != nil {
+		return err
+	}
+	g.repo = r
+	g.gitRepo, err = git.OpenRepository(r.RepoPath())
+	return err
+}
+
+// CreateMilestone creates milestone
+func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error {
+	var deadline util.TimeStamp
+	if milestone.Deadline != nil {
+		deadline = util.TimeStamp(milestone.Deadline.Unix())
+	}
+	if deadline == 0 {
+		deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix())
+	}
+	var ms = models.Milestone{
+		RepoID:       g.repo.ID,
+		Name:         milestone.Title,
+		Content:      milestone.Description,
+		IsClosed:     milestone.State == "close",
+		DeadlineUnix: deadline,
+	}
+	if ms.IsClosed && milestone.Closed != nil {
+		ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix())
+	}
+	err := models.NewMilestone(&ms)
+
+	if err != nil {
+		return err
+	}
+	g.milestones.Store(ms.Name, ms.ID)
+	return nil
+}
+
+// CreateLabel creates label
+func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error {
+	var lb = models.Label{
+		RepoID:      g.repo.ID,
+		Name:        label.Name,
+		Description: label.Description,
+		Color:       fmt.Sprintf("#%s", label.Color),
+	}
+	err := models.NewLabel(&lb)
+	if err != nil {
+		return err
+	}
+	g.labels.Store(lb.Name, lb.ID)
+	return nil
+}
+
+// CreateRelease creates release
+func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error {
+	var rel = models.Release{
+		RepoID:       g.repo.ID,
+		PublisherID:  g.doer.ID,
+		TagName:      release.TagName,
+		LowerTagName: strings.ToLower(release.TagName),
+		Target:       release.TargetCommitish,
+		Title:        release.Name,
+		Sha1:         release.TargetCommitish,
+		Note:         release.Body,
+		IsDraft:      release.Draft,
+		IsPrerelease: release.Prerelease,
+		IsTag:        false,
+		CreatedUnix:  util.TimeStamp(release.Created.Unix()),
+	}
+
+	// calc NumCommits
+	commit, err := g.gitRepo.GetCommit(rel.TagName)
+	if err != nil {
+		return fmt.Errorf("GetCommit: %v", err)
+	}
+	rel.NumCommits, err = commit.CommitsCount()
+	if err != nil {
+		return fmt.Errorf("CommitsCount: %v", err)
+	}
+
+	for _, asset := range release.Assets {
+		var attach = models.Attachment{
+			UUID:          gouuid.NewV4().String(),
+			Name:          asset.Name,
+			DownloadCount: int64(*asset.DownloadCount),
+			Size:          int64(*asset.Size),
+			CreatedUnix:   util.TimeStamp(asset.Created.Unix()),
+		}
+
+		// download attachment
+		resp, err := http.Get(asset.URL)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		localPath := attach.LocalPath()
+		if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
+			return fmt.Errorf("MkdirAll: %v", err)
+		}
+
+		fw, err := os.Create(localPath)
+		if err != nil {
+			return fmt.Errorf("Create: %v", err)
+		}
+		defer fw.Close()
+
+		if _, err := io.Copy(fw, resp.Body); err != nil {
+			return err
+		}
+
+		rel.Attachments = append(rel.Attachments, &attach)
+	}
+
+	return models.MigrateRelease(&rel)
+}
+
+// CreateIssue creates issue
+func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error {
+	var labelIDs []int64
+	for _, label := range issue.Labels {
+		id, ok := g.labels.Load(label.Name)
+		if !ok {
+			return fmt.Errorf("Label %s missing when create issue", label.Name)
+		}
+		labelIDs = append(labelIDs, id.(int64))
+	}
+
+	var milestoneID int64
+	if issue.Milestone != "" {
+		milestone, ok := g.milestones.Load(issue.Milestone)
+		if !ok {
+			return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone)
+		}
+		milestoneID = milestone.(int64)
+	}
+
+	var is = models.Issue{
+		RepoID:      g.repo.ID,
+		Repo:        g.repo,
+		Index:       issue.Number,
+		PosterID:    g.doer.ID,
+		Title:       issue.Title,
+		Content:     issue.Content,
+		IsClosed:    issue.State == "closed",
+		IsLocked:    issue.IsLocked,
+		MilestoneID: milestoneID,
+		CreatedUnix: util.TimeStamp(issue.Created.Unix()),
+	}
+	if issue.Closed != nil {
+		is.ClosedUnix = util.TimeStamp(issue.Closed.Unix())
+	}
+
+	err := models.InsertIssue(&is, labelIDs)
+	if err != nil {
+		return err
+	}
+	g.issues.Store(issue.Number, is.ID)
+	// TODO: add reactions
+	return err
+}
+
+// CreateComment creates comment
+func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error {
+	var issueID int64
+	if issueIDStr, ok := g.issues.Load(issueNumber); !ok {
+		issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber)
+		if err != nil {
+			return err
+		}
+		issueID = issue.ID
+		g.issues.Store(issueNumber, issueID)
+	} else {
+		issueID = issueIDStr.(int64)
+	}
+
+	var cm = models.Comment{
+		IssueID:     issueID,
+		Type:        models.CommentTypeComment,
+		PosterID:    g.doer.ID,
+		Content:     comment.Content,
+		CreatedUnix: util.TimeStamp(comment.Created.Unix()),
+	}
+	err := models.InsertComment(&cm)
+	// TODO: Reactions
+	return err
+}
+
+// CreatePullRequest creates pull request
+func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
+	var labelIDs []int64
+	for _, label := range pr.Labels {
+		id, ok := g.labels.Load(label.Name)
+		if !ok {
+			return fmt.Errorf("Label %s missing when create issue", label.Name)
+		}
+		labelIDs = append(labelIDs, id.(int64))
+	}
+
+	var milestoneID int64
+	if pr.Milestone != "" {
+		milestone, ok := g.milestones.Load(pr.Milestone)
+		if !ok {
+			return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone)
+		}
+		milestoneID = milestone.(int64)
+	}
+
+	// download patch file
+	resp, err := http.Get(pr.PatchURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
+	if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+		return err
+	}
+	f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// set head information
+	pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
+	if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
+		return err
+	}
+	p, err := os.Create(filepath.Join(pullHead, "head"))
+	if err != nil {
+		return err
+	}
+	defer p.Close()
+	_, err = p.WriteString(pr.Head.SHA)
+	if err != nil {
+		return err
+	}
+
+	var head = "unknown repository"
+	if pr.IsForkPullRequest() {
+		if pr.Head.OwnerName != "" {
+			remote := pr.Head.OwnerName
+			_, ok := g.prHeadCache[remote]
+			if !ok {
+				// git remote add
+				err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+				if err != nil {
+					log.Error("AddRemote failed: %s", err)
+				} else {
+					g.prHeadCache[remote] = struct{}{}
+					ok = true
+				}
+			}
+
+			if ok {
+				_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
+				if err != nil {
+					log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+				} else {
+					headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
+					if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
+						return err
+					}
+					b, err := os.Create(headBranch)
+					if err != nil {
+						return err
+					}
+					defer b.Close()
+					_, err = b.WriteString(pr.Head.SHA)
+					if err != nil {
+						return err
+					}
+					head = pr.Head.OwnerName + "/" + pr.Head.Ref
+				}
+			}
+		}
+	} else {
+		head = pr.Head.Ref
+	}
+
+	var pullRequest = models.PullRequest{
+		HeadRepoID:   g.repo.ID,
+		HeadBranch:   head,
+		HeadUserName: g.repoOwner,
+		BaseRepoID:   g.repo.ID,
+		BaseBranch:   pr.Base.Ref,
+		MergeBase:    pr.Base.SHA,
+		Index:        pr.Number,
+		HasMerged:    pr.Merged,
+
+		Issue: &models.Issue{
+			RepoID:      g.repo.ID,
+			Repo:        g.repo,
+			Title:       pr.Title,
+			Index:       pr.Number,
+			PosterID:    g.doer.ID,
+			Content:     pr.Content,
+			MilestoneID: milestoneID,
+			IsPull:      true,
+			IsClosed:    pr.State == "closed",
+			IsLocked:    pr.IsLocked,
+			CreatedUnix: util.TimeStamp(pr.Created.Unix()),
+		},
+	}
+
+	if pullRequest.Issue.IsClosed && pr.Closed != nil {
+		pullRequest.Issue.ClosedUnix = util.TimeStamp(pr.Closed.Unix())
+	}
+	if pullRequest.HasMerged && pr.MergedTime != nil {
+		pullRequest.MergedUnix = util.TimeStamp(pr.MergedTime.Unix())
+		pullRequest.MergedCommitID = pr.MergeCommitSHA
+		pullRequest.MergerID = g.doer.ID
+	}
+
+	// TODO: reactions
+	// TODO: assignees
+
+	return models.InsertPullRequest(&pullRequest, labelIDs)
+}
+
+// Rollback when migrating failed, this will rollback all the changes.
+func (g *GiteaLocalUploader) Rollback() error {
+	if g.repo != nil && g.repo.ID > 0 {
+		if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
+			return err
+		}
+	}
+	return nil
+}

modules/migrations/gitea_test.go

@@ -0,0 +1,95 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGiteaUploadRepo(t *testing.T) {
+	// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
+	t.Skip()
+
+	models.PrepareTestEnv(t)
+
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
+
+	var (
+		downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
+		repoName   = "builder-" + time.Now().Format("2006-01-02-15-04-05")
+		uploader   = NewGiteaLocalUploader(user, user.Name, repoName)
+	)
+
+	err := migrateRepository(downloader, uploader, MigrateOptions{
+		RemoteURL:    "https://github.com/go-xorm/builder",
+		Name:         repoName,
+		AuthUsername: "",
+
+		Wiki:              true,
+		Issues:            true,
+		Milestones:        true,
+		Labels:            true,
+		Releases:          true,
+		Comments:          true,
+		PullRequests:      true,
+		Private:           true,
+		Mirror:            false,
+		IgnoreIssueAuthor: false,
+	})
+	assert.NoError(t, err)
+
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
+	assert.True(t, repo.HasWiki())
+
+	milestones, err := models.GetMilestones(repo.ID, 0, false, "")
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, len(milestones))
+
+	milestones, err = models.GetMilestones(repo.ID, 0, true, "")
+	assert.NoError(t, err)
+	assert.EqualValues(t, 0, len(milestones))
+
+	labels, err := models.GetLabelsByRepoID(repo.ID, "")
+	assert.NoError(t, err)
+	assert.EqualValues(t, 11, len(labels))
+
+	releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
+		IncludeTags: true,
+	}, 0, 10)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 8, len(releases))
+
+	releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
+		IncludeTags: false,
+	}, 0, 10)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, len(releases))
+
+	issues, err := models.Issues(&models.IssuesOptions{
+		RepoIDs:  []int64{repo.ID},
+		IsPull:   util.OptionalBoolFalse,
+		SortType: "oldest",
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 14, len(issues))
+	assert.NoError(t, issues[0].LoadDiscussComments())
+	assert.EqualValues(t, 0, len(issues[0].Comments))
+
+	pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
+		SortType: "oldest",
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 34, len(pulls))
+	assert.NoError(t, pulls[0].LoadIssue())
+	assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
+	assert.EqualValues(t, 2, len(pulls[0].Issue.Comments))
+}

modules/migrations/github.go

@@ -0,0 +1,475 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+
+	"github.com/google/go-github/v24/github"
+	"golang.org/x/oauth2"
+)
+
+var (
+	_ base.Downloader        = &GithubDownloaderV3{}
+	_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
+)
+
+func init() {
+	RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
+}
+
+// GithubDownloaderV3Factory defines a github downloader v3 factory
+type GithubDownloaderV3Factory struct {
+}
+
+// Match returns ture if the migration remote URL matched this downloader factory
+func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
+	u, err := url.Parse(opts.RemoteURL)
+	if err != nil {
+		return false, err
+	}
+
+	return u.Host == "github.com" && opts.AuthUsername != "", nil
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
+	u, err := url.Parse(opts.RemoteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	fields := strings.Split(u.Path, "/")
+	oldOwner := fields[1]
+	oldName := strings.TrimSuffix(fields[2], ".git")
+
+	log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
+
+	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
+}
+
+// GithubDownloaderV3 implements a Downloader interface to get repository informations
+// from github via APIv3
+type GithubDownloaderV3 struct {
+	ctx       context.Context
+	client    *github.Client
+	repoOwner string
+	repoName  string
+	userName  string
+	password  string
+}
+
+// NewGithubDownloaderV3 creates a github Downloader via github v3 API
+func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
+	var downloader = GithubDownloaderV3{
+		userName:  userName,
+		password:  password,
+		ctx:       context.Background(),
+		repoOwner: repoOwner,
+		repoName:  repoName,
+	}
+
+	var client *http.Client
+	if userName != "" {
+		if password == "" {
+			ts := oauth2.StaticTokenSource(
+				&oauth2.Token{AccessToken: userName},
+			)
+			client = oauth2.NewClient(downloader.ctx, ts)
+		} else {
+			client = &http.Client{
+				Transport: &http.Transport{
+					Proxy: func(req *http.Request) (*url.URL, error) {
+						req.SetBasicAuth(userName, password)
+						return nil, nil
+					},
+				},
+			}
+		}
+	}
+	downloader.client = github.NewClient(client)
+	return &downloader
+}
+
+// GetRepoInfo returns a repository information
+func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
+	gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert github repo to stand Repo
+	return &base.Repository{
+		Owner:       g.repoOwner,
+		Name:        gr.GetName(),
+		IsPrivate:   *gr.Private,
+		Description: gr.GetDescription(),
+		CloneURL:    gr.GetCloneURL(),
+	}, nil
+}
+
+// GetMilestones returns milestones
+func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
+	var perPage = 100
+	var milestones = make([]*base.Milestone, 0, perPage)
+	for i := 1; ; i++ {
+		ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+			&github.MilestoneListOptions{
+				State: "all",
+				ListOptions: github.ListOptions{
+					Page:    i,
+					PerPage: perPage,
+				}})
+		if err != nil {
+			return nil, err
+		}
+
+		for _, m := range ms {
+			var desc string
+			if m.Description != nil {
+				desc = *m.Description
+			}
+			var state = "open"
+			if m.State != nil {
+				state = *m.State
+			}
+			milestones = append(milestones, &base.Milestone{
+				Title:       *m.Title,
+				Description: desc,
+				Deadline:    m.DueOn,
+				State:       state,
+				Created:     *m.CreatedAt,
+				Updated:     m.UpdatedAt,
+				Closed:      m.ClosedAt,
+			})
+		}
+		if len(ms) < perPage {
+			break
+		}
+	}
+	return milestones, nil
+}
+
+func convertGithubLabel(label *github.Label) *base.Label {
+	var desc string
+	if label.Description != nil {
+		desc = *label.Description
+	}
+	return &base.Label{
+		Name:        *label.Name,
+		Color:       *label.Color,
+		Description: desc,
+	}
+}
+
+// GetLabels returns labels
+func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
+	var perPage = 100
+	var labels = make([]*base.Label, 0, perPage)
+	for i := 1; ; i++ {
+		ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+			&github.ListOptions{
+				Page:    i,
+				PerPage: perPage,
+			})
+		if err != nil {
+			return nil, err
+		}
+
+		for _, label := range ls {
+			labels = append(labels, convertGithubLabel(label))
+		}
+		if len(ls) < perPage {
+			break
+		}
+	}
+	return labels, nil
+}
+
+func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
+	var (
+		name string
+		desc string
+	)
+	if rel.Body != nil {
+		desc = *rel.Body
+	}
+	if rel.Name != nil {
+		name = *rel.Name
+	}
+
+	r := &base.Release{
+		TagName:         *rel.TagName,
+		TargetCommitish: *rel.TargetCommitish,
+		Name:            name,
+		Body:            desc,
+		Draft:           *rel.Draft,
+		Prerelease:      *rel.Prerelease,
+		Created:         rel.CreatedAt.Time,
+		Published:       rel.PublishedAt.Time,
+	}
+
+	for _, asset := range rel.Assets {
+		u, _ := url.Parse(*asset.BrowserDownloadURL)
+		u.User = url.UserPassword(g.userName, g.password)
+		r.Assets = append(r.Assets, base.ReleaseAsset{
+			URL:           u.String(),
+			Name:          *asset.Name,
+			ContentType:   asset.ContentType,
+			Size:          asset.Size,
+			DownloadCount: asset.DownloadCount,
+			Created:       asset.CreatedAt.Time,
+			Updated:       asset.UpdatedAt.Time,
+		})
+	}
+	return r
+}
+
+// GetReleases returns releases
+func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
+	var perPage = 100
+	var releases = make([]*base.Release, 0, perPage)
+	for i := 1; ; i++ {
+		ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+			&github.ListOptions{
+				Page:    i,
+				PerPage: perPage,
+			})
+		if err != nil {
+			return nil, err
+		}
+
+		for _, release := range ls {
+			releases = append(releases, g.convertGithubRelease(release))
+		}
+		if len(ls) < perPage {
+			break
+		}
+	}
+	return releases, nil
+}
+
+func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
+	return &base.Reactions{
+		TotalCount: *reactions.TotalCount,
+		PlusOne:    *reactions.PlusOne,
+		MinusOne:   *reactions.MinusOne,
+		Laugh:      *reactions.Laugh,
+		Confused:   *reactions.Confused,
+		Heart:      *reactions.Heart,
+		Hooray:     *reactions.Hooray,
+	}
+}
+
+// GetIssues returns issues according start and limit
+func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) {
+	var perPage = 100
+	opt := &github.IssueListByRepoOptions{
+		Sort:      "created",
+		Direction: "asc",
+		State:     "all",
+		ListOptions: github.ListOptions{
+			PerPage: perPage,
+		},
+	}
+	var allIssues = make([]*base.Issue, 0, limit)
+	for {
+		issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+		if err != nil {
+			return nil, fmt.Errorf("error while listing repos: %v", err)
+		}
+		for _, issue := range issues {
+			if issue.IsPullRequest() {
+				continue
+			}
+			var body string
+			if issue.Body != nil {
+				body = *issue.Body
+			}
+			var milestone string
+			if issue.Milestone != nil {
+				milestone = *issue.Milestone.Title
+			}
+			var labels = make([]*base.Label, 0, len(issue.Labels))
+			for _, l := range issue.Labels {
+				labels = append(labels, convertGithubLabel(&l))
+			}
+			var reactions *base.Reactions
+			if issue.Reactions != nil {
+				reactions = convertGithubReactions(issue.Reactions)
+			}
+
+			var email string
+			if issue.User.Email != nil {
+				email = *issue.User.Email
+			}
+			allIssues = append(allIssues, &base.Issue{
+				Title:       *issue.Title,
+				Number:      int64(*issue.Number),
+				PosterName:  *issue.User.Login,
+				PosterEmail: email,
+				Content:     body,
+				Milestone:   milestone,
+				State:       *issue.State,
+				Created:     *issue.CreatedAt,
+				Labels:      labels,
+				Reactions:   reactions,
+				Closed:      issue.ClosedAt,
+				IsLocked:    *issue.Locked,
+			})
+			if len(allIssues) >= limit {
+				return allIssues, nil
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return allIssues, nil
+}
+
+// GetComments returns comments according issueNumber
+func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
+	var allComments = make([]*base.Comment, 0, 100)
+	opt := &github.IssueListCommentsOptions{
+		Sort:      "created",
+		Direction: "asc",
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+	for {
+		comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
+		if err != nil {
+			return nil, fmt.Errorf("error while listing repos: %v", err)
+		}
+		for _, comment := range comments {
+			var email string
+			if comment.User.Email != nil {
+				email = *comment.User.Email
+			}
+			var reactions *base.Reactions
+			if comment.Reactions != nil {
+				reactions = convertGithubReactions(comment.Reactions)
+			}
+			allComments = append(allComments, &base.Comment{
+				PosterName:  *comment.User.Login,
+				PosterEmail: email,
+				Content:     *comment.Body,
+				Created:     *comment.CreatedAt,
+				Reactions:   reactions,
+			})
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return allComments, nil
+}
+
+// GetPullRequests returns pull requests according start and limit
+func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
+	opt := &github.PullRequestListOptions{
+		Sort:      "created",
+		Direction: "asc",
+		State:     "all",
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+	var allPRs = make([]*base.PullRequest, 0, 100)
+	for {
+		prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+		if err != nil {
+			return nil, fmt.Errorf("error while listing repos: %v", err)
+		}
+		for _, pr := range prs {
+			var body string
+			if pr.Body != nil {
+				body = *pr.Body
+			}
+			var milestone string
+			if pr.Milestone != nil {
+				milestone = *pr.Milestone.Title
+			}
+			var labels = make([]*base.Label, 0, len(pr.Labels))
+			for _, l := range pr.Labels {
+				labels = append(labels, convertGithubLabel(l))
+			}
+
+			// FIXME: This API missing reactions, we may need another extra request to get reactions
+
+			var email string
+			if pr.User.Email != nil {
+				email = *pr.User.Email
+			}
+			var merged bool
+			// pr.Merged is not valid, so use MergedAt to test if it's merged
+			if pr.MergedAt != nil {
+				merged = true
+			}
+
+			var headRepoName string
+			var cloneURL string
+			if pr.Head.Repo != nil {
+				headRepoName = *pr.Head.Repo.Name
+				cloneURL = *pr.Head.Repo.CloneURL
+			}
+			var mergeCommitSHA string
+			if pr.MergeCommitSHA != nil {
+				mergeCommitSHA = *pr.MergeCommitSHA
+			}
+
+			allPRs = append(allPRs, &base.PullRequest{
+				Title:          *pr.Title,
+				Number:         int64(*pr.Number),
+				PosterName:     *pr.User.Login,
+				PosterEmail:    email,
+				Content:        body,
+				Milestone:      milestone,
+				State:          *pr.State,
+				Created:        *pr.CreatedAt,
+				Closed:         pr.ClosedAt,
+				Labels:         labels,
+				Merged:         merged,
+				MergeCommitSHA: mergeCommitSHA,
+				MergedTime:     pr.MergedAt,
+				IsLocked:       pr.ActiveLockReason != nil,
+				Head: base.PullRequestBranch{
+					Ref:       *pr.Head.Ref,
+					SHA:       *pr.Head.SHA,
+					RepoName:  headRepoName,
+					OwnerName: *pr.Head.User.Login,
+					CloneURL:  cloneURL,
+				},
+				Base: base.PullRequestBranch{
+					Ref:       *pr.Base.Ref,
+					SHA:       *pr.Base.SHA,
+					RepoName:  *pr.Base.Repo.Name,
+					OwnerName: *pr.Base.User.Login,
+				},
+				PatchURL: *pr.PatchURL,
+			})
+			if len(allPRs) >= limit {
+				return allPRs, nil
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return allPRs, nil
+}

modules/migrations/github_test.go

@@ -0,0 +1,448 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func assertMilestoneEqual(t *testing.T, title, dueOn, created, updated, closed, state string, ms *base.Milestone) {
+	var tmPtr *time.Time
+	if dueOn != "" {
+		tm, err := time.Parse("2006-01-02 15:04:05 -0700 MST", dueOn)
+		assert.NoError(t, err)
+		tmPtr = &tm
+	}
+	var (
+		createdTM time.Time
+		updatedTM *time.Time
+		closedTM  *time.Time
+	)
+	if created != "" {
+		var err error
+		createdTM, err = time.Parse("2006-01-02 15:04:05 -0700 MST", created)
+		assert.NoError(t, err)
+	}
+	if updated != "" {
+		updatedTemp, err := time.Parse("2006-01-02 15:04:05 -0700 MST", updated)
+		assert.NoError(t, err)
+		updatedTM = &updatedTemp
+	}
+	if closed != "" {
+		closedTemp, err := time.Parse("2006-01-02 15:04:05 -0700 MST", closed)
+		assert.NoError(t, err)
+		closedTM = &closedTemp
+	}
+
+	assert.EqualValues(t, &base.Milestone{
+		Title:    title,
+		Deadline: tmPtr,
+		State:    state,
+		Created:  createdTM,
+		Updated:  updatedTM,
+		Closed:   closedTM,
+	}, ms)
+}
+
+func assertLabelEqual(t *testing.T, name, color string, label *base.Label) {
+	assert.EqualValues(t, &base.Label{
+		Name:  name,
+		Color: color,
+	}, label)
+}
+
+func TestGitHubDownloadRepo(t *testing.T) {
+	downloader := NewGithubDownloaderV3("", "", "go-gitea", "gitea")
+	repo, err := downloader.GetRepoInfo()
+	assert.NoError(t, err)
+	assert.EqualValues(t, &base.Repository{
+		Name:        "gitea",
+		Owner:       "go-gitea",
+		Description: "Git with a cup of tea, painless self-hosted git service",
+		CloneURL:    "https://github.com/go-gitea/gitea.git",
+	}, repo)
+
+	milestones, err := downloader.GetMilestones()
+	assert.NoError(t, err)
+	// before this tool release, we have 39 milestones on github.com/go-gitea/gitea
+	assert.True(t, len(milestones) >= 39)
+
+	for _, milestone := range milestones {
+		switch milestone.Title {
+		case "1.0.0":
+			assertMilestoneEqual(t, "1.0.0", "2016-12-23 08:00:00 +0000 UTC",
+				"2016-11-02 18:06:55 +0000 UTC",
+				"2016-12-29 10:26:00 +0000 UTC",
+				"2016-12-24 00:40:56 +0000 UTC",
+				"closed", milestone)
+		case "1.1.0":
+			assertMilestoneEqual(t, "1.1.0", "2017-02-24 08:00:00 +0000 UTC",
+				"2016-11-03 08:40:10 +0000 UTC",
+				"2017-06-15 05:04:36 +0000 UTC",
+				"2017-03-09 21:22:21 +0000 UTC",
+				"closed", milestone)
+		case "1.2.0":
+			assertMilestoneEqual(t, "1.2.0", "2017-04-24 07:00:00 +0000 UTC",
+				"2016-11-03 08:40:15 +0000 UTC",
+				"2017-12-10 02:43:29 +0000 UTC",
+				"2017-10-12 08:24:28 +0000 UTC",
+				"closed", milestone)
+		case "1.3.0":
+			assertMilestoneEqual(t, "1.3.0", "2017-11-29 08:00:00 +0000 UTC",
+				"2017-03-03 08:08:59 +0000 UTC",
+				"2017-12-04 07:48:44 +0000 UTC",
+				"2017-11-29 18:39:00 +0000 UTC",
+				"closed", milestone)
+		case "1.4.0":
+			assertMilestoneEqual(t, "1.4.0", "2018-01-25 08:00:00 +0000 UTC",
+				"2017-08-23 11:02:37 +0000 UTC",
+				"2018-03-25 20:01:56 +0000 UTC",
+				"2018-03-25 20:01:56 +0000 UTC",
+				"closed", milestone)
+		case "1.5.0":
+			assertMilestoneEqual(t, "1.5.0", "2018-06-15 07:00:00 +0000 UTC",
+				"2017-12-30 04:21:56 +0000 UTC",
+				"2018-09-05 16:34:22 +0000 UTC",
+				"2018-08-11 08:45:01 +0000 UTC",
+				"closed", milestone)
+		case "1.6.0":
+			assertMilestoneEqual(t, "1.6.0", "2018-09-25 07:00:00 +0000 UTC",
+				"2018-05-11 05:37:01 +0000 UTC",
+				"2019-01-27 19:21:22 +0000 UTC",
+				"2018-11-23 13:23:16 +0000 UTC",
+				"closed", milestone)
+		case "1.7.0":
+			assertMilestoneEqual(t, "1.7.0", "2018-12-25 08:00:00 +0000 UTC",
+				"2018-08-28 14:20:14 +0000 UTC",
+				"2019-01-27 11:30:24 +0000 UTC",
+				"2019-01-23 08:58:23 +0000 UTC",
+				"closed", milestone)
+		}
+	}
+
+	labels, err := downloader.GetLabels()
+	assert.NoError(t, err)
+	assert.True(t, len(labels) >= 48)
+	for _, l := range labels {
+		switch l.Name {
+		case "backport/v1.7":
+			assertLabelEqual(t, "backport/v1.7", "fbca04", l)
+		case "backport/v1.8":
+			assertLabelEqual(t, "backport/v1.8", "fbca04", l)
+		case "kind/api":
+			assertLabelEqual(t, "kind/api", "5319e7", l)
+		case "kind/breaking":
+			assertLabelEqual(t, "kind/breaking", "fbca04", l)
+		case "kind/bug":
+			assertLabelEqual(t, "kind/bug", "ee0701", l)
+		case "kind/docs":
+			assertLabelEqual(t, "kind/docs", "c2e0c6", l)
+		case "kind/enhancement":
+			assertLabelEqual(t, "kind/enhancement", "84b6eb", l)
+		case "kind/feature":
+			assertLabelEqual(t, "kind/feature", "006b75", l)
+		}
+	}
+
+	releases, err := downloader.GetReleases()
+	assert.NoError(t, err)
+	assert.EqualValues(t, []*base.Release{
+		{
+			TagName:         "v0.9.99",
+			TargetCommitish: "master",
+			Name:            "fork",
+			Body:            "Forked source from Gogs into Gitea\n",
+			Created:         time.Date(2016, 10, 17, 02, 17, 59, 0, time.UTC),
+			Published:       time.Date(2016, 11, 17, 15, 37, 0, 0, time.UTC),
+		},
+	}, releases[len(releases)-1:])
+
+	// downloader.GetIssues()
+	issues, err := downloader.GetIssues(0, 3)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 3, len(issues))
+	var (
+		closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
+	)
+	assert.EqualValues(t, []*base.Issue{
+		{
+			Number:     6,
+			Title:      "Contribution system: History heatmap for user",
+			Content:    "Hi guys,\r\n\r\nI think that is a possible feature, a history heatmap similar to github or gitlab.\r\nActually exists a plugin called Calendar HeatMap. I used this on mine project to heat application log and worked fine here.\r\nThen, is only a idea, what you think? :)\r\n\r\nhttp://cal-heatmap.com/\r\nhttps://github.com/wa0x6e/cal-heatmap\r\n\r\nReference: https://github.com/gogits/gogs/issues/1640",
+			Milestone:  "1.7.0",
+			PosterName: "joubertredrat",
+			State:      "closed",
+			Created:    time.Date(2016, 11, 02, 18, 51, 55, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/feature",
+					Color: "006b75",
+				},
+				{
+					Name:  "kind/ui",
+					Color: "fef2c0",
+				},
+			},
+			Reactions: &base.Reactions{
+				TotalCount: 0,
+				PlusOne:    0,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   0,
+				Heart:      0,
+				Hooray:     0,
+			},
+			Closed: &closed1,
+		},
+		{
+			Number:     7,
+			Title:      "display page revisions on wiki",
+			Content:    "Hi guys,\r\n\r\nWiki on Gogs is very fine, I liked a lot, but I think that is good idea to be possible see other revisions from page as a page history.\r\n\r\nWhat you think?\r\n\r\nReference: https://github.com/gogits/gogs/issues/2991",
+			Milestone:  "1.x.x",
+			PosterName: "joubertredrat",
+			State:      "open",
+			Created:    time.Date(2016, 11, 02, 18, 57, 32, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/feature",
+					Color: "006b75",
+				},
+				{
+					Name:        "reviewed/confirmed",
+					Color:       "8d9b12",
+					Description: "Issue has been reviewed and confirmed to be present or accepted to be implemented",
+				},
+			},
+			Reactions: &base.Reactions{
+				TotalCount: 6,
+				PlusOne:    5,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   1,
+				Heart:      0,
+				Hooray:     0,
+			},
+		},
+		{
+			Number:     8,
+			Title:      "audit logs",
+			Content:    "Hi,\r\n\r\nI think that is good idea to have user operation log to admin see what the user is doing at Gogs. Similar to example below\r\n\r\n| user | operation | information |\r\n| --- | --- | --- |\r\n| joubertredrat | repo.create | Create repo MyProjectData |\r\n| joubertredrat | user.settings | Edit settings |\r\n| tboerger | repo.fork | Create Fork from MyProjectData to ForkMyProjectData |\r\n| bkcsoft | repo.remove | Remove repo MySource |\r\n| tboerger | admin.auth | Edit auth LDAP org-connection |\r\n\r\nThis resource can be used on user page too, as user activity, set that log row is public (repo._) or private (user._, admin.*) and display only public activity.\r\n\r\nWhat you think?\r\n\r\n[Chat summary from March 14, 2017](https://github.com/go-gitea/gitea/issues/8#issuecomment-286463807)\r\n\r\nReferences:\r\nhttps://github.com/gogits/gogs/issues/3016",
+			Milestone:  "1.x.x",
+			PosterName: "joubertredrat",
+			State:      "open",
+			Created:    time.Date(2016, 11, 02, 18, 59, 20, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/feature",
+					Color: "006b75",
+				},
+				{
+					Name:  "kind/proposal",
+					Color: "5319e7",
+				},
+			},
+			Reactions: &base.Reactions{
+				TotalCount: 9,
+				PlusOne:    8,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   0,
+				Heart:      1,
+				Hooray:     0,
+			},
+		},
+	}, issues)
+
+	// downloader.GetComments()
+	comments, err := downloader.GetComments(6)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 35, len(comments))
+	assert.EqualValues(t, []*base.Comment{
+		{
+			PosterName: "bkcsoft",
+			Created:    time.Date(2016, 11, 02, 18, 59, 48, 0, time.UTC),
+			Content: `I would prefer a solution that is in the backend, unless it's required to have it update without reloading. Unfortunately I can't seem to find anything that does that :unamused: 
+
+Also this would _require_ caching, since it will fetch huge amounts of data from disk...
+`,
+			Reactions: &base.Reactions{
+				TotalCount: 2,
+				PlusOne:    2,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   0,
+				Heart:      0,
+				Hooray:     0,
+			},
+		},
+		{
+			PosterName: "joubertredrat",
+			Created:    time.Date(2016, 11, 02, 19, 16, 56, 0, time.UTC),
+			Content: `Yes, this plugin build on front-end, with backend I don't know too, but we can consider make component for this.
+
+In my case I use ajax to get data, but build on frontend anyway
+`,
+			Reactions: &base.Reactions{
+				TotalCount: 0,
+				PlusOne:    0,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   0,
+				Heart:      0,
+				Hooray:     0,
+			},
+		},
+		{
+			PosterName: "xinity",
+			Created:    time.Date(2016, 11, 03, 13, 04, 56, 0, time.UTC),
+			Content: `following  @bkcsoft retention strategy in cache is a must if we don't want gitea to waste ressources.
+something like in the latest 15days could be enough don't you think ?
+`,
+			Reactions: &base.Reactions{
+				TotalCount: 2,
+				PlusOne:    2,
+				MinusOne:   0,
+				Laugh:      0,
+				Confused:   0,
+				Heart:      0,
+				Hooray:     0,
+			},
+		},
+	}, comments[:3])
+
+	// downloader.GetPullRequests()
+	prs, err := downloader.GetPullRequests(0, 3)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 3, len(prs))
+
+	closed1 = time.Date(2016, 11, 02, 18, 22, 21, 0, time.UTC)
+	var (
+		closed2 = time.Date(2016, 11, 03, 8, 06, 27, 0, time.UTC)
+		closed3 = time.Date(2016, 11, 02, 18, 22, 31, 0, time.UTC)
+	)
+
+	var (
+		merged1 = time.Date(2016, 11, 02, 18, 22, 21, 0, time.UTC)
+		merged2 = time.Date(2016, 11, 03, 8, 06, 27, 0, time.UTC)
+		merged3 = time.Date(2016, 11, 02, 18, 22, 31, 0, time.UTC)
+	)
+	assert.EqualValues(t, []*base.PullRequest{
+		{
+			Number:     1,
+			Title:      "Rename import paths: \"github.com/gogits/gogs\" -> \"github.com/go-gitea/gitea\"",
+			Content:    "",
+			Milestone:  "1.0.0",
+			PosterName: "andreynering",
+			State:      "closed",
+			Created:    time.Date(2016, 11, 02, 17, 01, 19, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/enhancement",
+					Color: "84b6eb",
+				},
+				{
+					Name:  "lgtm/done",
+					Color: "0e8a16",
+				},
+			},
+			PatchURL: "https://github.com/go-gitea/gitea/pull/1.patch",
+			Head: base.PullRequestBranch{
+				Ref:       "import-paths",
+				SHA:       "1b0ec3208db8501acba44a137c009a5a126ebaa9",
+				OwnerName: "andreynering",
+			},
+			Base: base.PullRequestBranch{
+				Ref:       "master",
+				SHA:       "6bcff7828f117af8d51285ce3acba01a7e40a867",
+				OwnerName: "go-gitea",
+				RepoName:  "gitea",
+			},
+			Closed:         &closed1,
+			Merged:         true,
+			MergedTime:     &merged1,
+			MergeCommitSHA: "142d35e8d2baec230ddb565d1265940d59141fab",
+		},
+		{
+			Number:     2,
+			Title:      "Fix sender of issue notifications",
+			Content:    "It is the FROM field in mailer configuration that needs be used,\r\nnot the USER field, which is for authentication.\r\n\r\nMigrated from https://github.com/gogits/gogs/pull/3616\r\n",
+			Milestone:  "1.0.0",
+			PosterName: "strk",
+			State:      "closed",
+			Created:    time.Date(2016, 11, 02, 17, 24, 19, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/bug",
+					Color: "ee0701",
+				},
+				{
+					Name:  "lgtm/done",
+					Color: "0e8a16",
+				},
+			},
+			PatchURL: "https://github.com/go-gitea/gitea/pull/2.patch",
+			Head: base.PullRequestBranch{
+				Ref:       "proper-from-on-issue-mail",
+				SHA:       "af03d00780a6ee70c58e135c6679542cde4f8d50",
+				RepoName:  "gogs",
+				OwnerName: "strk",
+				CloneURL:  "https://github.com/strk/gogs.git",
+			},
+			Base: base.PullRequestBranch{
+				Ref:       "develop",
+				SHA:       "5c5424301443ffa3659737d12de48ab1dfe39a00",
+				OwnerName: "go-gitea",
+				RepoName:  "gitea",
+			},
+			Closed:         &closed2,
+			Merged:         true,
+			MergedTime:     &merged2,
+			MergeCommitSHA: "d8de2beb5b92d02a0597ba7c7803839380666653",
+		},
+		{
+			Number:     3,
+			Title:      "Use proper url for libravatar dep",
+			Content:    "Fetch go-libravatar from its official source, rather than from an unmaintained fork\r\n",
+			Milestone:  "1.0.0",
+			PosterName: "strk",
+			State:      "closed",
+			Created:    time.Date(2016, 11, 02, 17, 34, 31, 0, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name:  "kind/enhancement",
+					Color: "84b6eb",
+				},
+				{
+					Name:  "lgtm/done",
+					Color: "0e8a16",
+				},
+			},
+			PatchURL: "https://github.com/go-gitea/gitea/pull/3.patch",
+			Head: base.PullRequestBranch{
+				Ref:       "libravatar-proper-url",
+				SHA:       "d59a48a2550abd4129b96d38473941b895a4859b",
+				RepoName:  "gogs",
+				OwnerName: "strk",
+				CloneURL:  "https://github.com/strk/gogs.git",
+			},
+			Base: base.PullRequestBranch{
+				Ref:       "develop",
+				SHA:       "6bcff7828f117af8d51285ce3acba01a7e40a867",
+				OwnerName: "go-gitea",
+				RepoName:  "gitea",
+			},
+			Closed:         &closed3,
+			Merged:         true,
+			MergedTime:     &merged3,
+			MergeCommitSHA: "5c5424301443ffa3659737d12de48ab1dfe39a00",
+		},
+	}, prs)
+}

modules/migrations/main_test.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+)
+
+func TestMain(m *testing.M) {
+	models.MainTest(m, filepath.Join("..", ".."))
+}

modules/migrations/migrate.go

@@ -0,0 +1,205 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+)
+
+// MigrateOptions is equal to base.MigrateOptions
+type MigrateOptions = base.MigrateOptions
+
+var (
+	factories []base.DownloaderFactory
+)
+
+// RegisterDownloaderFactory registers a downloader factory
+func RegisterDownloaderFactory(factory base.DownloaderFactory) {
+	factories = append(factories, factory)
+}
+
+// MigrateRepository migrate repository according MigrateOptions
+func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+	var (
+		downloader base.Downloader
+		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.Name)
+	)
+
+	for _, factory := range factories {
+		if match, err := factory.Match(opts); err != nil {
+			return nil, err
+		} else if match {
+			downloader, err = factory.New(opts)
+			if err != nil {
+				return nil, err
+			}
+			break
+		}
+	}
+
+	if downloader == nil {
+		opts.Wiki = true
+		opts.Milestones = false
+		opts.Labels = false
+		opts.Releases = false
+		opts.Comments = false
+		opts.Issues = false
+		opts.PullRequests = false
+		downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL)
+		log.Trace("Will migrate from git: %s", opts.RemoteURL)
+	}
+
+	if err := migrateRepository(downloader, uploader, opts); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		return nil, err
+	}
+
+	return uploader.repo, nil
+}
+
+// migrateRepository will download informations and upload to Uploader, this is a simple
+// process for small repository. For a big repository, save all the data to disk
+// before upload is better
+func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
+	repo, err := downloader.GetRepoInfo()
+	if err != nil {
+		return err
+	}
+	repo.IsPrivate = opts.Private
+	repo.IsMirror = opts.Mirror
+	log.Trace("migrating git data")
+	if err := uploader.CreateRepo(repo, opts.Wiki); err != nil {
+		return err
+	}
+
+	if opts.Milestones {
+		log.Trace("migrating milestones")
+		milestones, err := downloader.GetMilestones()
+		if err != nil {
+			return err
+		}
+
+		for _, milestone := range milestones {
+			if err := uploader.CreateMilestone(milestone); err != nil {
+				return err
+			}
+		}
+	}
+
+	if opts.Labels {
+		log.Trace("migrating labels")
+		labels, err := downloader.GetLabels()
+		if err != nil {
+			return err
+		}
+
+		for _, label := range labels {
+			if err := uploader.CreateLabel(label); err != nil {
+				return err
+			}
+		}
+	}
+
+	if opts.Releases {
+		log.Trace("migrating releases")
+		releases, err := downloader.GetReleases()
+		if err != nil {
+			return err
+		}
+
+		for _, release := range releases {
+			if err := uploader.CreateRelease(release); err != nil {
+				return err
+			}
+		}
+	}
+
+	if opts.Issues {
+		log.Trace("migrating issues and comments")
+		for {
+			issues, err := downloader.GetIssues(0, 100)
+			if err != nil {
+				return err
+			}
+			for _, issue := range issues {
+				if !opts.IgnoreIssueAuthor {
+					issue.Content = fmt.Sprintf("Author: @%s \n\n%s", issue.PosterName, issue.Content)
+				}
+
+				if err := uploader.CreateIssue(issue); err != nil {
+					return err
+				}
+
+				if !opts.Comments {
+					continue
+				}
+
+				comments, err := downloader.GetComments(issue.Number)
+				if err != nil {
+					return err
+				}
+				for _, comment := range comments {
+					if !opts.IgnoreIssueAuthor {
+						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
+					}
+					if err := uploader.CreateComment(issue.Number, comment); err != nil {
+						return err
+					}
+				}
+			}
+
+			if len(issues) < 100 {
+				break
+			}
+		}
+	}
+
+	if opts.PullRequests {
+		log.Trace("migrating pull requests and comments")
+		for {
+			prs, err := downloader.GetPullRequests(0, 100)
+			if err != nil {
+				return err
+			}
+
+			for _, pr := range prs {
+				if !opts.IgnoreIssueAuthor {
+					pr.Content = fmt.Sprintf("Author: @%s \n\n%s", pr.PosterName, pr.Content)
+				}
+				if err := uploader.CreatePullRequest(pr); err != nil {
+					return err
+				}
+				if !opts.Comments {
+					continue
+				}
+
+				comments, err := downloader.GetComments(pr.Number)
+				if err != nil {
+					return err
+				}
+				for _, comment := range comments {
+					if !opts.IgnoreIssueAuthor {
+						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
+					}
+					if err := uploader.CreateComment(pr.Number, comment); err != nil {
+						return err
+					}
+				}
+			}
+			if len(prs) < 100 {
+				break
+			}
+		}
+	}
+
+	return nil
+}