finn/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add proxy settings and support for migration and webhook (#16704)

Browse source 
* Add proxy settings and support for migration and webhook

* Fix default value

* Add newline for example ini

* Add lfs proxy support

* Fix lint

* Follow @zeripath's review

* Fix git clone

* Fix test

* missgin http requests for proxy

* use empty

Co-authored-by: zeripath <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
Lunny Xiao 2021-08-18 21:10:39 +08:00 • committed by GitHub
parent 422c30d315
commit f9acad82ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 302 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

51
modules/git/command.go
View file

@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
if timeout == -1 {
timeout = defaultCommandExecutionTimeout
return c.RunWithContext(&RunContext{
Env: env,
Timeout: timeout,
Dir: dir,
Stdout: stdout,
Stderr: stderr,
Stdin: stdin,
PipelineFunc: fn,
})
}
// RunContext represents parameters to run the command
type RunContext struct {
Env []string
Timeout time.Duration
Dir string
Stdout, Stderr io.Writer
Stdin io.Reader
PipelineFunc func(context.Context, context.CancelFunc) error
}
// RunWithContext run the command with context
func (c *Command) RunWithContext(rc *RunContext) error {
if rc.Timeout == -1 {
rc.Timeout = defaultCommandExecutionTimeout
}
if len(dir) == 0 {
if len(rc.Dir) == 0 {
log.Debug("%s", c)
} else {
log.Debug("%s: %v", dir, c)
log.Debug("%s: %v", rc.Dir, c)
}
ctx, cancel := context.WithTimeout(c.parentContext, timeout)
ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
defer cancel()
cmd := exec.CommandContext(ctx, c.name, c.args...)
if env == nil {
if rc.Env == nil {
cmd.Env = os.Environ()
} else {
cmd.Env = env
cmd.Env = rc.Env
}
cmd.Env = append(
@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
if goVersionLessThan115 {
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
}
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
cmd.Stdin = stdin
cmd.Dir = rc.Dir
cmd.Stdout = rc.Stdout
cmd.Stderr = rc.Stderr
cmd.Stdin = rc.Stdin
if err := cmd.Start(); err != nil {
return err
}
desc := c.desc
if desc == "" {
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
}
pid := process.GetManager().Add(desc, cancel)
defer process.GetManager().Remove(pid)
if fn != nil {
err := fn(ctx, cancel)
if rc.PipelineFunc != nil {
err := rc.PipelineFunc(ctx, cancel)
if err != nil {
cancel()
_ = cmd.Wait()

28
modules/git/repo.go
View file

@ -9,11 +9,15 @@ import (
"bytes"
"context"
"fmt"
"io"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/proxy"
)
// GPGSettings represents the default GPG settings for this repository
@ -99,12 +103,12 @@ type CloneRepoOptions struct {
}
// Clone clones original repository to target path.
func Clone(from, to string, opts CloneRepoOptions) (err error) {
func Clone(from, to string, opts CloneRepoOptions) error {
return CloneWithContext(DefaultContext, from, to, opts)
}
// CloneWithContext clones original repository to target path.
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
cargs := make([]string, len(GlobalCommandArgs))
copy(cargs, GlobalCommandArgs)
return CloneWithArgs(ctx, from, to, cargs, opts)
@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
opts.Timeout = -1
}
_, err = cmd.RunTimeout(opts.Timeout)
return err
var envs = os.Environ()
u, err := url.Parse(from)
if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
if proxy.Match(u.Host) {
envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
}
}
var stderr = new(bytes.Buffer)
if err = cmd.RunWithContext(&RunContext{
Timeout: opts.Timeout,
Env: envs,
Stdout: io.Discard,
Stderr: stderr,
}); err != nil {
return ConcatenateError(err, stderr.String())
}
return nil
}
// PullRemoteOptions options when pull from remote

4
modules/lfs/client.go
View file

@ -24,9 +24,9 @@ type Client interface {
}
// NewClient creates a LFS client
func NewClient(endpoint *url.URL) Client {
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
if endpoint.Scheme == "file" {
return newFilesystemClient(endpoint)
}
return newHTTPClient(endpoint)
return newHTTPClient(endpoint, skipTLSVerify)
}

4
modules/lfs/client_test.go
View file

@ -13,10 +13,10 @@ import (
func TestNewClient(t *testing.T) {
u, _ := url.Parse("file:///test")
c := NewClient(u)
c := NewClient(u, true)
assert.IsType(t, &FilesystemClient{}, c)
u, _ = url.Parse("https://test.com/lfs")
c = NewClient(u)
c = NewClient(u, true)
assert.IsType(t, &HTTPClient{}, c)
}

11
modules/lfs/http_client.go
View file

@ -7,6 +7,7 @@ package lfs
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"net/http"
@ -15,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/proxy"
)
const batchSize = 20
@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
return batchSize
}
func newHTTPClient(endpoint *url.URL) *HTTPClient {
hc := &http.Client{}
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
hc := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
Proxy: proxy.Proxy(),
},
}
client := &HTTPClient{
client: hc,

22
modules/migrations/gitea_downloader.go
View file

@ -6,6 +6,7 @@ package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
gitea_sdk "code.gitea.io/sdk/gitea"
@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetContext(ctx),
gitea_sdk.SetHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}),
)
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Attachments {
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
return nil, err
}
// FIXME: for a private download?
resp, err := http.Get(asset.DownloadURL)
req, err := http.NewRequest("GET", asset.DownloadURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}

14
modules/migrations/github.go
View file

@ -7,6 +7,7 @@ package migrations
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
return proxy.Proxy()(req)
},
},
}
@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
r.Published = rel.PublishedAt.Time
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Assets {
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{
@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
resp, err := httpClient.Do(req)
err1 := g.RefreshRate()
if err1 != nil {
log.Error("g.client.RateLimits: %s", err1)

20
modules/migrations/gitlab.go
View file

@ -6,6 +6,7 @@ package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
@ -77,7 +80,12 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}))
// Only use basic auth if token is blank and password is NOT
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
if token == "" && password != "" {
@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
PublisherName: rel.Author.Username,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID),
@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
return nil, err
}
req = req.WithContext(g.ctx)
resp, err := http.DefaultClient.Do(req)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}

6
modules/migrations/gogs.go
View file

@ -6,6 +6,7 @@ package migrations
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/url"
@ -14,6 +15,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client"
@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
downloader.userName = token
} else {
downloader.transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
return proxy.Proxy()(req)
},
}

83
modules/proxy/proxy.go Normal file
View file

@ -0,0 +1,83 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package proxy
import (
"net/http"
"net/url"
"os"
"sync"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/gobwas/glob"
)
var (
once sync.Once
hostMatchers []glob.Glob
)
// GetProxyURL returns proxy url
func GetProxyURL() string {
if !setting.Proxy.Enabled {
return ""
}
if setting.Proxy.ProxyURL == "" {
if os.Getenv("http_proxy") != "" {
return os.Getenv("http_proxy")
}
return os.Getenv("https_proxy")
}
return setting.Proxy.ProxyURL
}
// Match return true if url needs to be proxied
func Match(u string) bool {
if !setting.Proxy.Enabled {
return false
}
// enforce do once
Proxy()
for _, v := range hostMatchers {
if v.Match(u) {
return true
}
}
return false
}
// Proxy returns the system proxy
func Proxy() func(req *http.Request) (*url.URL, error) {
if !setting.Proxy.Enabled {
return nil
}
if setting.Proxy.ProxyURL == "" {
return http.ProxyFromEnvironment
}
once.Do(func() {
for _, h := range setting.Proxy.ProxyHosts {
if g, err := glob.Compile(h); err == nil {
hostMatchers = append(hostMatchers, g)
} else {
log.Error("glob.Compile %s failed: %v", h, err)
}
}
})
return func(req *http.Request) (*url.URL, error) {
for _, v := range hostMatchers {
if v.Match(req.URL.Host) {
return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
}
}
return http.ProxyFromEnvironment(req)
}
}

6
modules/repository/repo.go
View file

@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
if opts.LFS {
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil {
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
log.Error("Failed to store missing LFS objects for repository: %v", err)
}
}
@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
}
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error {
client := lfs.NewClient(endpoint)
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
client := lfs.NewClient(endpoint, skipTLSVerify)
contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob)

2
modules/setting/migrations.go
View file

@ -16,6 +16,7 @@ var (
AllowedDomains []string
BlockedDomains []string
AllowLocalNetworks bool
SkipTLSVerify bool
}{
MaxAttempts: 3,
RetryBackoff: 3,
@ -37,4 +38,5 @@ func newMigrationsService() {
}
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
}

40
modules/setting/proxy.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"net/url"
"code.gitea.io/gitea/modules/log"
)
var (
// Proxy settings
Proxy = struct {
Enabled bool
ProxyURL string
ProxyURLFixed *url.URL
ProxyHosts []string
}{
Enabled: false,
ProxyURL: "",
ProxyHosts: []string{},
}
)
func newProxyService() {
sec := Cfg.Section("proxy")
Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
if Proxy.ProxyURL != "" {
var err error
Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
if err != nil {
log.Error("Global PROXY_URL is not valid")
Proxy.ProxyURL = ""
}
}
Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
}

1
modules/setting/setting.go
View file

@ -1195,6 +1195,7 @@ func NewServices() {
newMailService()
newRegisterMailService()
newNotifyMailService()
newProxyService()
newWebhookService()
newMigrationsService()
newIndexerService()