Set self-adjusting deadline for connection writing (#16068)
* Set self-adjusting deadline for connection writing In #16055 it appears that the simple 5s deadline doesn't work for large file writes. Now we can't - or at least shouldn't just set no deadline as go will happily let these connections block indefinitely. However, what seems reasonable is to set some minimum rate we expect for writing. This PR suggests the following algorithm: * Every write has a minimum timeout of 5s (adjustable at compile time.) * If there has been a previous write - then consider its previous deadline, add half of the minimum timeout + 2s per kb about to written. * If that new deadline is after the minimum timeout use that. Fix #16055 * Linearly increase timeout * Make PerWriteTimeout, PerWritePerKbTimeouts configurable Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
zeripath
GitHub
parent
86e2789960
commit
daa5a23548
5 changed files with 90 additions and 39 deletions
|
|
@ -17,6 +17,7 @@ import (
|
|||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -26,11 +27,12 @@ var (
|
|||
DefaultWriteTimeOut time.Duration
|
||||
// DefaultMaxHeaderBytes default max header bytes
|
||||
DefaultMaxHeaderBytes int
|
||||
// PerWriteWriteTimeout timeout for writes
|
||||
PerWriteWriteTimeout = 30 * time.Second
|
||||
// PerWriteWriteTimeoutKbTime is a timeout taking account of how much there is to be written
|
||||
PerWriteWriteTimeoutKbTime = 10 * time.Second
|
||||
)
|
||||
|
||||
// PerWriteWriteTimeout timeout for writes
|
||||
const PerWriteWriteTimeout = 5 * time.Second
|
||||
|
||||
func init() {
|
||||
DefaultMaxHeaderBytes = 0 // use http.DefaultMaxHeaderBytes - which currently is 1 << 20 (1MB)
|
||||
}
|
||||
|
|
@ -40,14 +42,16 @@ type ServeFunction = func(net.Listener) error
|
|||
|
||||
// Server represents our graceful server
|
||||
type Server struct {
|
||||
network string
|
||||
address string
|
||||
listener net.Listener
|
||||
wg sync.WaitGroup
|
||||
state state
|
||||
lock *sync.RWMutex
|
||||
BeforeBegin func(network, address string)
|
||||
OnShutdown func()
|
||||
network string
|
||||
address string
|
||||
listener net.Listener
|
||||
wg sync.WaitGroup
|
||||
state state
|
||||
lock *sync.RWMutex
|
||||
BeforeBegin func(network, address string)
|
||||
OnShutdown func()
|
||||
PerWriteTimeout time.Duration
|
||||
PerWritePerKbTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewServer creates a server on network at provided address
|
||||
|
|
@ -58,11 +62,13 @@ func NewServer(network, address, name string) *Server {
|
|||
log.Info("Starting new %s server: %s:%s on PID: %d", name, network, address, os.Getpid())
|
||||
}
|
||||
srv := &Server{
|
||||
wg: sync.WaitGroup{},
|
||||
state: stateInit,
|
||||
lock: &sync.RWMutex{},
|
||||
network: network,
|
||||
address: address,
|
||||
wg: sync.WaitGroup{},
|
||||
state: stateInit,
|
||||
lock: &sync.RWMutex{},
|
||||
network: network,
|
||||
address: address,
|
||||
PerWriteTimeout: setting.PerWriteTimeout,
|
||||
PerWritePerKbTimeout: setting.PerWritePerKbTimeout,
|
||||
}
|
||||
|
||||
srv.BeforeBegin = func(network, addr string) {
|
||||
|
|
@ -224,9 +230,11 @@ func (wl *wrappedListener) Accept() (net.Conn, error) {
|
|||
closed := int32(0)
|
||||
|
||||
c = wrappedConn{
|
||||
Conn: c,
|
||||
server: wl.server,
|
||||
closed: &closed,
|
||||
Conn: c,
|
||||
server: wl.server,
|
||||
closed: &closed,
|
||||
perWriteTimeout: wl.server.PerWriteTimeout,
|
||||
perWritePerKbTimeout: wl.server.PerWritePerKbTimeout,
|
||||
}
|
||||
|
||||
wl.server.wg.Add(1)
|
||||
|
|
@ -249,13 +257,23 @@ func (wl *wrappedListener) File() (*os.File, error) {
|
|||
|
||||
type wrappedConn struct {
|
||||
net.Conn
|
||||
server *Server
|
||||
closed *int32
|
||||
server *Server
|
||||
closed *int32
|
||||
deadline time.Time
|
||||
perWriteTimeout time.Duration
|
||||
perWritePerKbTimeout time.Duration
|
||||
}
|
||||
|
||||
func (w wrappedConn) Write(p []byte) (n int, err error) {
|
||||
if PerWriteWriteTimeout > 0 {
|
||||
_ = w.Conn.SetWriteDeadline(time.Now().Add(PerWriteWriteTimeout))
|
||||
if w.perWriteTimeout > 0 {
|
||||
minTimeout := time.Duration(len(p)/1024) * w.perWritePerKbTimeout
|
||||
minDeadline := time.Now().Add(minTimeout).Add(w.perWriteTimeout)
|
||||
|
||||
w.deadline = w.deadline.Add(minTimeout)
|
||||
if minDeadline.After(w.deadline) {
|
||||
w.deadline = minDeadline
|
||||
}
|
||||
_ = w.Conn.SetWriteDeadline(w.deadline)
|
||||
}
|
||||
return w.Conn.Write(p)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ var (
|
|||
GracefulRestartable bool
|
||||
GracefulHammerTime time.Duration
|
||||
StartupTimeout time.Duration
|
||||
PerWriteTimeout = 30 * time.Second
|
||||
PerWritePerKbTimeout = 10 * time.Second
|
||||
StaticURLPrefix string
|
||||
AbsoluteAssetURL string
|
||||
|
||||
|
|
@ -147,18 +149,22 @@ var (
|
|||
TrustedUserCAKeys []string `ini:"SSH_TRUSTED_USER_CA_KEYS"`
|
||||
TrustedUserCAKeysFile string `ini:"SSH_TRUSTED_USER_CA_KEYS_FILENAME"`
|
||||
TrustedUserCAKeysParsed []gossh.PublicKey `ini:"-"`
|
||||
PerWriteTimeout time.Duration `ini:"SSH_PER_WRITE_TIMEOUT"`
|
||||
PerWritePerKbTimeout time.Duration `ini:"SSH_PER_WRITE_PER_KB_TIMEOUT"`
|
||||
}{
|
||||
Disabled: false,
|
||||
StartBuiltinServer: false,
|
||||
Domain: "",
|
||||
Port: 22,
|
||||
ServerCiphers: []string{"aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "arcfour256", "arcfour128"},
|
||||
ServerKeyExchanges: []string{"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "curve25519-sha256@libssh.org"},
|
||||
ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1", "hmac-sha1-96"},
|
||||
KeygenPath: "ssh-keygen",
|
||||
MinimumKeySizeCheck: true,
|
||||
MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 2048},
|
||||
ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"},
|
||||
Disabled: false,
|
||||
StartBuiltinServer: false,
|
||||
Domain: "",
|
||||
Port: 22,
|
||||
ServerCiphers: []string{"aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "arcfour256", "arcfour128"},
|
||||
ServerKeyExchanges: []string{"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "curve25519-sha256@libssh.org"},
|
||||
ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1", "hmac-sha1-96"},
|
||||
KeygenPath: "ssh-keygen",
|
||||
MinimumKeySizeCheck: true,
|
||||
MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 2048},
|
||||
ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"},
|
||||
PerWriteTimeout: PerWriteTimeout,
|
||||
PerWritePerKbTimeout: PerWritePerKbTimeout,
|
||||
}
|
||||
|
||||
// Security settings
|
||||
|
|
@ -612,6 +618,8 @@ func NewContext() {
|
|||
GracefulRestartable = sec.Key("ALLOW_GRACEFUL_RESTARTS").MustBool(true)
|
||||
GracefulHammerTime = sec.Key("GRACEFUL_HAMMER_TIME").MustDuration(60 * time.Second)
|
||||
StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(0 * time.Second)
|
||||
PerWriteTimeout = sec.Key("PER_WRITE_TIMEOUT").MustDuration(PerWriteTimeout)
|
||||
PerWritePerKbTimeout = sec.Key("PER_WRITE_PER_KB_TIMEOUT").MustDuration(PerWritePerKbTimeout)
|
||||
|
||||
defaultAppURL := string(Protocol) + "://" + Domain
|
||||
if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") {
|
||||
|
|
@ -777,6 +785,8 @@ func NewContext() {
|
|||
}
|
||||
|
||||
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
|
||||
SSH.PerWriteTimeout = sec.Key("SSH_PER_WRITE_TIMEOUT").MustDuration(PerWriteTimeout)
|
||||
SSH.PerWritePerKbTimeout = sec.Key("SSH_PER_WRITE_PER_KB_TIMEOUT").MustDuration(PerWritePerKbTimeout)
|
||||
|
||||
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
|
||||
log.Fatal("Failed to OAuth2 settings: %v", err)
|
||||
|
|
|
|||
|
|
@ -7,12 +7,15 @@ package ssh
|
|||
import (
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"github.com/gliderlabs/ssh"
|
||||
)
|
||||
|
||||
func listen(server *ssh.Server) {
|
||||
gracefulServer := graceful.NewServer("tcp", server.Addr, "SSH")
|
||||
gracefulServer.PerWriteTimeout = setting.SSH.PerWriteTimeout
|
||||
gracefulServer.PerWritePerKbTimeout = setting.SSH.PerWritePerKbTimeout
|
||||
|
||||
err := gracefulServer.ListenAndServe(server.Serve)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue