Make environment-to-ini support loading key value from file (#24832)
Replace #19857 Close #19856 Close #10311 Close #10123 Major changes: 1. Move a lot of code from `environment-to-ini.go` to `config_env.go` to make them testable. 2. Add `__FILE` support 3. Update documents 4. Add tests
This commit is contained in:
wxiaoguang
GitHub
parent
1aa9107fea
commit
c21605951b
5 changed files with 276 additions and 109 deletions
142
modules/setting/config_env.go
Normal file
142
modules/setting/config_env.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package setting
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
|
||||
|
||||
var escapeRegex = regexp.MustCompile(escapeRegexpString)
|
||||
|
||||
// decodeEnvSectionKey will decode a portable string encoded Section__Key pair
|
||||
// Portable strings are considered to be of the form [A-Z0-9_]*
|
||||
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
|
||||
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
|
||||
// Section and Key are separated by a plain '__'.
|
||||
// The entire section can be encoded as a UTF8 byte string
|
||||
func decodeEnvSectionKey(encoded string) (ok bool, section, key string) {
|
||||
inKey := false
|
||||
last := 0
|
||||
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
|
||||
for _, unescapeIdx := range escapeStringIndices {
|
||||
preceding := encoded[last:unescapeIdx[0]]
|
||||
if !inKey {
|
||||
if splitter := strings.Index(preceding, "__"); splitter > -1 {
|
||||
section += preceding[:splitter]
|
||||
inKey = true
|
||||
key += preceding[splitter+2:]
|
||||
} else {
|
||||
section += preceding
|
||||
}
|
||||
} else {
|
||||
key += preceding
|
||||
}
|
||||
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
|
||||
decodedBytes := make([]byte, len(toDecode)/2)
|
||||
for i := 0; i < len(toDecode)/2; i++ {
|
||||
// Can ignore error here as we know these should be hexadecimal from the regexp
|
||||
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
|
||||
decodedBytes[i] = byte(byteInt)
|
||||
}
|
||||
if inKey {
|
||||
key += string(decodedBytes)
|
||||
} else {
|
||||
section += string(decodedBytes)
|
||||
}
|
||||
last = unescapeIdx[1]
|
||||
}
|
||||
remaining := encoded[last:]
|
||||
if !inKey {
|
||||
if splitter := strings.Index(remaining, "__"); splitter > -1 {
|
||||
section += remaining[:splitter]
|
||||
key += remaining[splitter+2:]
|
||||
} else {
|
||||
section += remaining
|
||||
}
|
||||
} else {
|
||||
key += remaining
|
||||
}
|
||||
section = strings.ToLower(section)
|
||||
ok = section != "" && key != ""
|
||||
if !ok {
|
||||
section = ""
|
||||
key = ""
|
||||
}
|
||||
return ok, section, key
|
||||
}
|
||||
|
||||
// decodeEnvironmentKey decode the environment key to section and key
|
||||
// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE
|
||||
func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) {
|
||||
if !strings.HasPrefix(envKey, prefixGitea) {
|
||||
return false, "", "", false
|
||||
}
|
||||
if strings.HasSuffix(envKey, suffixFile) {
|
||||
useFileValue = true
|
||||
envKey = envKey[:len(envKey)-len(suffixFile)]
|
||||
}
|
||||
ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):])
|
||||
return ok, section, key, useFileValue
|
||||
}
|
||||
|
||||
func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) {
|
||||
for _, kv := range envs {
|
||||
idx := strings.IndexByte(kv, '=')
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// parse the environment variable to config section name and key name
|
||||
envKey := kv[:idx]
|
||||
envValue := kv[idx+1:]
|
||||
ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// use environment value as config value, or read the file content as value if the key indicates a file
|
||||
keyValue := envValue
|
||||
if useFileValue {
|
||||
fileContent, err := os.ReadFile(envValue)
|
||||
if err != nil {
|
||||
log.Error("Error reading file for %s : %v", envKey, envValue, err)
|
||||
continue
|
||||
}
|
||||
keyValue = string(fileContent)
|
||||
}
|
||||
|
||||
// try to set the config value if necessary
|
||||
section, err := cfg.GetSection(sectionName)
|
||||
if err != nil {
|
||||
section, err = cfg.NewSection(sectionName)
|
||||
if err != nil {
|
||||
log.Error("Error creating section: %s : %v", sectionName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
key := section.Key(keyName)
|
||||
if key == nil {
|
||||
key, err = section.NewKey(keyName, keyValue)
|
||||
if err != nil {
|
||||
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
oldValue := key.Value()
|
||||
if !changed && oldValue != keyValue {
|
||||
changed = true
|
||||
}
|
||||
key.SetValue(keyValue)
|
||||
}
|
||||
return changed
|
||||
}
|
||||
97
modules/setting/config_env_test.go
Normal file
97
modules/setting/config_env_test.go
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package setting
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
func TestDecodeEnvSectionKey(t *testing.T) {
|
||||
ok, section, key := decodeEnvSectionKey("SEC__KEY")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "sec", section)
|
||||
assert.Equal(t, "KEY", key)
|
||||
|
||||
ok, section, key = decodeEnvSectionKey("sec__key")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "sec", section)
|
||||
assert.Equal(t, "key", key)
|
||||
|
||||
ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "log.console", section)
|
||||
assert.Equal(t, "STDERR", key)
|
||||
|
||||
ok, section, key = decodeEnvSectionKey("SEC")
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, "", section)
|
||||
assert.Equal(t, "", key)
|
||||
}
|
||||
|
||||
func TestDecodeEnvironmentKey(t *testing.T) {
|
||||
prefix := "GITEA__"
|
||||
suffix := "__FILE"
|
||||
|
||||
ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY")
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, "", section)
|
||||
assert.Equal(t, "", key)
|
||||
assert.False(t, file)
|
||||
|
||||
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC")
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, "", section)
|
||||
assert.Equal(t, "", key)
|
||||
assert.False(t, file)
|
||||
|
||||
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "sec", section)
|
||||
assert.Equal(t, "KEY", key)
|
||||
assert.False(t, file)
|
||||
|
||||
// with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea)
|
||||
// but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either)
|
||||
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE")
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, "", section)
|
||||
assert.Equal(t, "", key)
|
||||
assert.True(t, file)
|
||||
|
||||
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "sec", section)
|
||||
assert.Equal(t, "KEY", key)
|
||||
assert.True(t, file)
|
||||
}
|
||||
|
||||
func TestEnvironmentToConfig(t *testing.T) {
|
||||
cfg := ini.Empty()
|
||||
|
||||
changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil)
|
||||
assert.False(t, changed)
|
||||
|
||||
cfg, err := ini.Load([]byte(`
|
||||
[sec]
|
||||
key = old
|
||||
`))
|
||||
assert.NoError(t, err)
|
||||
|
||||
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "new", cfg.Section("sec").Key("key").String())
|
||||
|
||||
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
|
||||
assert.False(t, changed)
|
||||
|
||||
tmpFile := t.TempDir() + "/the-file"
|
||||
_ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644)
|
||||
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile})
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue