Make environment-to-ini support loading key value from file (#24832)
Replace #19857 Close #19856 Close #10311 Close #10123 Major changes: 1. Move a lot of code from `environment-to-ini.go` to `config_env.go` to make them testable. 2. Add `__FILE` support 3. Update documents 4. Add testsforgejo-federated-star
parent
1aa9107fea
commit
c21605951b
@ -0,0 +1,142 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package setting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
|
||||||
|
"gopkg.in/ini.v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
|
||||||
|
|
||||||
|
var escapeRegex = regexp.MustCompile(escapeRegexpString)
|
||||||
|
|
||||||
|
// decodeEnvSectionKey will decode a portable string encoded Section__Key pair
|
||||||
|
// Portable strings are considered to be of the form [A-Z0-9_]*
|
||||||
|
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
|
||||||
|
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
|
||||||
|
// Section and Key are separated by a plain '__'.
|
||||||
|
// The entire section can be encoded as a UTF8 byte string
|
||||||
|
func decodeEnvSectionKey(encoded string) (ok bool, section, key string) {
|
||||||
|
inKey := false
|
||||||
|
last := 0
|
||||||
|
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
|
||||||
|
for _, unescapeIdx := range escapeStringIndices {
|
||||||
|
preceding := encoded[last:unescapeIdx[0]]
|
||||||
|
if !inKey {
|
||||||
|
if splitter := strings.Index(preceding, "__"); splitter > -1 {
|
||||||
|
section += preceding[:splitter]
|
||||||
|
inKey = true
|
||||||
|
key += preceding[splitter+2:]
|
||||||
|
} else {
|
||||||
|
section += preceding
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
key += preceding
|
||||||
|
}
|
||||||
|
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
|
||||||
|
decodedBytes := make([]byte, len(toDecode)/2)
|
||||||
|
for i := 0; i < len(toDecode)/2; i++ {
|
||||||
|
// Can ignore error here as we know these should be hexadecimal from the regexp
|
||||||
|
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
|
||||||
|
decodedBytes[i] = byte(byteInt)
|
||||||
|
}
|
||||||
|
if inKey {
|
||||||
|
key += string(decodedBytes)
|
||||||
|
} else {
|
||||||
|
section += string(decodedBytes)
|
||||||
|
}
|
||||||
|
last = unescapeIdx[1]
|
||||||
|
}
|
||||||
|
remaining := encoded[last:]
|
||||||
|
if !inKey {
|
||||||
|
if splitter := strings.Index(remaining, "__"); splitter > -1 {
|
||||||
|
section += remaining[:splitter]
|
||||||
|
key += remaining[splitter+2:]
|
||||||
|
} else {
|
||||||
|
section += remaining
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
key += remaining
|
||||||
|
}
|
||||||
|
section = strings.ToLower(section)
|
||||||
|
ok = section != "" && key != ""
|
||||||
|
if !ok {
|
||||||
|
section = ""
|
||||||
|
key = ""
|
||||||
|
}
|
||||||
|
return ok, section, key
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeEnvironmentKey decode the environment key to section and key
|
||||||
|
// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE
|
||||||
|
func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) {
|
||||||
|
if !strings.HasPrefix(envKey, prefixGitea) {
|
||||||
|
return false, "", "", false
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(envKey, suffixFile) {
|
||||||
|
useFileValue = true
|
||||||
|
envKey = envKey[:len(envKey)-len(suffixFile)]
|
||||||
|
}
|
||||||
|
ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):])
|
||||||
|
return ok, section, key, useFileValue
|
||||||
|
}
|
||||||
|
|
||||||
|
func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) {
|
||||||
|
for _, kv := range envs {
|
||||||
|
idx := strings.IndexByte(kv, '=')
|
||||||
|
if idx < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse the environment variable to config section name and key name
|
||||||
|
envKey := kv[:idx]
|
||||||
|
envValue := kv[idx+1:]
|
||||||
|
ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// use environment value as config value, or read the file content as value if the key indicates a file
|
||||||
|
keyValue := envValue
|
||||||
|
if useFileValue {
|
||||||
|
fileContent, err := os.ReadFile(envValue)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error reading file for %s : %v", envKey, envValue, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keyValue = string(fileContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// try to set the config value if necessary
|
||||||
|
section, err := cfg.GetSection(sectionName)
|
||||||
|
if err != nil {
|
||||||
|
section, err = cfg.NewSection(sectionName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error creating section: %s : %v", sectionName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
key := section.Key(keyName)
|
||||||
|
if key == nil {
|
||||||
|
key, err = section.NewKey(keyName, keyValue)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
oldValue := key.Value()
|
||||||
|
if !changed && oldValue != keyValue {
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
key.SetValue(keyValue)
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
@ -0,0 +1,97 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package setting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"gopkg.in/ini.v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeEnvSectionKey(t *testing.T) {
|
||||||
|
ok, section, key := decodeEnvSectionKey("SEC__KEY")
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sec", section)
|
||||||
|
assert.Equal(t, "KEY", key)
|
||||||
|
|
||||||
|
ok, section, key = decodeEnvSectionKey("sec__key")
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sec", section)
|
||||||
|
assert.Equal(t, "key", key)
|
||||||
|
|
||||||
|
ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR")
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "log.console", section)
|
||||||
|
assert.Equal(t, "STDERR", key)
|
||||||
|
|
||||||
|
ok, section, key = decodeEnvSectionKey("SEC")
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, "", section)
|
||||||
|
assert.Equal(t, "", key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeEnvironmentKey(t *testing.T) {
|
||||||
|
prefix := "GITEA__"
|
||||||
|
suffix := "__FILE"
|
||||||
|
|
||||||
|
ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY")
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, "", section)
|
||||||
|
assert.Equal(t, "", key)
|
||||||
|
assert.False(t, file)
|
||||||
|
|
||||||
|
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC")
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, "", section)
|
||||||
|
assert.Equal(t, "", key)
|
||||||
|
assert.False(t, file)
|
||||||
|
|
||||||
|
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY")
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sec", section)
|
||||||
|
assert.Equal(t, "KEY", key)
|
||||||
|
assert.False(t, file)
|
||||||
|
|
||||||
|
// with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea)
|
||||||
|
// but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either)
|
||||||
|
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE")
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, "", section)
|
||||||
|
assert.Equal(t, "", key)
|
||||||
|
assert.True(t, file)
|
||||||
|
|
||||||
|
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE")
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sec", section)
|
||||||
|
assert.Equal(t, "KEY", key)
|
||||||
|
assert.True(t, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnvironmentToConfig(t *testing.T) {
|
||||||
|
cfg := ini.Empty()
|
||||||
|
|
||||||
|
changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil)
|
||||||
|
assert.False(t, changed)
|
||||||
|
|
||||||
|
cfg, err := ini.Load([]byte(`
|
||||||
|
[sec]
|
||||||
|
key = old
|
||||||
|
`))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
|
||||||
|
assert.True(t, changed)
|
||||||
|
assert.Equal(t, "new", cfg.Section("sec").Key("key").String())
|
||||||
|
|
||||||
|
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
|
||||||
|
assert.False(t, changed)
|
||||||
|
|
||||||
|
tmpFile := t.TempDir() + "/the-file"
|
||||||
|
_ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644)
|
||||||
|
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile})
|
||||||
|
assert.True(t, changed)
|
||||||
|
assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String())
|
||||||
|
}
|
Loading…
Reference in New Issue