From df68c0ac384a407a73032e08ace4b0fc126ea71e Mon Sep 17 00:00:00 2001 From: jacobstr Date: Wed, 22 Oct 2014 02:27:34 -0600 Subject: [PATCH] Materialized path key traversal. Allows deep access of config settings via a dot-delimited materialized path. Environment variables may also be set to corresponding deep keys using double underscores in place of the usual period delimiter. E.g: clothing.jacket => CLOTHING__JACKET --- viper.go | 85 ++++++++++++++++++++++++++++++++++++++++----------- viper_test.go | 18 ++++++++++- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/viper.go b/viper.go index e658a8e..c7b46c5 100644 --- a/viper.go +++ b/viper.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "reflect" "runtime" "strings" "time" @@ -50,6 +51,7 @@ var configFile string var configType string var config map[string]interface{} = make(map[string]interface{}) +var config_index map[string]interface{} = make(map[string]interface{}) var override map[string]interface{} = make(map[string]interface{}) var env map[string]string = make(map[string]string) var defaults map[string]interface{} = make(map[string]interface{}) @@ -133,7 +135,7 @@ func Marshal(rawVal interface{}) error { return err } - insensativiseMaps() + indexMaps() return nil } @@ -172,7 +174,7 @@ func BindEnv(input ...string) (err error) { key = input[0] if len(input) == 1 { - envkey = strings.ToUpper(key) + envkey = strings.Replace(strings.ToUpper(key), ".", "__", -1) } else { envkey = input[1] } @@ -204,6 +206,8 @@ func find(key string) interface{} { return val } + // Periods are not supported. Allow the usage of double underscores to specify + // nested configuration options. envkey, exists := env[key] if exists { jww.TRACE.Println(key, "registered as env var", envkey) @@ -215,7 +219,7 @@ func find(key string) interface{} { } } - val, exists = config[key] + val, exists = config_index[key] if exists { jww.TRACE.Println(key, "found in config:", val) return val @@ -265,7 +269,7 @@ func IsSet(key string) bool { // Have viper check ENV variables for all // keys set in config, default & flags func AutomaticEnv() { - for _, x := range AllKeys() { + for _, x := range AllDeepKeys() { BindEnv(x) } } @@ -286,8 +290,8 @@ func registerAlias(alias string, key string) { // name, we'll never be able to get that value using the original // name, so move the config value to the new realkey. if val, ok := config[alias]; ok { - delete(config, alias) - config[key] = val + delete(config_index, alias) + config_index[key] = val } if val, ok := defaults[alias]; ok { delete(defaults, alias) @@ -327,7 +331,7 @@ func InConfig(key string) bool { func SetDefault(key string, value interface{}) { // If alias passed in, then set the proper default key = realKey(strings.ToLower(key)) - defaults[key] = value + defaults[strings.ToLower(key)] = value } // The user provided value (via flag) @@ -382,25 +386,70 @@ func MarshallReader(in io.Reader) { } } - insensativiseMap(config) + indexMap(config, "", config_index) } -func insensativiseMaps() { - insensativiseMap(config) - insensativiseMap(defaults) - insensativiseMap(override) +func indexMaps() { + indexMap(config, "", config_index) } -func insensativiseMap(m map[string]interface{}) { +// Creates a flat index consisting of materialized paths into the potentially +// nested config structure we've loaded from various sources. The index allows us +// to access the original map contents in a case insensitive manner without +// having to modify the original keys - which may lead to unexpected results +// depending on how a user intends to use the config variable (e.g. passing the +// entire map to function that may not be case-insensitive) or accessing nested +// keys via materialized paths. +func indexMap(m map[string]interface{}, path string, index map[string]interface{}) { for key, val := range m { lower := strings.ToLower(key) - if key != lower { - delete(m, key) - m[lower] = val + var joined_key string + if len(path) > 0 { + joined_key = path + "." + lower + } else { + joined_key = lower + } + index[joined_key] = val + if reflect.TypeOf(val).Kind() == reflect.Map { + // YAML maps may be map[interface{}]interface{} + indexMap(cast.ToStringMap(val), joined_key, index) } } } +// AllDeepKeys returns all keys, including deep materialized paths. +func AllDeepKeys() []string { + all := map[string]struct{}{} + var traverse func(m map[string]interface{}, path string) + traverse = func(m map[string]interface{}, path string) { + for key, val := range m { + lower := strings.ToLower(key) + var joined_key string + if len(path) > 0 { + joined_key = path + "." + lower + } else { + joined_key = lower + } + + all[joined_key] = struct{}{} + if reflect.TypeOf(val).Kind() == reflect.Map { + traverse(cast.ToStringMap(val), joined_key) + } + } + } + + traverse(defaults, "") + traverse(config, "") + traverse(override, "") + + a := []string{} + for x, _ := range all { + a = append(a, x) + } + + return a +} + func AllKeys() []string { m := map[string]struct{}{} @@ -418,7 +467,8 @@ func AllKeys() []string { a := []string{} for x, _ := range m { - a = append(a, x) + // LowerCase the key for backwards-compatibility. + a = append(a, strings.ToLower(x)) } return a @@ -614,6 +664,7 @@ func Reset() { configType = "" config = make(map[string]interface{}) + config_index = make(map[string]interface{}) override = make(map[string]interface{}) env = make(map[string]string) defaults = make(map[string]interface{}) diff --git a/viper_test.go b/viper_test.go index 9818138..a7dfece 100644 --- a/viper_test.go +++ b/viper_test.go @@ -149,7 +149,7 @@ func TestEnv(t *testing.T) { func TestAllKeys(t *testing.T) { ks := sort.StringSlice{"title", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type"} dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - all := map[string]interface{}{"hacker": true, "beard": true, "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "ppu": 0.55, "clothing": map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, "name": "crunk", "owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "id": "13", "title": "TOML Example", "age": 35, "type": "donut"} + all := map[string]interface{}{"beard": true, "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "id": "13", "type": "donut", "owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "ppu": 0.55, "age": 35, "title": "TOML Example", "clothing": map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, "name": "crunk"} var allkeys sort.StringSlice allkeys = AllKeys() @@ -202,3 +202,19 @@ func TestMarshal(t *testing.T) { } assert.Equal(t, &C, &config{Name: "Steve", Port: 1234}) } + +func TestDeepAccess(t *testing.T) { + assert.Equal(t, "leather", Get("clothing.jacket")) +} + +func TestDeepBindEnv(t *testing.T) { + BindEnv("clothing.jacket") + os.Setenv("CLOTHING__JACKET", "peacoat") + assert.Equal(t, "peacoat", Get("clothing.jacket")) +} + +func TestDeepAutomaticEnv(t *testing.T) { + AutomaticEnv() + os.Setenv("CLOTHING__JACKET", "jean") + assert.Equal(t, "jean", Get("clothing.jacket")) +}