Nested maps (#195)

Fixes #71, #93, #158, #168, #209, #141, #160, #162, #190

* Fixed: indentation in comment
* Fixed: Get() returns nil when nested element not found
* Fixed: insensitiviseMaps() made recursive so that nested keys are lowercased
* Fixed: order of expected<=>actual in assert.Equal() statements
* Fixed: find() looks into "overrides" first
* Fixed: TestBindPFlags() to use a new Viper instance
* Fixed: removed extra aliases from display in Debug()
* Added: test for checking precedence of dot-containing keys.
* Fixed: Set() and SetDefault() insert nested values
* Added: tests for overriding nested values
* Changed: AllKeys() includes all keys / AllSettings() includes overridden nested values
* Added: test for shadowed nested key
* Fixed: properties parsing generates nested maps
* Fixed: Get() and IsSet() work correctly on nested values
* Changed: modifier README.md to reflect changes

README.md

@@ -458,16 +458,17 @@ Viper can access a nested field by passing a `.` delimited path of keys:
 GetString("datastore.metric.host") // (returns "127.0.0.1")
 ```
 
-This obeys the precedence rules established above; the search for the root key
+This obeys the precedence rules established above; the search for the path
-(in this example, `datastore`) will cascade through the remaining configuration
+will cascade through the remaining configuration registries until found.
-registries until found. The search for the sub-keys (`metric` and `host`),
-however, will not.
 
-For example, if the `metric` key was not defined in the configuration loaded
+For example, given this configuration file, both `datastore.metric.host` and
-from file, but was defined in the defaults, Viper would return the zero value.
+`datastore.metric.port` are already defined (and may be overridden). If in addition
+`datastore.metric.protocol` was defined in the defaults, Viper would also find it.
 
-On the other hand, if the primary key was not defined, Viper would go through
+However, if `datastore.metric` was overridden (by a flag, an environment variable,
-the remaining registries looking for it.
+the `Set()` method, …) with an immediate value, then all sub-keys of
+`datastore.metric` become undefined, they are “shadowed” by the higher-priority
+configuration level.
 
 Lastly, if there exists a key that matches the delimited key path, its value
 will be returned instead. E.g.

overrides_test.go

@@ -0,0 +1,173 @@
+package viper
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cast"
+	"github.com/stretchr/testify/assert"
+)
+
+type layer int
+
+const (
+	defaultLayer layer = iota + 1
+	overrideLayer
+)
+
+func TestNestedOverrides(t *testing.T) {
+	assert := assert.New(t)
+	var v *Viper
+
+	// Case 0: value overridden by a value
+	overrideDefault(assert, "tom", 10, "tom", 20) // "tom" is first given 10 as default value, then overridden by 20
+	override(assert, "tom", 10, "tom", 20)        // "tom" is first given value 10, then overridden by 20
+	overrideDefault(assert, "tom.age", 10, "tom.age", 20)
+	override(assert, "tom.age", 10, "tom.age", 20)
+	overrideDefault(assert, "sawyer.tom.age", 10, "sawyer.tom.age", 20)
+	override(assert, "sawyer.tom.age", 10, "sawyer.tom.age", 20)
+
+	// Case 1: key:value overridden by a value
+	v = overrideDefault(assert, "tom.age", 10, "tom", "boy") // "tom.age" is first given 10 as default value, then "tom" is overridden by "boy"
+	assert.Nil(v.Get("tom.age"))                             // "tom.age" should not exist anymore
+	v = override(assert, "tom.age", 10, "tom", "boy")
+	assert.Nil(v.Get("tom.age"))
+
+	// Case 2: value overridden by a key:value
+	overrideDefault(assert, "tom", "boy", "tom.age", 10) // "tom" is first given "boy" as default value, then "tom" is overridden by map{"age":10}
+	override(assert, "tom.age", 10, "tom", "boy")
+
+	// Case 3: key:value overridden by a key:value
+	v = overrideDefault(assert, "tom.size", 4, "tom.age", 10)
+	assert.Equal(4, v.Get("tom.size")) // value should still be reachable
+	v = override(assert, "tom.size", 4, "tom.age", 10)
+	assert.Equal(4, v.Get("tom.size"))
+	deepCheckValue(assert, v, overrideLayer, []string{"tom", "size"}, 4)
+
+	// Case 4: key:value overridden by a map
+	v = overrideDefault(assert, "tom.size", 4, "tom", map[string]interface{}{"age": 10}) // "tom.size" is first given "4" as default value, then "tom" is overridden by map{"age":10}
+	assert.Equal(4, v.Get("tom.size"))                                                   // "tom.size" should still be reachable
+	assert.Equal(10, v.Get("tom.age"))                                                   // new value should be there
+	deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, 10)                 // new value should be there
+	v = override(assert, "tom.size", 4, "tom", map[string]interface{}{"age": 10})
+	assert.Nil(v.Get("tom.size"))
+	assert.Equal(10, v.Get("tom.age"))
+	deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, 10)
+
+	// Case 5: array overridden by a value
+	overrideDefault(assert, "tom", []int{10, 20}, "tom", 30)
+	override(assert, "tom", []int{10, 20}, "tom", 30)
+	overrideDefault(assert, "tom.age", []int{10, 20}, "tom.age", 30)
+	override(assert, "tom.age", []int{10, 20}, "tom.age", 30)
+
+	// Case 6: array overridden by an array
+	overrideDefault(assert, "tom", []int{10, 20}, "tom", []int{30, 40})
+	override(assert, "tom", []int{10, 20}, "tom", []int{30, 40})
+	overrideDefault(assert, "tom.age", []int{10, 20}, "tom.age", []int{30, 40})
+	v = override(assert, "tom.age", []int{10, 20}, "tom.age", []int{30, 40})
+	// explicit array merge:
+	s, ok := v.Get("tom.age").([]int)
+	if assert.True(ok, "tom[\"age\"] is not a slice") {
+		v.Set("tom.age", append(s, []int{50, 60}...))
+		assert.Equal([]int{30, 40, 50, 60}, v.Get("tom.age"))
+		deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, []int{30, 40, 50, 60})
+	}
+}
+
+func overrideDefault(assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper {
+	return overrideFromLayer(defaultLayer, assert, firstPath, firstValue, secondPath, secondValue)
+}
+func override(assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper {
+	return overrideFromLayer(overrideLayer, assert, firstPath, firstValue, secondPath, secondValue)
+}
+
+// overrideFromLayer performs the sequential override and low-level checks.
+//
+// First assignment is made on layer l for path firstPath with value firstValue,
+// the second one on the override layer (i.e., with the Set() function)
+// for path secondPath with value secondValue.
+//
+// firstPath and secondPath can include an arbitrary number of dots to indicate
+// a nested element.
+//
+// After each assignment, the value is checked, retrieved both by its full path
+// and by its key sequence (successive maps).
+func overrideFromLayer(l layer, assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper {
+	v := New()
+	firstKeys := strings.Split(firstPath, v.keyDelim)
+	if assert == nil ||
+		len(firstKeys) == 0 || len(firstKeys[0]) == 0 {
+		return v
+	}
+
+	// Set and check first value
+	switch l {
+	case defaultLayer:
+		v.SetDefault(firstPath, firstValue)
+	case overrideLayer:
+		v.Set(firstPath, firstValue)
+	default:
+		return v
+	}
+	assert.Equal(firstValue, v.Get(firstPath))
+	deepCheckValue(assert, v, l, firstKeys, firstValue)
+
+	// Override and check new value
+	secondKeys := strings.Split(secondPath, v.keyDelim)
+	if len(secondKeys) == 0 || len(secondKeys[0]) == 0 {
+		return v
+	}
+	v.Set(secondPath, secondValue)
+	assert.Equal(secondValue, v.Get(secondPath))
+	deepCheckValue(assert, v, overrideLayer, secondKeys, secondValue)
+
+	return v
+}
+
+// deepCheckValue checks that all given keys correspond to a valid path in the
+// configuration map of the given layer, and that the final value equals the one given
+func deepCheckValue(assert *assert.Assertions, v *Viper, l layer, keys []string, value interface{}) {
+	if assert == nil || v == nil ||
+		len(keys) == 0 || len(keys[0]) == 0 {
+		return
+	}
+
+	// init
+	var val interface{}
+	var ms string
+	switch l {
+	case defaultLayer:
+		val = v.defaults
+		ms = "v.defaults"
+	case overrideLayer:
+		val = v.override
+		ms = "v.override"
+	}
+
+	// loop through map
+	var m map[string]interface{}
+	err := false
+	for _, k := range keys {
+		if val == nil {
+			assert.Fail(fmt.Sprintf("%s is not a map[string]interface{}", ms))
+			return
+		}
+
+		// deep scan of the map to get the final value
+		switch val.(type) {
+		case map[interface{}]interface{}:
+			m = cast.ToStringMap(val)
+		case map[string]interface{}:
+			m = val.(map[string]interface{})
+		default:
+			assert.Fail(fmt.Sprintf("%s is not a map[string]interface{}", ms))
+			return
+		}
+		ms = ms + "[\"" + k + "\"]"
+		val = m[k]
+	}
+	if !err {
+		assert.Equal(value, val)
+	}
+}

util.go

@@ -45,6 +45,10 @@ func insensitiviseMap(m map[string]interface{}) {
 		if key != lower {
 			delete(m, key)
 			m[lower] = val
+			if m2, ok := val.(map[string]interface{}); ok {
+				// nested map: recursively insensitivise
+				insensitiviseMap(m2)
+			}
 		}
 	}
 }
@@ -149,7 +153,12 @@ func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType s
 		}
 		for _, key := range p.Keys() {
 			value, _ := p.Get(key)
-			c[key] = value
+			// recursively build nested maps
+			path := strings.Split(key, ".")
+			lastKey := strings.ToLower(path[len(path)-1])
+			deepestMap := deepSearch(c, path[0:len(path)-1])
+			// set innermost value
+			deepestMap[lastKey] = value
 		}
 	}
 
@@ -199,3 +208,34 @@ func parseSizeInBytes(sizeStr string) uint {
 
 	return safeMul(uint(size), multiplier)
 }
+
+// deepSearch scans deep maps, following the key indexes listed in the
+// sequence "path".
+// The last value is expected to be another map, and is returned.
+//
+// In case intermediate keys do not exist, or map to a non-map value,
+// a new map is created and inserted, and the search continues from there:
+// the initial map "m" may be modified!
+func deepSearch(m map[string]interface{}, path []string) map[string]interface{} {
+	for _, k := range path {
+		m2, ok := m[k]
+		if !ok {
+			// intermediate key does not exist
+			// => create it and continue from there
+			m3 := make(map[string]interface{})
+			m[k] = m3
+			m = m3
+			continue
+		}
+		m3, ok := m2.(map[string]interface{})
+		if !ok {
+			// intermediate key is a value
+			// => replace with a new map
+			m3 = make(map[string]interface{})
+			m[k] = m3
+		}
+		// continue search from here
+		m = m3
+	}
+	return m
+}

viper.go

@@ -399,8 +399,9 @@ func (v *Viper) providerPathExists(p *defaultRemoteProvider) bool {
 	return false
 }
 
+// searchMap recursively searches for a value for path in source map.
+// Returns nil if not found.
 func (v *Viper) searchMap(source map[string]interface{}, path []string) interface{} {
-
 	if len(path) == 0 {
 		return source
 	}
@@ -424,12 +425,134 @@ func (v *Viper) searchMap(source map[string]interface{}, path []string) interfac
 			// if the type of `next` is the same as the type being asserted
 			return v.searchMap(next.(map[string]interface{}), path[1:])
 		default:
+			if len(path) == 1 {
 				return next
 			}
-	} else {
+			// got a value but nested key expected, return "nil" for not found
 			return nil
 		}
 	}
+	return nil
+}
+
+// searchMapWithPathPrefixes recursively searches for a value for path in source map.
+//
+// While searchMap() considers each path element as a single map key, this
+// function searches for, and prioritizes, merged path elements.
+// e.g., if in the source, "foo" is defined with a sub-key "bar", and "foo.bar"
+// is also defined, this latter value is returned for path ["foo", "bar"].
+//
+// This should be useful only at config level (other maps may not contain dots
+// in their keys).
+func (v *Viper) searchMapWithPathPrefixes(source map[string]interface{}, path []string) interface{} {
+	if len(path) == 0 {
+		return source
+	}
+
+	// search for path prefixes, starting from the longest one
+	for i := len(path); i > 0; i-- {
+		prefixKey := strings.ToLower(strings.Join(path[0:i], v.keyDelim))
+
+		var ok bool
+		var next interface{}
+		for k, v := range source {
+			if strings.ToLower(k) == prefixKey {
+				ok = true
+				next = v
+				break
+			}
+		}
+
+		if ok {
+			var val interface{}
+			switch next.(type) {
+			case map[interface{}]interface{}:
+				val = v.searchMapWithPathPrefixes(cast.ToStringMap(next), path[i:])
+			case map[string]interface{}:
+				// Type assertion is safe here since it is only reached
+				// if the type of `next` is the same as the type being asserted
+				val = v.searchMapWithPathPrefixes(next.(map[string]interface{}), path[i:])
+			default:
+				if len(path) == i {
+					val = next
+				}
+				// got a value but nested key expected, do nothing and look for next prefix
+			}
+			if val != nil {
+				return val
+			}
+		}
+	}
+
+	// not found
+	return nil
+}
+
+// isPathShadowedInDeepMap makes sure the given path is not shadowed somewhere
+// on its path in the map.
+// e.g., if "foo.bar" has a value in the given map, it “shadows”
+//       "foo.bar.baz" in a lower-priority map
+func (v *Viper) isPathShadowedInDeepMap(path []string, m map[string]interface{}) string {
+	var parentVal interface{}
+	for i := 1; i < len(path); i++ {
+		parentVal = v.searchMap(m, path[0:i])
+		if parentVal == nil {
+			// not found, no need to add more path elements
+			return ""
+		}
+		switch parentVal.(type) {
+		case map[interface{}]interface{}:
+			continue
+		case map[string]interface{}:
+			continue
+		default:
+			// parentVal is a regular value which shadows "path"
+			return strings.Join(path[0:i], v.keyDelim)
+		}
+	}
+	return ""
+}
+
+// isPathShadowedInFlatMap makes sure the given path is not shadowed somewhere
+// in a sub-path of the map.
+// e.g., if "foo.bar" has a value in the given map, it “shadows”
+//       "foo.bar.baz" in a lower-priority map
+func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string {
+	// unify input map
+	var m map[string]interface{}
+	switch mi.(type) {
+	case map[string]string, map[string]FlagValue:
+		m = cast.ToStringMap(mi)
+	default:
+		return ""
+	}
+
+	// scan paths
+	var parentKey string
+	for i := 1; i < len(path); i++ {
+		parentKey = strings.Join(path[0:i], v.keyDelim)
+		if _, ok := m[parentKey]; ok {
+			return parentKey
+		}
+	}
+	return ""
+}
+
+// isPathShadowedInAutoEnv makes sure the given path is not shadowed somewhere
+// in the environment, when automatic env is on.
+// e.g., if "foo.bar" has a value in the environment, it “shadows”
+//       "foo.bar.baz" in a lower-priority map
+func (v *Viper) isPathShadowedInAutoEnv(path []string) string {
+	var parentKey string
+	var val string
+	for i := 1; i < len(path); i++ {
+		parentKey = strings.Join(path[0:i], v.keyDelim)
+		if val = v.getEnv(v.mergeWithEnvPrefix(parentKey)); val != "" {
+			return parentKey
+		}
+	}
+	return ""
+}
 
 // SetTypeByDefaultValue enables or disables the inference of a key value's
 // type when the Get function is used based upon a key's default value as
@@ -465,46 +588,16 @@ func Get(key string) interface{} { return v.Get(key) }
 func (v *Viper) Get(key string) interface{} {
 	lcaseKey := strings.ToLower(key)
 	val := v.find(lcaseKey)
-
-	if val == nil {
-		path := strings.Split(key, v.keyDelim)
-		source := v.find(strings.ToLower(path[0]))
-		if source != nil {
-			if reflect.TypeOf(source).Kind() == reflect.Map {
-				val = v.searchMap(cast.ToStringMap(source), path[1:])
-			}
-		}
-	}
-
-	// if no other value is returned and a flag does exist for the value,
-	// get the flag's value even if the flag's value has not changed
-	if val == nil {
-		if flag, exists := v.pflags[lcaseKey]; exists {
-			jww.TRACE.Println(key, "get pflag default", val)
-			switch flag.ValueType() {
-			case "int", "int8", "int16", "int32", "int64":
-				val = cast.ToInt(flag.ValueString())
-			case "bool":
-				val = cast.ToBool(flag.ValueString())
-			default:
-				val = flag.ValueString()
-			}
-		}
-	}
-
 	if val == nil {
 		return nil
 	}
 
-	var valType interface{}
+	valType := val
-	if !v.typeByDefValue {
+	if v.typeByDefValue {
-		valType = val
+		path := strings.Split(lcaseKey, v.keyDelim)
-	} else {
+		defVal := v.searchMap(v.defaults, path)
-		defVal, defExists := v.defaults[lcaseKey]
+		if defVal != nil {
-		if defExists {
 			valType = defVal
-		} else {
-			valType = val
 		}
 	}
 
@@ -752,10 +845,27 @@ func (v *Viper) find(key string) interface{} {
 	var val interface{}
 	var exists bool
 
+	// compute the path through the nested maps to the nested value
+	path := strings.Split(key, v.keyDelim)
+	if shadow := v.isPathShadowedInDeepMap(path, castMapStringToMapInterface(v.aliases)); shadow != "" {
+		return nil
+	}
+
 	// if the requested key is an alias, then return the proper key
 	key = v.realKey(key)
+	// re-compute the path
+	path = strings.Split(key, v.keyDelim)
 
-	// PFlag Override first
+	// Set() override first
+	val = v.searchMap(v.override, path)
+	if val != nil {
+		return val
+	}
+	if shadow := v.isPathShadowedInDeepMap(path, v.override); shadow != "" {
+		return nil
+	}
+
+	// PFlag override next
 	flag, exists := v.pflags[key]
 	if exists && flag.HasChanged() {
 		switch flag.ValueType() {
@@ -770,56 +880,74 @@ func (v *Viper) find(key string) interface{} {
 			return flag.ValueString()
 		}
 	}
-
+	if shadow := v.isPathShadowedInFlatMap(path, v.pflags); shadow != "" {
-	val, exists = v.override[key]
+		return nil
-	if exists {
-		return val
 	}
 
+	// Env override next
 	if v.automaticEnvApplied {
 		// even if it hasn't been registered, if automaticEnv is used,
 		// check any Get request
 		if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" {
 			return val
 		}
+		if shadow := v.isPathShadowedInAutoEnv(path); shadow != "" {
+			return nil
+		}
 	}
-
 	envkey, exists := v.env[key]
 	if exists {
 		if val = v.getEnv(envkey); val != "" {
 			return val
 		}
 	}
-
+	if shadow := v.isPathShadowedInFlatMap(path, v.env); shadow != "" {
-	val, exists = v.config[key]
+		return nil
-	if exists {
-		return val
 	}
 
-	// Test for nested config parameter
+	// Config file next
-	if strings.Contains(key, v.keyDelim) {
+	val = v.searchMapWithPathPrefixes(v.config, path)
-		path := strings.Split(key, v.keyDelim)
-
-		source := v.find(path[0])
-		if source != nil {
-			if reflect.TypeOf(source).Kind() == reflect.Map {
-				val := v.searchMap(cast.ToStringMap(source), path[1:])
 	if val != nil {
 		return val
 	}
-			}
+	if shadow := v.isPathShadowedInDeepMap(path, v.config); shadow != "" {
-		}
+		return nil
 	}
 
-	val, exists = v.kvstore[key]
+	// K/V store next
-	if exists {
+	val = v.searchMap(v.kvstore, path)
+	if val != nil {
 		return val
 	}
+	if shadow := v.isPathShadowedInDeepMap(path, v.kvstore); shadow != "" {
+		return nil
+	}
 
-	val, exists = v.defaults[key]
+	// Default next
-	if exists {
+	val = v.searchMap(v.defaults, path)
+	if val != nil {
 		return val
 	}
+	if shadow := v.isPathShadowedInDeepMap(path, v.defaults); shadow != "" {
+		return nil
+	}
+
+	// last chance: if no other value is returned and a flag does exist for the value,
+	// get the flag's value even if the flag's value has not changed
+	if flag, exists := v.pflags[key]; exists {
+		switch flag.ValueType() {
+		case "int", "int8", "int16", "int32", "int64":
+			return cast.ToInt(flag.ValueString())
+		case "bool":
+			return cast.ToBool(flag.ValueString())
+		case "stringSlice":
+			s := strings.TrimPrefix(flag.ValueString(), "[")
+			return strings.TrimSuffix(s, "]")
+		default:
+			return flag.ValueString()
+		}
+	}
+	// last item, no need to check shadowing
 
 	return nil
 }
@@ -827,20 +955,8 @@ func (v *Viper) find(key string) interface{} {
 // IsSet checks to see if the key has been set in any of the data locations.
 func IsSet(key string) bool { return v.IsSet(key) }
 func (v *Viper) IsSet(key string) bool {
-	path := strings.Split(key, v.keyDelim)
-
 	lcaseKey := strings.ToLower(key)
 	val := v.find(lcaseKey)
-
-	if val == nil {
-		source := v.find(strings.ToLower(path[0]))
-		if source != nil {
-			if reflect.TypeOf(source).Kind() == reflect.Map {
-				val = v.searchMap(cast.ToStringMap(source), path[1:])
-			}
-		}
-	}
-
 	return val != nil
 }
 
@@ -923,7 +1039,13 @@ func SetDefault(key string, value interface{}) { v.SetDefault(key, value) }
 func (v *Viper) SetDefault(key string, value interface{}) {
 	// If alias passed in, then set the proper default
 	key = v.realKey(strings.ToLower(key))
-	v.defaults[key] = value
+
+	path := strings.Split(key, v.keyDelim)
+	lastKey := strings.ToLower(path[len(path)-1])
+	deepestMap := deepSearch(v.defaults, path[0:len(path)-1])
+
+	// set innermost value
+	deepestMap[lastKey] = value
 }
 
 // Set sets the value for the key in the override regiser.
@@ -933,7 +1055,13 @@ func Set(key string, value interface{}) { v.Set(key, value) }
 func (v *Viper) Set(key string, value interface{}) {
 	// If alias passed in, then set the proper override
 	key = v.realKey(strings.ToLower(key))
-	v.override[key] = value
+
+	path := strings.Split(key, v.keyDelim)
+	lastKey := strings.ToLower(path[len(path)-1])
+	deepestMap := deepSearch(v.override, path[0:len(path)-1])
+
+	// set innermost value
+	deepestMap[lastKey] = value
 }
 
 // ReadInConfig will discover and load the configuration file from disk
@@ -1013,6 +1141,14 @@ func castToMapStringInterface(
 	return tgt
 }
 
+func castMapStringToMapInterface(src map[string]string) map[string]interface{} {
+	tgt := map[string]interface{}{}
+	for k, v := range src {
+		tgt[k] = v
+	}
+	return tgt
+}
+
 // mergeMaps merges two maps. The `itgt` parameter is for handling go-yaml's
 // insistence on parsing nested structures as `map[interface{}]interface{}`
 // instead of using a `string` as the key for nest structures beyond one level
@@ -1150,55 +1286,114 @@ func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface
 	return v.kvstore, err
 }
 
-// AllKeys returns all keys regardless where they are set.
+// AllKeys returns all keys holding a value, regardless of where they are set.
+// Nested keys are returned with a v.keyDelim (= ".") separator
 func AllKeys() []string { return v.AllKeys() }
 func (v *Viper) AllKeys() []string {
-	m := map[string]struct{}{}
+	m := map[string]bool{}
-
+	// add all paths, by order of descending priority to ensure correct shadowing
-	for key := range v.defaults {
+	m = v.flattenAndMergeMap(m, castMapStringToMapInterface(v.aliases), "")
-		m[strings.ToLower(key)] = struct{}{}
+	m = v.flattenAndMergeMap(m, v.override, "")
-	}
+	m = v.mergeFlatMap(m, v.pflags)
-
+	m = v.mergeFlatMap(m, v.env)
-	for key := range v.pflags {
+	m = v.flattenAndMergeMap(m, v.config, "")
-		m[strings.ToLower(key)] = struct{}{}
+	m = v.flattenAndMergeMap(m, v.kvstore, "")
-	}
+	m = v.flattenAndMergeMap(m, v.defaults, "")
-
-	for key := range v.env {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.config {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.kvstore {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.override {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.aliases {
-		m[strings.ToLower(key)] = struct{}{}
-	}
 
+	// convert set of paths to list
 	a := []string{}
 	for x := range m {
 		a = append(a, x)
 	}
-
 	return a
 }
 
-// AllSettings returns all settings as a map[string]interface{}.
+// flattenAndMergeMap recursively flattens the given map into a map[string]bool
+// of key paths (used as a set, easier to manipulate than a []string):
+// - each path is merged into a single key string, delimited with v.keyDelim (= ".")
+// - if a path is shadowed by an earlier value in the initial shadow map,
+//   it is skipped.
+// The resulting set of paths is merged to the given shadow set at the same time.
+func (v *Viper) flattenAndMergeMap(shadow map[string]bool, m map[string]interface{}, prefix string) map[string]bool {
+	if shadow != nil && prefix != "" && shadow[prefix] {
+		// prefix is shadowed => nothing more to flatten
+		return shadow
+	}
+	if shadow == nil {
+		shadow = make(map[string]bool)
+	}
+
+	var m2 map[string]interface{}
+	if prefix != "" {
+		prefix += v.keyDelim
+	}
+	for k, val := range m {
+		fullKey := prefix + k
+		switch val.(type) {
+		case map[string]interface{}:
+			m2 = val.(map[string]interface{})
+		case map[interface{}]interface{}:
+			m2 = cast.ToStringMap(val)
+		default:
+			// immediate value
+			shadow[strings.ToLower(fullKey)] = true
+			continue
+		}
+		// recursively merge to shadow map
+		shadow = v.flattenAndMergeMap(shadow, m2, fullKey)
+	}
+	return shadow
+}
+
+// mergeFlatMap merges the given maps, excluding values of the second map
+// shadowed by values from the first map.
+func (v *Viper) mergeFlatMap(shadow map[string]bool, mi interface{}) map[string]bool {
+	// unify input map
+	var m map[string]interface{}
+	switch mi.(type) {
+	case map[string]string, map[string]FlagValue:
+		m = cast.ToStringMap(mi)
+	default:
+		return shadow
+	}
+
+	// scan keys
+outer:
+	for k, _ := range m {
+		path := strings.Split(k, v.keyDelim)
+		// scan intermediate paths
+		var parentKey string
+		for i := 1; i < len(path); i++ {
+			parentKey = strings.Join(path[0:i], v.keyDelim)
+			if shadow[parentKey] {
+				// path is shadowed, continue
+				continue outer
+			}
+		}
+		// add key
+		shadow[strings.ToLower(k)] = true
+	}
+	return shadow
+}
+
+// AllSettings merges all settings and returns them as a map[string]interface{}.
 func AllSettings() map[string]interface{} { return v.AllSettings() }
 func (v *Viper) AllSettings() map[string]interface{} {
 	m := map[string]interface{}{}
-	for _, x := range v.AllKeys() {
+	// start from the list of keys, and construct the map one value at a time
-		m[x] = v.Get(x)
+	for _, k := range v.AllKeys() {
+		value := v.Get(k)
+		if value == nil {
+			// should not happen, since AllKeys() returns only keys holding a value,
+			// check just in case anything changes
+			continue
+		}
+		path := strings.Split(k, v.keyDelim)
+		lastKey := strings.ToLower(path[len(path)-1])
+		deepestMap := deepSearch(m, path[0:len(path)-1])
+		// set innermost value
+		deepestMap[lastKey] = value
 	}
-
 	return m
 }
 
@@ -1289,7 +1484,6 @@ func (v *Viper) findConfigFile() (string, error) {
 // purposes.
 func Debug() { v.Debug() }
 func (v *Viper) Debug() {
-	fmt.Println("Aliases:")
 	fmt.Printf("Aliases:\n%#v\n", v.aliases)
 	fmt.Printf("Override:\n%#v\n", v.override)
 	fmt.Printf("PFlags:\n%#v\n", v.pflags)

viper_test.go

@@ -8,6 +8,7 @@ package viper
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -104,8 +105,9 @@ var remoteExample = []byte(`{
 
 func initConfigs() {
 	Reset()
+	var r io.Reader
 	SetConfigType("yaml")
-	r := bytes.NewReader(yamlExample)
+	r = bytes.NewReader(yamlExample)
 	unmarshalReader(r, v.config)
 
 	SetConfigType("json")
@@ -259,7 +261,7 @@ func TestUnmarshalling(t *testing.T) {
 	assert.False(t, InConfig("state"))
 	assert.Equal(t, "steve", Get("name"))
 	assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, Get("hobbies"))
-	assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, Get("clothing"))
+	assert.Equal(t, map[string]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, Get("clothing"))
 	assert.Equal(t, 35, Get("age"))
 }
 
@@ -420,9 +422,9 @@ func TestSetEnvReplacer(t *testing.T) {
 func TestAllKeys(t *testing.T) {
 	initConfigs()
 
-	ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"}
+	ks := sort.StringSlice{"title", "newkey", "owner.organization", "owner.dob", "owner.bio", "name", "beard", "ppu", "batters.batter", "hobbies", "clothing.jacket", "clothing.trousers", "clothing.pants.size", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"}
 	dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
-	all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[interface{}]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[interface{}]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters.batter.type": "Regular", "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}}
+	all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[string]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}}
 
 	var allkeys sort.StringSlice
 	allkeys = AllKeys()
@@ -468,17 +470,18 @@ func TestUnmarshal(t *testing.T) {
 		t.Fatalf("unable to decode into struct, %v", err)
 	}
 
-	assert.Equal(t, &C, &config{Name: "Steve", Port: 1313, Duration: time.Second + time.Millisecond})
+	assert.Equal(t, &config{Name: "Steve", Port: 1313, Duration: time.Second + time.Millisecond}, &C)
 
 	Set("port", 1234)
 	err = Unmarshal(&C)
 	if err != nil {
 		t.Fatalf("unable to decode into struct, %v", err)
 	}
-	assert.Equal(t, &C, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond})
+	assert.Equal(t, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}, &C)
 }
 
 func TestBindPFlags(t *testing.T) {
+	v := New() // create independent Viper object
 	flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError)
 
 	var testValues = map[string]*string{
@@ -497,7 +500,7 @@ func TestBindPFlags(t *testing.T) {
 		testValues[name] = flagSet.String(name, "", "test")
 	}
 
-	err := BindPFlags(flagSet)
+	err := v.BindPFlags(flagSet)
 	if err != nil {
 		t.Fatalf("error binding flag set, %v", err)
 	}
@@ -508,7 +511,7 @@ func TestBindPFlags(t *testing.T) {
 	})
 
 	for name, expected := range mutatedTestValues {
-		assert.Equal(t, Get(name), expected)
+		assert.Equal(t, expected, v.Get(name))
 	}
 
 }
@@ -641,7 +644,7 @@ func TestFindsNestedKeys(t *testing.T) {
 		"name":      "Cake",
 		"hacker":    true,
 		"ppu":       0.55,
-		"clothing": map[interface{}]interface{}{
+		"clothing": map[string]interface{}{
 			"jacket":   "leather",
 			"trousers": "denim",
 			"pants": map[interface{}]interface{}{
@@ -690,7 +693,7 @@ func TestReadBufConfig(t *testing.T) {
 	assert.False(t, v.InConfig("state"))
 	assert.Equal(t, "steve", v.Get("name"))
 	assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, v.Get("hobbies"))
-	assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, v.Get("clothing"))
+	assert.Equal(t, map[string]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, v.Get("clothing"))
 	assert.Equal(t, 35, v.Get("age"))
 }
 
@@ -759,10 +762,10 @@ func TestSub(t *testing.T) {
 	assert.Equal(t, v.Get("clothing.pants.size"), subv.Get("size"))
 
 	subv = v.Sub("clothing.pants.size")
-	assert.Equal(t, subv, (*Viper)(nil))
+	assert.Equal(t, (*Viper)(nil), subv)
 
 	subv = v.Sub("missing.key")
-	assert.Equal(t, subv, (*Viper)(nil))
+	assert.Equal(t, (*Viper)(nil), subv)
 }
 
 var yamlMergeExampleTgt = []byte(`
@@ -883,28 +886,28 @@ func TestMergeConfigNoMerge(t *testing.T) {
 }
 
 func TestUnmarshalingWithAliases(t *testing.T) {
-	SetDefault("Id", 1)
+	v := New()
-	Set("name", "Steve")
+	v.SetDefault("ID", 1)
-	Set("lastname", "Owen")
+	v.Set("name", "Steve")
+	v.Set("lastname", "Owen")
 
-	RegisterAlias("UserID", "Id")
+	v.RegisterAlias("UserID", "ID")
-	RegisterAlias("Firstname", "name")
+	v.RegisterAlias("Firstname", "name")
-	RegisterAlias("Surname", "lastname")
+	v.RegisterAlias("Surname", "lastname")
 
 	type config struct {
-		Id        int
+		ID        int
 		FirstName string
 		Surname   string
 	}
 
 	var C config
-
+	err := v.Unmarshal(&C)
-	err := Unmarshal(&C)
 	if err != nil {
 		t.Fatalf("unable to decode into struct, %v", err)
 	}
 
-	assert.Equal(t, &C, &config{Id: 1, FirstName: "Steve", Surname: "Owen"})
+	assert.Equal(t, &config{ID: 1, FirstName: "Steve", Surname: "Owen"}, &C)
 }
 
 func TestSetConfigNameClearsFileCache(t *testing.T) {
@@ -917,9 +920,26 @@ func TestShadowedNestedValue(t *testing.T) {
 	polyester := "polyester"
 	initYAML()
 	SetDefault("clothing.shirt", polyester)
+	SetDefault("clothing.jacket.price", 100)
 
-	assert.Equal(t, GetString("clothing.jacket"), "leather")
+	assert.Equal(t, "leather", GetString("clothing.jacket"))
-	assert.Equal(t, GetString("clothing.shirt"), polyester)
+	assert.Nil(t, Get("clothing.jacket.price"))
+	assert.Equal(t, polyester, GetString("clothing.shirt"))
+
+	clothingSettings := AllSettings()["clothing"].(map[string]interface{})
+	assert.Equal(t, "leather", clothingSettings["jacket"])
+	assert.Equal(t, polyester, clothingSettings["shirt"])
+}
+
+func TestDotParameter(t *testing.T) {
+	initJSON()
+	// shoud take precedence over batters defined in jsonExample
+	r := bytes.NewReader([]byte(`{ "batters.batter": [ { "type": "Small" } ] }`))
+	unmarshalReader(r, v.config)
+
+	actual := Get("batters.batter")
+	expected := []interface{}{map[string]interface{}{"type": "Small"}}
+	assert.Equal(t, expected, actual)
 }
 
 func TestGetBool(t *testing.T) {