From e0f7631cf3ac7e7530949c7e154855076b0a4c17 Mon Sep 17 00:00:00 2001 From: Xavier Coulon Date: Wed, 3 Jan 2018 10:37:18 +0100 Subject: [PATCH 01/25] WatchConfig and Kubernetes (#284) Support override of symlink to config file Include tests for WatchConfig of regular files, as well as config file which links to a folder which is itself a link to another folder in the same "watch dir" (the way Kubernetes exposes config files from ConfigMaps mounted on a volume in a Pod) Also: - Add synchronization with WaitGroup to ensure that the WatchConfig is properly started before returning - Remove the watcher when the Config file is removed. Fixes #284 Signed-off-by: Xavier Coulon --- .gitignore | 7 +++- viper.go | 39 +++++++++++++------ viper_test.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 352a34a..01b5c44 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,9 @@ _testmain.go *.exe *.test -*.bench \ No newline at end of file +*.bench + +.vscode + +# exclude dependencies in the `/vendor` folder +vendor diff --git a/viper.go b/viper.go index ad8a037..148d5cc 100644 --- a/viper.go +++ b/viper.go @@ -30,6 +30,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "time" yaml "gopkg.in/yaml.v2" @@ -260,13 +261,14 @@ func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { func WatchConfig() { v.WatchConfig() } func (v *Viper) WatchConfig() { + wg := sync.WaitGroup{} + wg.Add(1) go func() { watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) } defer watcher.Close() - // we have to watch the entire directory to pick up renames/atomic saves in a cross-platform way filename, err := v.getConfigFile() if err != nil { @@ -276,31 +278,46 @@ func (v *Viper) WatchConfig() { configFile := filepath.Clean(filename) configDir, _ := filepath.Split(configFile) + realConfigFile, _ := filepath.EvalSymlinks(filename) done := make(chan bool) go func() { + loop: for { select { case event := <-watcher.Events: - // we only care about the config file - if filepath.Clean(event.Name) == configFile { - if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create { - err := v.ReadInConfig() - if err != nil { - log.Println("error:", err) - } + currentConfigFile, _ := filepath.EvalSymlinks(filename) + // we only care about the config file with the following cases: + // 1 - if the config file was modified or created + // 2 - if the real path to the config file changed (eg: k8s ConfigMap replacement) + if (filepath.Clean(event.Name) == configFile && + (event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create)) || + (currentConfigFile != "" && currentConfigFile != realConfigFile) { + realConfigFile = currentConfigFile + err := v.ReadInConfig() + if err != nil { + log.Println("error reading file:", err.Error()) + } + if v.onConfigChange != nil { v.onConfigChange(event) } + } else if filepath.Clean(event.Name) == configFile && + event.Op&fsnotify.Remove == fsnotify.Remove { + done <- true + break loop } + case err := <-watcher.Errors: - log.Println("error:", err) + log.Printf("watcher error: %v\n", err) } } }() - watcher.Add(configDir) - <-done + wg.Done() // done initalizing the watch in this go routine, so the parent routine can move on... + <-done // block until the watched file is removed... }() + // make sure that the go routine above fully started before returning + wg.Wait() } // SetConfigFile explicitly defines the path, name and extension of the config file. diff --git a/viper_test.go b/viper_test.go index c93480e..7e2140e 100644 --- a/viper_test.go +++ b/viper_test.go @@ -11,18 +11,23 @@ import ( "io" "io/ioutil" "os" + "os/exec" "path" "reflect" + "runtime" "sort" "strings" + "sync" "testing" "time" + "github.com/fsnotify/fsnotify" "github.com/spf13/afero" "github.com/spf13/cast" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var yamlExample = []byte(`Hacker: true @@ -1368,6 +1373,104 @@ func doTestCaseInsensitive(t *testing.T, typ, config string) { } +func newViperWithConfigFile(t *testing.T) (*Viper, string, func()) { + watchDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + configFile := path.Join(watchDir, "config.yaml") + err = ioutil.WriteFile(configFile, []byte("foo: bar\n"), 0640) + require.Nil(t, err) + cleanup := func() { + os.RemoveAll(watchDir) + } + v := New() + v.SetConfigFile(configFile) + err = v.ReadInConfig() + require.Nil(t, err) + require.Equal(t, "bar", v.Get("foo")) + return v, configFile, cleanup +} + +func newViperWithSymlinkedConfigFile(t *testing.T) (*Viper, string, string, func()) { + watchDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + dataDir1 := path.Join(watchDir, "data1") + err = os.Mkdir(dataDir1, 0777) + require.Nil(t, err) + realConfigFile := path.Join(dataDir1, "config.yaml") + t.Logf("Real config file location: %s\n", realConfigFile) + err = ioutil.WriteFile(realConfigFile, []byte("foo: bar\n"), 0640) + require.Nil(t, err) + cleanup := func() { + os.RemoveAll(watchDir) + } + // now, symlink the tm `data1` dir to `data` in the baseDir + os.Symlink(dataDir1, path.Join(watchDir, "data")) + // and link the `/datadir1/config.yaml` to `/config.yaml` + configFile := path.Join(watchDir, "config.yaml") + os.Symlink(path.Join(watchDir, "data", "config.yaml"), configFile) + fmt.Printf("Config file location: %s\n", path.Join(watchDir, "config.yaml")) + // init Viper + v := New() + v.SetConfigFile(configFile) + err = v.ReadInConfig() + require.Nil(t, err) + require.Equal(t, "bar", v.Get("foo")) + return v, watchDir, configFile, cleanup +} + +func TestWatchFile(t *testing.T) { + t.Run("file content changed", func(t *testing.T) { + // given a `config.yaml` file being watched + v, configFile, cleanup := newViperWithConfigFile(t) + defer cleanup() + wg := sync.WaitGroup{} + v.WatchConfig() + v.OnConfigChange(func(in fsnotify.Event) { + t.Logf("config file changed") + wg.Done() + }) + wg.Add(1) + // when overwriting the file and waiting for the custom change notification handler to be triggered + err := ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) + wg.Wait() + // then the config value should have changed + require.Nil(t, err) + assert.Equal(t, "baz", v.Get("foo")) + }) + + t.Run("link to real file changed (à la Kubernetes)", func(t *testing.T) { + // skip if not executed on Linux + if runtime.GOOS != "linux" { + t.Skipf("Skipping test as symlink replacements don't work on non-linux environment...") + } + v, watchDir, _, _ := newViperWithSymlinkedConfigFile(t) + // defer cleanup() + wg := sync.WaitGroup{} + v.WatchConfig() + v.OnConfigChange(func(in fsnotify.Event) { + t.Logf("config file changed") + wg.Done() + }) + wg.Add(1) + // when link to another `config.yaml` file + dataDir2 := path.Join(watchDir, "data2") + err := os.Mkdir(dataDir2, 0777) + require.Nil(t, err) + configFile2 := path.Join(dataDir2, "config.yaml") + err = ioutil.WriteFile(configFile2, []byte("foo: baz\n"), 0640) + require.Nil(t, err) + // change the symlink using the `ln -sfn` command + err = exec.Command("ln", "-sfn", dataDir2, path.Join(watchDir, "data")).Run() + require.Nil(t, err) + wg.Wait() + // then + require.Nil(t, err) + assert.Equal(t, "baz", v.Get("foo")) + + }) + +} + func BenchmarkGetBool(b *testing.B) { key := "BenchmarkGetBool" v = New() From 242f4890f5f6239217fa5c45bb6de3b4b5509d3b Mon Sep 17 00:00:00 2001 From: Xavier Coulon Date: Thu, 24 May 2018 10:09:29 +0200 Subject: [PATCH 02/25] Refactor with WaitGroup and check channel is open Signed-off-by: Xavier Coulon --- viper.go | 37 ++++++++++++++++++++++--------------- viper_test.go | 3 ++- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/viper.go b/viper.go index 148d5cc..72230f5 100644 --- a/viper.go +++ b/viper.go @@ -261,8 +261,8 @@ func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { func WatchConfig() { v.WatchConfig() } func (v *Viper) WatchConfig() { - wg := sync.WaitGroup{} - wg.Add(1) + initWG := sync.WaitGroup{} + initWG.Add(1) go func() { watcher, err := fsnotify.NewWatcher() if err != nil { @@ -272,7 +272,7 @@ func (v *Viper) WatchConfig() { // we have to watch the entire directory to pick up renames/atomic saves in a cross-platform way filename, err := v.getConfigFile() if err != nil { - log.Println("error:", err) + log.Printf("error: %v\n", err) return } @@ -280,12 +280,16 @@ func (v *Viper) WatchConfig() { configDir, _ := filepath.Split(configFile) realConfigFile, _ := filepath.EvalSymlinks(filename) - done := make(chan bool) + eventsWG := sync.WaitGroup{} + eventsWG.Add(1) go func() { - loop: for { select { - case event := <-watcher.Events: + case event, ok := <-watcher.Events: + if !ok { // 'Events' channel is closed + eventsWG.Done() + return + } currentConfigFile, _ := filepath.EvalSymlinks(filename) // we only care about the config file with the following cases: // 1 - if the config file was modified or created @@ -296,28 +300,31 @@ func (v *Viper) WatchConfig() { realConfigFile = currentConfigFile err := v.ReadInConfig() if err != nil { - log.Println("error reading file:", err.Error()) + log.Printf("error reading file: %v\n", err) } if v.onConfigChange != nil { v.onConfigChange(event) } } else if filepath.Clean(event.Name) == configFile && event.Op&fsnotify.Remove == fsnotify.Remove { - done <- true - break loop + eventsWG.Done() + return } - case err := <-watcher.Errors: - log.Printf("watcher error: %v\n", err) + case err, ok := <-watcher.Errors: + if ok { // 'Errors' channel is not closed + log.Printf("watcher error: %v\n", err) + } + eventsWG.Done() + return } } }() watcher.Add(configDir) - wg.Done() // done initalizing the watch in this go routine, so the parent routine can move on... - <-done // block until the watched file is removed... + initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... + eventsWG.Wait() // now, wait for event loop to end in this go-routine... }() - // make sure that the go routine above fully started before returning - wg.Wait() + initWG.Wait() // make sure that the go routine above fully ended before returning } // SetConfigFile explicitly defines the path, name and extension of the config file. diff --git a/viper_test.go b/viper_test.go index 7e2140e..9961970 100644 --- a/viper_test.go +++ b/viper_test.go @@ -1419,9 +1419,11 @@ func newViperWithSymlinkedConfigFile(t *testing.T) (*Viper, string, string, func } func TestWatchFile(t *testing.T) { + t.Run("file content changed", func(t *testing.T) { // given a `config.yaml` file being watched v, configFile, cleanup := newViperWithConfigFile(t) + fmt.Printf("test config file: %s\n", configFile) defer cleanup() wg := sync.WaitGroup{} v.WatchConfig() @@ -1466,7 +1468,6 @@ func TestWatchFile(t *testing.T) { // then require.Nil(t, err) assert.Equal(t, "baz", v.Get("foo")) - }) } From fb7a06477f21a08e24c82d4bddc131d56fdef129 Mon Sep 17 00:00:00 2001 From: Brice Fernandes Date: Tue, 10 Jul 2018 12:30:24 +0100 Subject: [PATCH 03/25] Add example of marshalling to string (#531) --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 64bf474..d1b8737 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,7 @@ The following functions and methods exist: * `GetTime(key string) : time.Time` * `GetDuration(key string) : time.Duration` * `IsSet(key string) : bool` + * `AllSettings() : map[string]interface{}` One important thing to recognize is that each Get function will return a zero value if it’s not found. To check if a given key exists, the `IsSet()` method @@ -590,6 +591,27 @@ if err != nil { } ``` +### Marshalling to string + +You may need to marhsal all the settings held in viper into a string rather than write them to a file. +You can use your favorite format's marshaller with the config returned by `AllSettings()`. + +```go +import ( + yaml "gopkg.in/yaml.v2" + // ... +) + +func yamlStringSettings() string { + c := viper.AllSettings() + bs, err := yaml.Marshal(c) + if err != nil { + t.Fatalf("unable to marshal config to YAML: %v", err) + } + return string(bs) +} +``` + ## Viper or Vipers? Viper comes ready to use out of the box. There is no configuration or From d493c32b69b8c6f2377bf30bc4d70267ffbc0793 Mon Sep 17 00:00:00 2001 From: Adriano Date: Tue, 10 Jul 2018 08:30:20 -0400 Subject: [PATCH 04/25] Update README.md (#470) Fix typo in the environment variable usage documentation. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d1b8737..d752822 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ _When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ Viper provides a mechanism to try to ensure that ENV variables are unique. By -using `SetEnvPrefix`, you can tell Viper to use add a prefix while reading from +using `SetEnvPrefix`, you can tell Viper to use a prefix while reading from the environment variables. Both `BindEnv` and `AutomaticEnv` will use this prefix. From c1250e5dd7e8b4e2911c6c6aa5b83be016bd2a3c Mon Sep 17 00:00:00 2001 From: Xavier Coulon Date: Fri, 13 Jul 2018 10:30:23 +0200 Subject: [PATCH 05/25] apply review comments use masks for checking the events. Signed-off-by: Xavier Coulon --- viper.go | 7 +++++-- viper_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/viper.go b/viper.go index 72230f5..8284149 100644 --- a/viper.go +++ b/viper.go @@ -260,6 +260,7 @@ func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { } func WatchConfig() { v.WatchConfig() } + func (v *Viper) WatchConfig() { initWG := sync.WaitGroup{} initWG.Add(1) @@ -294,8 +295,9 @@ func (v *Viper) WatchConfig() { // we only care about the config file with the following cases: // 1 - if the config file was modified or created // 2 - if the real path to the config file changed (eg: k8s ConfigMap replacement) + const writeOrCreateMask = fsnotify.Write | fsnotify.Create if (filepath.Clean(event.Name) == configFile && - (event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create)) || + event.Op&writeOrCreateMask != 0) || (currentConfigFile != "" && currentConfigFile != realConfigFile) { realConfigFile = currentConfigFile err := v.ReadInConfig() @@ -306,7 +308,7 @@ func (v *Viper) WatchConfig() { v.onConfigChange(event) } } else if filepath.Clean(event.Name) == configFile && - event.Op&fsnotify.Remove == fsnotify.Remove { + event.Op&fsnotify.Remove&fsnotify.Remove != 0 { eventsWG.Done() return } @@ -324,6 +326,7 @@ func (v *Viper) WatchConfig() { initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... eventsWG.Wait() // now, wait for event loop to end in this go-routine... }() + fmt.Println(" init WG done") initWG.Wait() // make sure that the go routine above fully ended before returning } diff --git a/viper_test.go b/viper_test.go index 9961970..0985821 100644 --- a/viper_test.go +++ b/viper_test.go @@ -1426,12 +1426,12 @@ func TestWatchFile(t *testing.T) { fmt.Printf("test config file: %s\n", configFile) defer cleanup() wg := sync.WaitGroup{} + wg.Add(1) v.WatchConfig() v.OnConfigChange(func(in fsnotify.Event) { t.Logf("config file changed") wg.Done() }) - wg.Add(1) // when overwriting the file and waiting for the custom change notification handler to be triggered err := ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) wg.Wait() From 907c19d40d9a6c9bb55f040ff4ae45271a4754b9 Mon Sep 17 00:00:00 2001 From: Chris Reeves Date: Thu, 28 Jun 2018 10:55:33 +0100 Subject: [PATCH 06/25] Support customising mapstructure.DecoderConfig for Unmarshal * Added a new `DecoderConfigOption` type allowing the user to write custom functions that can override the default mapstructure.DecoderConfig settings * Added a new `DecodeHook` function which returns a `DecoderConfigOption`. This allows the user to easily set their own Decode hooks when Unmarshaling * Updated Unmarshal, UnmarshalKey and defaultDecoderConfig to support variadic trailing `DecoderConfigOption` functions to allow for customisation of the default mapstructure.DecoderConfig * Added a test case with example usage --- viper.go | 41 +++++++++++++++++++++++++++++++++-------- viper_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/viper.go b/viper.go index 907a102..f657b20 100644 --- a/viper.go +++ b/viper.go @@ -113,6 +113,23 @@ func (fnfe ConfigFileNotFoundError) Error() string { return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) } +// A DecoderConfigOption can be passed to viper.Unmarshal to configure +// mapstructure.DecoderConfig options +type DecoderConfigOption func(*mapstructure.DecoderConfig) + +// DecodeHook returns a DecoderConfigOption which overrides the default +// DecoderConfig.DecodeHook value, the default is: +// +// mapstructure.ComposeDecodeHookFunc( +// mapstructure.StringToTimeDurationHookFunc(), +// mapstructure.StringToSliceHookFunc(","), +// ) +func DecodeHook(hook mapstructure.DecodeHookFunc) DecoderConfigOption { + return func(c *mapstructure.DecoderConfig) { + c.DecodeHook = hook + } +} + // Viper is a prioritized configuration registry. It // maintains a set of configuration sources, fetches // values to populate those, and provides them according @@ -745,9 +762,11 @@ func (v *Viper) GetSizeInBytes(key string) uint { } // UnmarshalKey takes a single key and unmarshals it into a Struct. -func UnmarshalKey(key string, rawVal interface{}) error { return v.UnmarshalKey(key, rawVal) } -func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { - err := decode(v.Get(key), defaultDecoderConfig(rawVal)) +func UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + return v.UnmarshalKey(key, rawVal, opts...) +} +func (v *Viper) UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.Get(key), defaultDecoderConfig(rawVal, opts...)) if err != nil { return err @@ -760,9 +779,11 @@ func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { // Unmarshal unmarshals the config into a Struct. Make sure that the tags // on the fields of the structure are properly set. -func Unmarshal(rawVal interface{}) error { return v.Unmarshal(rawVal) } -func (v *Viper) Unmarshal(rawVal interface{}) error { - err := decode(v.AllSettings(), defaultDecoderConfig(rawVal)) +func Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + return v.Unmarshal(rawVal, opts...) +} +func (v *Viper) Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.AllSettings(), defaultDecoderConfig(rawVal, opts...)) if err != nil { return err @@ -775,8 +796,8 @@ func (v *Viper) Unmarshal(rawVal interface{}) error { // defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot // of time.Duration values & string slices -func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { - return &mapstructure.DecoderConfig{ +func defaultDecoderConfig(output interface{}, opts ...DecoderConfigOption) *mapstructure.DecoderConfig { + c := &mapstructure.DecoderConfig{ Metadata: nil, Result: output, WeaklyTypedInput: true, @@ -785,6 +806,10 @@ func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { mapstructure.StringToSliceHookFunc(","), ), } + for _, opt := range opts { + opt(c) + } + return c } // A wrapper around mapstructure.Decode that mimics the WeakDecode functionality diff --git a/viper_test.go b/viper_test.go index 60543f5..be11f3e 100644 --- a/viper_test.go +++ b/viper_test.go @@ -7,6 +7,7 @@ package viper import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -18,6 +19,7 @@ import ( "testing" "time" + "github.com/mitchellh/mapstructure" "github.com/spf13/afero" "github.com/spf13/cast" @@ -503,6 +505,42 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}, &C) } +func TestUnmarshalWithDecoderOptions(t *testing.T) { + Set("credentials", "{\"foo\":\"bar\"}") + + opt := DecodeHook(mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + // Custom Decode Hook Function + func(rf reflect.Kind, rt reflect.Kind, data interface{}) (interface{}, error) { + if rf != reflect.String || rt != reflect.Map { + return data, nil + } + m := map[string]string{} + raw := data.(string) + if raw == "" { + return m, nil + } + return m, json.Unmarshal([]byte(raw), &m) + }, + )) + + type config struct { + Credentials map[string]string + } + + var C config + + err := Unmarshal(&C, opt) + if err != nil { + t.Fatalf("unable to decode into struct, %v", err) + } + + assert.Equal(t, &config{ + Credentials: map[string]string{"foo": "bar"}, + }, &C) +} + func TestBindPFlags(t *testing.T) { v := New() // create independent Viper object flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) From 41f829b2c9ecf78bcc45a9c18cd9afe1051d86a5 Mon Sep 17 00:00:00 2001 From: Xavier Coulon Date: Mon, 6 Aug 2018 09:29:13 +0200 Subject: [PATCH 07/25] refactor test to avoid negative counter on WG Signed-off-by: Xavier Coulon --- viper.go | 3 +-- viper_test.go | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/viper.go b/viper.go index bda3ea8..630d9df 100644 --- a/viper.go +++ b/viper.go @@ -319,7 +319,7 @@ func (v *Viper) WatchConfig() { realConfigFile = currentConfigFile err := v.ReadInConfig() if err != nil { - log.Printf("error reading file: %v\n", err) + log.Printf("error reading config file: %v\n", err) } if v.onConfigChange != nil { v.onConfigChange(event) @@ -343,7 +343,6 @@ func (v *Viper) WatchConfig() { initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... eventsWG.Wait() // now, wait for event loop to end in this go-routine... }() - fmt.Println(" init WG done") initWG.Wait() // make sure that the go routine above fully ended before returning } diff --git a/viper_test.go b/viper_test.go index 7acd910..76ec02e 100644 --- a/viper_test.go +++ b/viper_test.go @@ -1469,17 +1469,19 @@ func TestWatchFile(t *testing.T) { t.Run("file content changed", func(t *testing.T) { // given a `config.yaml` file being watched v, configFile, cleanup := newViperWithConfigFile(t) - fmt.Printf("test config file: %s\n", configFile) defer cleanup() + _, err := os.Stat(configFile) + require.NoError(t, err) + t.Logf("test config file: %s\n", configFile) wg := sync.WaitGroup{} wg.Add(1) - v.WatchConfig() v.OnConfigChange(func(in fsnotify.Event) { t.Logf("config file changed") wg.Done() }) + v.WatchConfig() // when overwriting the file and waiting for the custom change notification handler to be triggered - err := ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) + err = ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) wg.Wait() // then the config value should have changed require.Nil(t, err) From 40b1bbb9a8eaf94fd560b34dde04fa267a3c5871 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 28 Aug 2018 07:08:14 +0000 Subject: [PATCH 08/25] travis: update go versions (#558) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fa39805..22a8a00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,8 @@ go_import_path: github.com/spf13/viper language: go go: - - 1.9.x - 1.10.x + - 1.11.x - tip os: From e436d04e6d88ca621a687f97fbe6dbbf1195bf04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Br=C3=A4mer?= Date: Tue, 28 Aug 2018 09:29:26 +0200 Subject: [PATCH 09/25] correct a comment on viper.Set() (#553) correct regiser with register --- viper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper.go b/viper.go index f657b20..cfa1241 100644 --- a/viper.go +++ b/viper.go @@ -1139,7 +1139,7 @@ func (v *Viper) SetDefault(key string, value interface{}) { deepestMap[lastKey] = value } -// Set sets the value for the key in the override regiser. +// Set sets the value for the key in the override register. // Set is case-insensitive for a key. // Will be used instead of values obtained via // flags, config file, ENV, default, or key/value store. From b7a62b2c001b4ad829bff1775d83509d19d89a68 Mon Sep 17 00:00:00 2001 From: kun Date: Tue, 28 Aug 2018 15:37:55 +0800 Subject: [PATCH 10/25] fix dep wrong case (#484) --- util.go | 2 +- viper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index 952cad4..e50d69b 100644 --- a/util.go +++ b/util.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" + jww "github.com/spf13/jWalterWeatherman" ) // ConfigParseError denotes failing to parse configuration file. diff --git a/viper.go b/viper.go index cfa1241..16de6c3 100644 --- a/viper.go +++ b/viper.go @@ -42,7 +42,7 @@ import ( toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" + jww "github.com/spf13/jWalterWeatherman" "github.com/spf13/pflag" ) From 05116ad639474f7858b82b48004bfbdb4c51e367 Mon Sep 17 00:00:00 2001 From: Aarti Parikh Date: Tue, 28 Aug 2018 02:34:36 -0600 Subject: [PATCH 11/25] Revert "fix dep wrong case (#484)" This reverts commit b7a62b2c001b4ad829bff1775d83509d19d89a68. --- util.go | 2 +- viper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index e50d69b..952cad4 100644 --- a/util.go +++ b/util.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jWalterWeatherman" + jww "github.com/spf13/jwalterweatherman" ) // ConfigParseError denotes failing to parse configuration file. diff --git a/viper.go b/viper.go index 16de6c3..cfa1241 100644 --- a/viper.go +++ b/viper.go @@ -42,7 +42,7 @@ import ( toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jWalterWeatherman" + jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) From 8addaed22d65eb9b54dc9e9a28537f5b8b9ca756 Mon Sep 17 00:00:00 2001 From: Adhatama Date: Wed, 29 Aug 2018 05:05:06 +0700 Subject: [PATCH 12/25] Add README.md for Consul remote provider (#489) --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index d752822..87bbc8b 100644 --- a/README.md +++ b/README.md @@ -373,12 +373,33 @@ how to use Consul. ### Remote Key/Value Store Example - Unencrypted +#### etcd ```go viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" err := viper.ReadRemoteConfig() ``` +#### Consul +You need to set a key to Consul key/value storage with JSON value containing your desired config. +For example, create a Consul key/value store key `MY_CONSUL_KEY` with value: + +```json +{ + "port": 8080, + "hostname": "myhostname.com" +} +``` + +```go +viper.AddRemoteProvider("consul", "localhost:8500", "MY_CONSUL_KEY") +viper.SetConfigType("json") // Need to explicitly set this to json +err := viper.ReadRemoteConfig() + +fmt.Println(viper.Get("port")) // 8080 +fmt.Println(viper.Get("hostname")) // myhostname.com +``` + ### Remote Key/Value Store Example - Encrypted ```go From 0ac2068de99fd349edc4954d0e9b4a600d75da44 Mon Sep 17 00:00:00 2001 From: "Dr. Tobias Quathamer" Date: Sat, 1 Sep 2018 22:59:01 +0200 Subject: [PATCH 13/25] Fix overflow error on 32 bit architectures (#340) * Handle int64 separately for 32 bit architectures * Remove tests which result in an overflow error on 32 bit architectures --- viper.go | 4 +++- viper_test.go | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/viper.go b/viper.go index cfa1241..051a3ef 100644 --- a/viper.go +++ b/viper.go @@ -648,8 +648,10 @@ func (v *Viper) Get(key string) interface{} { return cast.ToBool(val) case string: return cast.ToString(val) - case int64, int32, int16, int8, int: + case int32, int16, int8, int: return cast.ToInt(val) + case int64: + return cast.ToInt64(val) case float64, float32: return cast.ToFloat64(val) case time.Time: diff --git a/viper_test.go b/viper_test.go index be11f3e..15966e4 100644 --- a/viper_test.go +++ b/viper_test.go @@ -1102,10 +1102,6 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 37890, = %d", pop) } - if pop := v.GetInt("hello.lagrenum"); pop != 765432101234567 { - t.Fatalf("lagrenum != 765432101234567, = %d", pop) - } - if pop := v.GetInt32("hello.pop"); pop != int32(37890) { t.Fatalf("pop != 37890, = %d", pop) } @@ -1130,10 +1126,6 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 45000, = %d", pop) } - if pop := v.GetInt("hello.lagrenum"); pop != 7654321001234567 { - t.Fatalf("lagrenum != 7654321001234567, = %d", pop) - } - if pop := v.GetInt32("hello.pop"); pop != int32(45000) { t.Fatalf("pop != 45000, = %d", pop) } From 8fb642006536c8d3760c99d4fa2389f5e2205631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 7 Sep 2018 11:30:55 +0200 Subject: [PATCH 14/25] Add go.mod --- go.mod | 16 ++++++++++++++++ go.sum | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3f4e1c2 --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/spf13/viper + +require ( + github.com/fsnotify/fsnotify v1.4.7 + github.com/hashicorp/hcl v1.0.0 + github.com/magiconair/properties v1.8.0 + github.com/mitchellh/mapstructure v1.0.0 + github.com/pelletier/go-toml v1.2.0 + github.com/spf13/afero v1.1.2 + github.com/spf13/cast v1.2.0 + github.com/spf13/jwalterweatherman v1.0.0 + github.com/spf13/pflag v1.0.2 + golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 // indirect + golang.org/x/text v0.3.0 // indirect + gopkg.in/yaml.v2 v2.2.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3e3b874 --- /dev/null +++ b/go.sum @@ -0,0 +1,26 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I= +github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 841bd4ebcd794977d3df5c7e868861366415455b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 7 Sep 2018 11:52:15 +0200 Subject: [PATCH 15/25] Add go.sum to .gitignore Seems that it's better practice to keep that in Git for "main modules". --- .gitignore | 2 ++ go.sum | 26 -------------------------- 2 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 go.sum diff --git a/.gitignore b/.gitignore index 352a34a..5548fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ _cgo_defun.c _cgo_gotypes.go _cgo_export.* +go.sum + _testmain.go *.exe diff --git a/go.sum b/go.sum deleted file mode 100644 index 3e3b874..0000000 --- a/go.sum +++ /dev/null @@ -1,26 +0,0 @@ -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I= -github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg= -golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 3171ef9a229903ce60a9513ec3899b63c003e91c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 7 Sep 2018 15:06:02 +0200 Subject: [PATCH 16/25] Revert "Add go.sum to .gitignore" This reverts commit 841bd4ebcd794977d3df5c7e868861366415455b. --- .gitignore | 2 -- go.sum | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 go.sum diff --git a/.gitignore b/.gitignore index 5548fa6..352a34a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,6 @@ _cgo_defun.c _cgo_gotypes.go _cgo_export.* -go.sum - _testmain.go *.exe diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3e3b874 --- /dev/null +++ b/go.sum @@ -0,0 +1,26 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I= +github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 2c12c60302a5a0e62ee102ca9bc996277c2f64f5 Mon Sep 17 00:00:00 2001 From: Andrew Stuart Date: Fri, 28 Sep 2018 00:53:21 -0700 Subject: [PATCH 17/25] Fix nil pointer on watch function (#568) --- viper.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/viper.go b/viper.go index 051a3ef..3debcde 100644 --- a/viper.go +++ b/viper.go @@ -306,7 +306,9 @@ func (v *Viper) WatchConfig() { if err != nil { log.Println("error:", err) } - v.onConfigChange(event) + if v.onConfigChange != nil { + v.onConfigChange(event) + } } } case err := <-watcher.Errors: From 0d783e7344b1c5a1dee7c838c22b4fd9523bf7b4 Mon Sep 17 00:00:00 2001 From: Andrew Stuart Date: Fri, 28 Sep 2018 01:04:19 -0700 Subject: [PATCH 18/25] Use test log --- viper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper_test.go b/viper_test.go index 76ec02e..78d80ef 100644 --- a/viper_test.go +++ b/viper_test.go @@ -1454,7 +1454,7 @@ func newViperWithSymlinkedConfigFile(t *testing.T) (*Viper, string, string, func // and link the `/datadir1/config.yaml` to `/config.yaml` configFile := path.Join(watchDir, "config.yaml") os.Symlink(path.Join(watchDir, "data", "config.yaml"), configFile) - fmt.Printf("Config file location: %s\n", path.Join(watchDir, "config.yaml")) + t.Logf("Config file location: %s\n", path.Join(watchDir, "config.yaml")) // init Viper v := New() v.SetConfigFile(configFile) From b56071875ac96633371c07c0ff77c0ba69f7fd82 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Sun, 30 Sep 2018 06:32:38 +0300 Subject: [PATCH 19/25] Cater for case-sensitive dependencies (#463) Switching jww from jwalterweatherman to jWalterWeatherman for cases when vgo is broken --- util.go | 2 +- viper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index 952cad4..e50d69b 100644 --- a/util.go +++ b/util.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" + jww "github.com/spf13/jWalterWeatherman" ) // ConfigParseError denotes failing to parse configuration file. diff --git a/viper.go b/viper.go index a32ab73..44da43d 100644 --- a/viper.go +++ b/viper.go @@ -43,7 +43,7 @@ import ( toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" + jww "github.com/spf13/jWalterWeatherman" "github.com/spf13/pflag" ) From 62edee319679b6ceaec16de03b966102d2dea709 Mon Sep 17 00:00:00 2001 From: Andrew Stuart Date: Sat, 29 Sep 2018 21:41:27 -0700 Subject: [PATCH 20/25] Revert "Cater for case-sensitive dependencies (#463)" This reverts commit b56071875ac96633371c07c0ff77c0ba69f7fd82. --- util.go | 2 +- viper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index e50d69b..952cad4 100644 --- a/util.go +++ b/util.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jWalterWeatherman" + jww "github.com/spf13/jwalterweatherman" ) // ConfigParseError denotes failing to parse configuration file. diff --git a/viper.go b/viper.go index 44da43d..a32ab73 100644 --- a/viper.go +++ b/viper.go @@ -43,7 +43,7 @@ import ( toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" - jww "github.com/spf13/jWalterWeatherman" + jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) From b7a3b954760cf2c8e97dbcf7f842e721b8d24110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rk=20S=C3=A1gi-Kaz=C3=A1r?= Date: Tue, 6 Nov 2018 22:53:21 +0100 Subject: [PATCH 21/25] Lookup environment variables instead of checking if the value is empty This commit adds an `AllowEmptyEnv` option that, default off, that when set will allow set, but empty, environment variables Fixes #317 --- README.md | 7 ++++++- viper.go | 23 +++++++++++++++++------ viper_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 87bbc8b..0208eac 100644 --- a/README.md +++ b/README.md @@ -179,13 +179,14 @@ viper.GetBool("verbose") // true ### Working with Environment Variables Viper has full support for environment variables. This enables 12 factor -applications out of the box. There are four methods that exist to aid working +applications out of the box. There are five methods that exist to aid working with ENV: * `AutomaticEnv()` * `BindEnv(string...) : error` * `SetEnvPrefix(string)` * `SetEnvKeyReplacer(string...) *strings.Replacer` + * `AllowEmptyEnvVar(bool)` _When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ @@ -217,6 +218,10 @@ keys to an extent. This is useful if you want to use `-` or something in your `Get()` calls, but want your environmental variables to use `_` delimiters. An example of using it can be found in `viper_test.go`. +By default empty environment variables are considered unset and will fall back to +the next configuration source. To treat empty environment variables as set, use +the `AllowEmptyEnv` method. + #### Env example ```go diff --git a/viper.go b/viper.go index a32ab73..1875627 100644 --- a/viper.go +++ b/viper.go @@ -187,6 +187,7 @@ type Viper struct { automaticEnvApplied bool envKeyReplacer *strings.Replacer + allowEmptyEnv bool config map[string]interface{} override map[string]interface{} @@ -373,6 +374,14 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { return strings.ToUpper(in) } +// AllowEmptyEnv tells Viper to consider set, +// but empty environment variables as valid values instead of falling back. +// For backward compatibility reasons this is false by default. +func AllowEmptyEnv(allowEmptyEnv bool) { v.AllowEmptyEnv(allowEmptyEnv) } +func (v *Viper) AllowEmptyEnv(allowEmptyEnv bool) { + v.allowEmptyEnv = allowEmptyEnv +} + // TODO: should getEnv logic be moved into find(). Can generalize the use of // rewriting keys many things, Ex: Get('someKey') -> some_key // (camel case to snake case for JSON keys perhaps) @@ -380,11 +389,14 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { // getEnv is a wrapper around os.Getenv which replaces characters in the original // key. This allows env vars which have different keys than the config object // keys. -func (v *Viper) getEnv(key string) string { +func (v *Viper) getEnv(key string) (string, bool) { if v.envKeyReplacer != nil { key = v.envKeyReplacer.Replace(key) } - return os.Getenv(key) + + val, ok := os.LookupEnv(key) + + return val, ok && (v.allowEmptyEnv || val != "") } // ConfigFileUsed returns the file used to populate the config registry. @@ -611,10 +623,9 @@ func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string { // "foo.bar.baz" in a lower-priority map func (v *Viper) isPathShadowedInAutoEnv(path []string) string { var parentKey string - var val string for i := 1; i < len(path); i++ { parentKey = strings.Join(path[0:i], v.keyDelim) - if val = v.getEnv(v.mergeWithEnvPrefix(parentKey)); val != "" { + if _, ok := v.getEnv(v.mergeWithEnvPrefix(parentKey)); ok { return parentKey } } @@ -993,7 +1004,7 @@ func (v *Viper) find(lcaseKey string) interface{} { if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request - if val = v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); val != "" { + if val, ok := v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); ok { return val } if nested && v.isPathShadowedInAutoEnv(path) != "" { @@ -1002,7 +1013,7 @@ func (v *Viper) find(lcaseKey string) interface{} { } envkey, exists := v.env[lcaseKey] if exists { - if val = v.getEnv(envkey); val != "" { + if val, ok := v.getEnv(envkey); ok { return val } } diff --git a/viper_test.go b/viper_test.go index c8fa1f4..f7262a7 100644 --- a/viper_test.go +++ b/viper_test.go @@ -388,6 +388,36 @@ func TestEnv(t *testing.T) { } +func TestEmptyEnv(t *testing.T) { + initJSON() + + BindEnv("type") // Empty environment variable + BindEnv("name") // Bound, but not set environment variable + + os.Clearenv() + + os.Setenv("TYPE", "") + + assert.Equal(t, "donut", Get("type")) + assert.Equal(t, "Cake", Get("name")) +} + +func TestEmptyEnv_Allowed(t *testing.T) { + initJSON() + + AllowEmptyEnv(true) + + BindEnv("type") // Empty environment variable + BindEnv("name") // Bound, but not set environment variable + + os.Clearenv() + + os.Setenv("TYPE", "") + + assert.Equal(t, "", Get("type")) + assert.Equal(t, "Cake", Get("name")) +} + func TestEnvPrefix(t *testing.T) { initJSON() From cc7e906d8847d935cffacac702f8e9d98156d604 Mon Sep 17 00:00:00 2001 From: Benoit Masson Date: Mon, 17 Apr 2017 18:33:30 +0200 Subject: [PATCH 22/25] Updated TestBindPFlagsStringSlice() to highlight a failure When pflag marked as changed, the value is not detected (and lower priority value used) --- viper_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/viper_test.go b/viper_test.go index f7262a7..d5c637d 100644 --- a/viper_test.go +++ b/viper_test.go @@ -613,6 +613,8 @@ func TestBindPFlags(t *testing.T) { } func TestBindPFlagsStringSlice(t *testing.T) { + defaultVal := []string{"default"} + for _, testValue := range []struct { Expected []string Value string @@ -624,6 +626,8 @@ func TestBindPFlagsStringSlice(t *testing.T) { for _, changed := range []bool{true, false} { v := New() // create independent Viper object + v.SetDefault("stringslice", defaultVal) + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) flagSet.StringSlice("stringslice", testValue.Expected, "test") flagSet.Visit(func(f *pflag.Flag) { @@ -645,7 +649,11 @@ func TestBindPFlagsStringSlice(t *testing.T) { if err := v.Unmarshal(val); err != nil { t.Fatalf("%+#v cannot unmarshal: %s", testValue.Value, err) } - assert.Equal(t, testValue.Expected, val.StringSlice) + if changed { + assert.Equal(t, testValue.Expected, val.StringSlice) + } else { + assert.Equal(t, defaultVal, val.StringSlice) + } } } } From 69647fb42256db99f5b51b9ffa4a423ce2d38aba Mon Sep 17 00:00:00 2001 From: Benoit Masson Date: Mon, 17 Apr 2017 18:52:25 +0200 Subject: [PATCH 23/25] Fixed TestBindPFlagsStringSlice() Replaced Visit() by VisitAll(), so that the Changed attribute of the updated flag is correctly set. --- viper_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/viper_test.go b/viper_test.go index d5c637d..21b49c9 100644 --- a/viper_test.go +++ b/viper_test.go @@ -630,11 +630,9 @@ func TestBindPFlagsStringSlice(t *testing.T) { flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) flagSet.StringSlice("stringslice", testValue.Expected, "test") - flagSet.Visit(func(f *pflag.Flag) { - if len(testValue.Value) > 0 { - f.Value.Set(testValue.Value) - f.Changed = changed - } + flagSet.VisitAll(func(f *pflag.Flag) { + f.Value.Set(testValue.Value) + f.Changed = changed }) err := v.BindPFlags(flagSet) From ae103d7e593e371c69e832d5eb3347e2b80cbbc9 Mon Sep 17 00:00:00 2001 From: Benoit Masson Date: Mon, 17 Apr 2017 18:52:44 +0200 Subject: [PATCH 24/25] Moved shared resources out of the loops in TestBindPFlagsStringSlice() Common code and resources put out of the loops, to improve efficiency and readability. --- viper_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/viper_test.go b/viper_test.go index 21b49c9..fa3903e 100644 --- a/viper_test.go +++ b/viper_test.go @@ -613,23 +613,25 @@ func TestBindPFlags(t *testing.T) { } func TestBindPFlagsStringSlice(t *testing.T) { - defaultVal := []string{"default"} - - for _, testValue := range []struct { + tests := []struct { Expected []string Value string }{ {[]string{}, ""}, {[]string{"jeden"}, "jeden"}, {[]string{"dwa", "trzy"}, "dwa,trzy"}, - {[]string{"cztery", "piec , szesc"}, "cztery,\"piec , szesc\""}} { + {[]string{"cztery", "piec , szesc"}, "cztery,\"piec , szesc\""}, + } + + v := New() // create independent Viper object + defaultVal := []string{"default"} + v.SetDefault("stringslice", defaultVal) + + for _, testValue := range tests { + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + flagSet.StringSlice("stringslice", testValue.Expected, "test") for _, changed := range []bool{true, false} { - v := New() // create independent Viper object - v.SetDefault("stringslice", defaultVal) - - flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) - flagSet.StringSlice("stringslice", testValue.Expected, "test") flagSet.VisitAll(func(f *pflag.Flag) { f.Value.Set(testValue.Value) f.Changed = changed From 06c7c0d9b3c7cfca13deff6e353a052e08828ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 19 Nov 2018 10:38:40 +0100 Subject: [PATCH 25/25] Use assert.EqualValues for slice checking --- viper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper_test.go b/viper_test.go index fa3903e..6097867 100644 --- a/viper_test.go +++ b/viper_test.go @@ -650,7 +650,7 @@ func TestBindPFlagsStringSlice(t *testing.T) { t.Fatalf("%+#v cannot unmarshal: %s", testValue.Value, err) } if changed { - assert.Equal(t, testValue.Expected, val.StringSlice) + assert.EqualValues(t, testValue.Expected, val.StringSlice) } else { assert.Equal(t, defaultVal, val.StringSlice) }