diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 29b63b666df3..9df99cb41c1b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -19,6 +19,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Update the command line library cobra and add support for zsh completion {pull}5761[5761] - The log format may differ due to logging library changes. {pull}5901[5901] - Adding a local keystore to allow user to obfuscate password {pull}5687[5687] +- Update go-ucfg library to support top level key reference and cyclic key reference for the + keystore {pull}6098[6098] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index abb4f8e022f8..ca91b40c4f8f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -361,7 +361,8 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-ucfg -Revision: ec8488a52542c0c51e42e8ea204dcaff400bc644 +Version: v0.5.0 +Revision: bda09c7b9afc6263a3fd592fcd7063d03f6acf0f License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-ucfg/LICENSE: -------------------------------------------------------------------- diff --git a/libbeat/keystore/file_keystore.go b/libbeat/keystore/file_keystore.go index 35edce454660..4ca43ec18bf1 100644 --- a/libbeat/keystore/file_keystore.go +++ b/libbeat/keystore/file_keystore.go @@ -13,7 +13,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "sync" "golang.org/x/crypto/pbkdf2" @@ -89,9 +88,6 @@ func (k *FileKeystore) Retrieve(key string) (*SecureString, error) { // Store add the key pair to the secret store and mark the store as dirty. func (k *FileKeystore) Store(key string, value []byte) error { - if err := k.validateKey(key); err != nil { - return err - } k.Lock() defer k.Unlock() @@ -386,11 +382,3 @@ func (k *FileKeystore) checkPermissions(f string) error { func (k *FileKeystore) hashPassword(password, salt []byte) []byte { return pbkdf2.Key(password, salt, iterationsCount, keyLength, sha512.New) } - -func (k *FileKeystore) validateKey(key string) error { - if strings.IndexAny(key, ".") != -1 { - return fmt.Errorf("invalid key format. '.' in keys are not supported yet. key: %s", key) - } - - return nil -} diff --git a/libbeat/keystore/file_keystore_test.go b/libbeat/keystore/file_keystore_test.go index baaf6599c0cc..4d4a9f8d2d32 100644 --- a/libbeat/keystore/file_keystore_test.go +++ b/libbeat/keystore/file_keystore_test.go @@ -13,21 +13,9 @@ import ( "github.com/elastic/beats/libbeat/common" ) -var keyValue = "output_elasticsearch_password" +var keyValue = "output.elasticsearch.password" var secretValue = []byte("secret") -func TestInvalidKey(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keystore, err := NewFileKeystore(path) - assert.NoError(t, err) - err = keystore.Store("output.elasticsearch.password", secretValue) - if assert.Error(t, err) { - assert.Equal(t, err.Error(), "invalid key format. '.' in keys are not supported yet. key: output.elasticsearch.password") - } -} - func TestCanCreateAKeyStore(t *testing.T) { path := GetTemporaryKeystoreFile() defer os.Remove(path) @@ -197,18 +185,18 @@ func TestGetConfig(t *testing.T) { keystore := CreateAnExistingKeystore(path) // Add a bit more data of different type - keystore.Store("super_nested", []byte("hello")) + keystore.Store("super.nested", []byte("hello")) keystore.Save() cfg, err := keystore.GetConfig() assert.NotNil(t, cfg) assert.NoError(t, err) - secret, err := cfg.String("output_elasticsearch_password", 0) + secret, err := cfg.String("output.elasticsearch.password", 0) assert.NoError(t, err) assert.Equal(t, secret, "secret") - port, err := cfg.String("super_nested", 0) + port, err := cfg.String("super.nested", 0) assert.Equal(t, port, "hello") } diff --git a/libbeat/keystore/keystore_test.go b/libbeat/keystore/keystore_test.go index 5346cb4784b9..5b7167db0c11 100644 --- a/libbeat/keystore/keystore_test.go +++ b/libbeat/keystore/keystore_test.go @@ -27,7 +27,7 @@ func TestResolverWhenTheKeyExist(t *testing.T) { keystore := CreateAnExistingKeystore(path) resolver := ResolverWrap(keystore) - v, err := resolver("output_elasticsearch_password") + v, err := resolver("output.elasticsearch.password") assert.NoError(t, err) assert.Equal(t, v, "secret") } diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index f77b1e17ef04..84f5ddb2f5a5 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -49,3 +49,24 @@ def test_keystore_with_key_not_present(self): assert self.log_contains( "missing field accessing 'output.elasticsearch.hosts'") assert exit_code == 1 + + def test_keystore_with_nested_key(self): + """ + test that we support nested key + """ + + key = "output.elasticsearch.hosts.0" + secret = "myeleasticsearchsecrethost" + + self.render_config_template(keystore_path=self.keystore_path, elasticsearch={ + 'hosts': "${%s}" % key + }) + + exit_code = self.run_beat(extra_args=["keystore", "create"]) + assert exit_code == 0 + + self.add_secret(key, secret) + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("no such host")) + assert self.log_contains(secret) + proc.check_kill_and_wait() diff --git a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md index b3359fea4949..cb1e377bc432 100644 --- a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md +++ b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md @@ -14,6 +14,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [0.5.0] + +### Added +- Detect cyclic reference and allow to search top level key with the other resolvers. #97 +- Allow to diff keys of two different configuration #93 + ## [0.4.6] ### Added @@ -34,7 +40,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [0.4.4] ### Added -- Add support for pure array config files #82 +- Add support for pure array config files #82 ### Changed - Invalid top-level types return non-critical error (no stack-trace) on merge #82 @@ -177,7 +183,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Introduced CHANGELOG.md for documenting changes to ucfg. -[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.4.6...HEAD +[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.5.0...HEAD +[0.5.0]: https://github.com/elastic/go-ucfg/compare/v0.4.6...v0.5.0 [0.4.6]: https://github.com/elastic/go-ucfg/compare/v0.4.5...v0.4.6 [0.4.5]: https://github.com/elastic/go-ucfg/compare/v0.4.4...v0.4.5 [0.4.4]: https://github.com/elastic/go-ucfg/compare/v0.4.3...v0.4.4 diff --git a/vendor/github.com/elastic/go-ucfg/error.go b/vendor/github.com/elastic/go-ucfg/error.go index 5c8e8114c796..870ea67b0a53 100644 --- a/vendor/github.com/elastic/go-ucfg/error.go +++ b/vendor/github.com/elastic/go-ucfg/error.go @@ -44,6 +44,8 @@ type criticalError struct { var ( ErrMissing = errors.New("missing field") + ErrCyclicReference = errors.New("cyclic reference detected") + ErrDuplicateValidator = errors.New("validator already registered") ErrTypeNoArray = errors.New("field is no array") @@ -66,7 +68,7 @@ var ( ErrTODO = errors.New("TODO - implement me") - ErrDuplicateKeey = errors.New("duplicate key") + ErrDuplicateKey = errors.New("duplicate key") ErrOverflow = errors.New("integer overflow") @@ -161,7 +163,17 @@ func messagePath(reason error, meta *Meta, message, path string) string { } func raiseDuplicateKey(cfg *Config, name string) Error { - return raisePathErr(ErrDuplicateKeey, cfg.metadata, "", cfg.PathOf(name, ".")) + return raisePathErr(ErrDuplicateKey, cfg.metadata, "", cfg.PathOf(name, ".")) +} + +func raiseCyclicErr(field string) Error { + message := fmt.Sprintf("cyclic reference detected for key: '%s'", field) + + return baseError{ + reason: ErrCyclicReference, + class: ErrConfig, + message: message, + } } func raiseMissing(c *Config, field string) Error { diff --git a/vendor/github.com/elastic/go-ucfg/errpred.go b/vendor/github.com/elastic/go-ucfg/errpred.go new file mode 100644 index 000000000000..7379068004a5 --- /dev/null +++ b/vendor/github.com/elastic/go-ucfg/errpred.go @@ -0,0 +1,24 @@ +package ucfg + +func isCyclicError(err error) bool { + switch v := err.(type) { + case Error: + return v.Reason() == ErrCyclicReference + } + return false +} + +func isMissingError(err error) bool { + switch v := err.(type) { + case Error: + return v.Reason() == ErrMissing + } + return false +} + +func criticalResolveError(err error) bool { + if err == nil { + return false + } + return !(isCyclicError(err) || isMissingError(err)) +} diff --git a/vendor/github.com/elastic/go-ucfg/fieldset.go b/vendor/github.com/elastic/go-ucfg/fieldset.go new file mode 100644 index 000000000000..9fc86a7bac2c --- /dev/null +++ b/vendor/github.com/elastic/go-ucfg/fieldset.go @@ -0,0 +1,43 @@ +package ucfg + +type fieldSet struct { + fields map[string]struct{} + parent *fieldSet +} + +func NewFieldSet(parent *fieldSet) *fieldSet { + return &fieldSet{ + fields: map[string]struct{}{}, + parent: parent, + } +} + +func (s *fieldSet) Has(name string) (exists bool) { + if _, exists = s.fields[name]; !exists && s.parent != nil { + exists = s.parent.Has(name) + } + return +} + +func (s *fieldSet) Add(name string) { + s.fields[name] = struct{}{} +} + +func (s *fieldSet) AddNew(name string) (ok bool) { + if ok = !s.Has(name); ok { + s.Add(name) + } + return +} + +func (s *fieldSet) Names() []string { + var names []string + for k := range s.fields { + names = append(names, k) + } + + if s.parent != nil { + names = append(names, s.parent.Names()...) + } + return names +} diff --git a/vendor/github.com/elastic/go-ucfg/opts.go b/vendor/github.com/elastic/go-ucfg/opts.go index c2ecd9fe958c..cb36b9678e9a 100644 --- a/vendor/github.com/elastic/go-ucfg/opts.go +++ b/vendor/github.com/elastic/go-ucfg/opts.go @@ -1,6 +1,8 @@ package ucfg -import "os" +import ( + "os" +) // Option type implementing additional options to be passed // to go-ucfg library functions. @@ -18,6 +20,8 @@ type options struct { // temporary cache of parsed splice values for lifetime of call to // Unpack/Pack/Get/... parsed valueCache + + activeFields *fieldSet } type valueCache map[string]spliceValue @@ -111,6 +115,7 @@ func makeOptions(opts []Option) *options { validatorTag: "validate", pathSep: "", // no separator by default parsed: map[string]spliceValue{}, + activeFields: NewFieldSet(nil), } for _, opt := range opts { opt(&o) @@ -130,6 +135,10 @@ func (cache valueCache) cachedValue( } v, err := f() - cache[string(id)] = spliceValue{err, v} + + // Only primitives can be cached, allowing us to get out of infinite loop + if v != nil && v.canCache() { + cache[string(id)] = spliceValue{err, v} + } return v, err } diff --git a/vendor/github.com/elastic/go-ucfg/reify.go b/vendor/github.com/elastic/go-ucfg/reify.go index d28e68d7a35c..c9a256c0e7f2 100644 --- a/vendor/github.com/elastic/go-ucfg/reify.go +++ b/vendor/github.com/elastic/go-ucfg/reify.go @@ -151,6 +151,9 @@ func reifyInto(opts *options, to reflect.Value, from *Config) Error { } func reifyMap(opts *options, to reflect.Value, from *Config) Error { + parentFields := opts.activeFields + defer func() { opts.activeFields = parentFields }() + if to.Type().Key().Kind() != reflect.String { return raiseKeyInvalidTypeUnpack(to.Type(), from) } @@ -164,6 +167,7 @@ func reifyMap(opts *options, to reflect.Value, from *Config) Error { to.Set(reflect.MakeMap(to.Type())) } for k, value := range fields { + opts.activeFields = NewFieldSet(parentFields) key := reflect.ValueOf(k) old := to.MapIndex(key) @@ -186,6 +190,9 @@ func reifyMap(opts *options, to reflect.Value, from *Config) Error { } func reifyStruct(opts *options, orig reflect.Value, cfg *Config) Error { + parentFields := opts.activeFields + defer func() { opts.activeFields = parentFields }() + orig = chaseValuePointers(orig) to := chaseValuePointers(reflect.New(chaseTypePointers(orig.Type()))) @@ -212,6 +219,8 @@ func reifyStruct(opts *options, orig reflect.Value, cfg *Config) Error { continue } + opts.activeFields = NewFieldSet(parentFields) + vField := to.Field(i) validators, err := parseValidatorTags(stField.Tag.Get(opts.validatorTag)) if err != nil { diff --git a/vendor/github.com/elastic/go-ucfg/types.go b/vendor/github.com/elastic/go-ucfg/types.go index 3bb266819c1d..b926b862b20a 100644 --- a/vendor/github.com/elastic/go-ucfg/types.go +++ b/vendor/github.com/elastic/go-ucfg/types.go @@ -8,8 +8,9 @@ import ( "strings" "sync/atomic" - "github.com/elastic/go-ucfg/internal/parse" uuid "github.com/satori/go.uuid" + + "github.com/elastic/go-ucfg/internal/parse" ) type value interface { @@ -34,6 +35,7 @@ type value interface { toUint(opts *options) (uint64, error) toFloat(opts *options) (float64, error) toConfig(opts *options) (*Config, error) + canCache() bool } type typeInfo struct { @@ -182,6 +184,7 @@ func (cfgPrimitive) toInt(*options) (int64, error) { return 0, ErrTypeMisma func (cfgPrimitive) toUint(*options) (uint64, error) { return 0, ErrTypeMismatch } func (cfgPrimitive) toFloat(*options) (float64, error) { return 0, ErrTypeMismatch } func (cfgPrimitive) toConfig(*options) (*Config, error) { return nil, ErrTypeMismatch } +func (cfgPrimitive) canCache() bool { return true } func (c *cfgNil) cpy(ctx context) value { return &cfgNil{cfgPrimitive{ctx, c.metadata}} } func (*cfgNil) Len(*options) (int, error) { return 0, nil } @@ -283,6 +286,7 @@ func (cfgSub) toInt(*options) (int64, error) { return 0, ErrTypeMismatch func (cfgSub) toUint(*options) (uint64, error) { return 0, ErrTypeMismatch } func (cfgSub) toFloat(*options) (float64, error) { return 0, ErrTypeMismatch } func (c cfgSub) toConfig(*options) (*Config, error) { return c.c, nil } +func (c cfgSub) canCache() bool { return false } func (c cfgSub) Len(*options) (int, error) { arr := c.c.fields.array() @@ -343,6 +347,9 @@ func (c cfgSub) SetContext(ctx context) { } func (c cfgSub) reify(opts *options) (interface{}, error) { + parentFields := opts.activeFields + defer func() { opts.activeFields = parentFields }() + fields := c.c.fields.dict() arr := c.c.fields.array() @@ -352,6 +359,7 @@ func (c cfgSub) reify(opts *options) (interface{}, error) { case len(fields) > 0 && len(arr) == 0: m := make(map[string]interface{}) for k, v := range fields { + opts.activeFields = NewFieldSet(parentFields) var err error if m[k], err = v.reify(opts); err != nil { return nil, err @@ -361,6 +369,7 @@ func (c cfgSub) reify(opts *options) (interface{}, error) { case len(fields) == 0 && len(arr) > 0: m := make([]interface{}, len(arr)) for i, v := range arr { + opts.activeFields = NewFieldSet(parentFields) var err error if m[i], err = v.reify(opts); err != nil { return nil, err @@ -370,12 +379,14 @@ func (c cfgSub) reify(opts *options) (interface{}, error) { default: m := make(map[string]interface{}) for k, v := range fields { + opts.activeFields = NewFieldSet(parentFields) var err error if m[k], err = v.reify(opts); err != nil { return nil, err } } for i, v := range arr { + opts.activeFields = NewFieldSet(parentFields) var err error m[fmt.Sprintf("%d", i)], err = v.reify(opts) if err != nil { @@ -473,6 +484,10 @@ func (d *cfgDynamic) getValue(opts *options) (value, error) { }) } +func (d cfgDynamic) canCache() bool { + return false +} + func (r *refDynValue) String() string { ref := (*reference)(r) return ref.String() @@ -484,12 +499,20 @@ func (r *refDynValue) getValue( ) (value, error) { ref := (*reference)(r) v, err := ref.resolveRef(p.ctx.getParent(), opts) - if v != nil || err != nil { + // If not found or we have a cyclic reference we try the environment resolvers + if v != nil || criticalResolveError(err) { return v, err } + previousErr := err str, err := ref.resolveEnv(p.ctx.getParent(), opts) if err != nil { + // TODO(ph): Not everything is an Error, will do some cleanup in another PR. + if v, ok := previousErr.(Error); ok { + if v.Reason() == ErrCyclicReference { + return nil, previousErr + } + } return nil, err } return parseValue(p, opts, str) diff --git a/vendor/github.com/elastic/go-ucfg/ucfg.go b/vendor/github.com/elastic/go-ucfg/ucfg.go index f0ee3bcef0e7..ebad946a4c98 100644 --- a/vendor/github.com/elastic/go-ucfg/ucfg.go +++ b/vendor/github.com/elastic/go-ucfg/ucfg.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "regexp" + "sort" "time" ) @@ -132,6 +133,47 @@ func (c *Config) Parent() *Config { } } +// FlattenedKeys return a sorted flattened views of the set keys in the configuration +func (c *Config) FlattenedKeys(opts ...Option) []string { + var keys []string + normalizedOptions := makeOptions(opts) + + if normalizedOptions.pathSep == "" { + normalizedOptions.pathSep = "." + } + + if c.IsDict() { + for _, v := range c.fields.dict() { + + subcfg, err := v.toConfig(normalizedOptions) + if err != nil { + ctx := v.Context() + p := ctx.path(normalizedOptions.pathSep) + keys = append(keys, p) + } else { + newKeys := subcfg.FlattenedKeys(opts...) + keys = append(keys, newKeys...) + } + } + } else if c.IsArray() { + for _, a := range c.fields.array() { + scfg, err := a.toConfig(normalizedOptions) + + if err != nil { + ctx := a.Context() + p := ctx.path(normalizedOptions.pathSep) + keys = append(keys, p) + } else { + newKeys := scfg.FlattenedKeys(opts...) + keys = append(keys, newKeys...) + } + } + } + + sort.Strings(keys) + return keys +} + func (f *fields) get(name string) (value, bool) { if f.d == nil { return nil, false diff --git a/vendor/github.com/elastic/go-ucfg/variables.go b/vendor/github.com/elastic/go-ucfg/variables.go index b80376862599..8d45b7473f5a 100644 --- a/vendor/github.com/elastic/go-ucfg/variables.go +++ b/vendor/github.com/elastic/go-ucfg/variables.go @@ -89,7 +89,12 @@ func (r *reference) String() string { func (r *reference) resolveRef(cfg *Config, opts *options) (value, error) { env := opts.env - var err error + + if ok := opts.activeFields.AddNew(r.Path.String()); !ok { + return nil, raiseCyclicErr(r.Path.String()) + } + + var err Error for { var v value @@ -103,6 +108,7 @@ func (r *reference) resolveRef(cfg *Config, opts *options) (value, error) { if v == nil { break } + return v, nil } @@ -137,15 +143,27 @@ func (r *reference) resolveEnv(cfg *Config, opts *options) (string, error) { func (r *reference) resolve(cfg *Config, opts *options) (value, error) { v, err := r.resolveRef(cfg, opts) - if v != nil || err != nil { + if v != nil || criticalResolveError(err) { return v, err } + previousErr := err + s, err := r.resolveEnv(cfg, opts) - if s == "" || err != nil { + if err != nil { + // TODO(ph): Not everything is an Error, will do some cleanup in another PR. + if v, ok := previousErr.(Error); ok { + if v.Reason() == ErrCyclicReference { + return nil, previousErr + } + } return nil, err } + if s == "" { + return nil, nil + } + return newString(context{field: r.Path.String()}, nil, s), nil } diff --git a/vendor/vendor.json b/vendor/vendor.json index 15fdeefb4b4b..306183eaa084 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -443,10 +443,12 @@ "revisionTime": "2016-06-17T14:03:01Z" }, { - "checksumSHA1": "6UaJp3hUpEJvcu7COoRNhHPlqfg=", + "checksumSHA1": "FPMs2e/K5HRqZyFvU/VTioGBnwk=", "path": "github.com/elastic/go-ucfg", - "revision": "ec8488a52542c0c51e42e8ea204dcaff400bc644", - "revisionTime": "2017-02-07T06:38:51Z" + "revision": "bda09c7b9afc6263a3fd592fcd7063d03f6acf0f", + "revisionTime": "2018-01-22T22:35:02Z", + "version": "v0.5.0", + "versionExact": "v0.5.0" }, { "checksumSHA1": "8cr5YhslUMgpvF2JebYvKC+Ezr4=",