From 656cffb6d1644a9a305f56caf0a5c8c0827cce95 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 1 Mar 2021 15:04:02 -0500 Subject: [PATCH] Remove saved objects code (#110) * Remove saved objects code * Make check happy --- NOTICE.txt | 136 ++++++------- cmd/fleet/bulkCheckin.go | 15 +- cmd/fleet/dsl.go | 36 ---- cmd/fleet/handleCheckin.go | 7 +- cmd/fleet/main.go | 6 +- go.mod | 2 - go.sum | 2 - internal/pkg/migrate/migrate.go | 121 ------------ internal/pkg/saved/crud.go | 338 -------------------------------- internal/pkg/saved/crypto.go | 180 ----------------- internal/pkg/saved/encode.go | 232 ---------------------- internal/pkg/saved/errors.go | 23 --- internal/pkg/saved/fields.go | 53 ----- internal/pkg/saved/id.go | 63 ------ internal/pkg/saved/nonce.go | 39 ---- internal/pkg/saved/opts.go | 84 -------- internal/pkg/saved/query.go | 237 ---------------------- 17 files changed, 80 insertions(+), 1494 deletions(-) delete mode 100644 cmd/fleet/dsl.go delete mode 100644 internal/pkg/migrate/migrate.go delete mode 100644 internal/pkg/saved/crud.go delete mode 100644 internal/pkg/saved/crypto.go delete mode 100644 internal/pkg/saved/encode.go delete mode 100644 internal/pkg/saved/errors.go delete mode 100644 internal/pkg/saved/fields.go delete mode 100644 internal/pkg/saved/id.go delete mode 100644 internal/pkg/saved/nonce.go delete mode 100644 internal/pkg/saved/opts.go delete mode 100644 internal/pkg/saved/query.go diff --git a/NOTICE.txt b/NOTICE.txt index af04cc594..b7f8e4f8a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1755,37 +1755,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/mitchellh/mapstructure -Version: v1.3.3 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.3.3/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2013 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/rs/xid Version: v1.2.1 @@ -2061,43 +2030,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/crypto -Version: v0.0.0-20200622213623-75b288015ac9 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200622213623-75b288015ac9/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/sync Version: v0.0.0-20200625203802-6e8e738ad208 @@ -27443,6 +27375,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/mitchellh/mapstructure +Version: v1.1.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.1.2/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/modern-go/concurrent Version: v0.0.0-20180306012644-bacd9c7ef1dd @@ -35190,6 +35153,43 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/crypto +Version: v0.0.0-20200622213623-75b288015ac9 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200622213623-75b288015ac9/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/exp Version: v0.0.0-20191227195350-da58074b4299 diff --git a/cmd/fleet/bulkCheckin.go b/cmd/fleet/bulkCheckin.go index 17496da15..bb508c917 100644 --- a/cmd/fleet/bulkCheckin.go +++ b/cmd/fleet/bulkCheckin.go @@ -12,15 +12,16 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/dl" - "github.com/elastic/fleet-server/v7/internal/pkg/saved" "github.com/rs/zerolog/log" ) +type Fields map[string]interface{} + const kBulkCheckinFlushInterval = 10 * time.Second type PendingData struct { - fields saved.Fields + fields Fields seqNo int64 } @@ -37,10 +38,10 @@ func NewBulkCheckin(bulker bulk.Bulk) *BulkCheckin { } } -func (bc *BulkCheckin) CheckIn(id string, fields saved.Fields, seqno int64) error { +func (bc *BulkCheckin) CheckIn(id string, fields Fields, seqno int64) error { if fields == nil { - fields = make(saved.Fields) + fields = make(Fields) } timeNow := time.Now().UTC().Format(time.RFC3339) @@ -52,7 +53,7 @@ func (bc *BulkCheckin) CheckIn(id string, fields saved.Fields, seqno int64) erro return nil } -func (bc *BulkCheckin) Run(ctx context.Context, sv saved.CRUD) error { +func (bc *BulkCheckin) Run(ctx context.Context) error { tick := time.NewTicker(kBulkCheckinFlushInterval) @@ -61,7 +62,7 @@ LOOP: for { select { case <-tick.C: - if err = bc.flush(ctx, sv); err != nil { + if err = bc.flush(ctx); err != nil { log.Error().Err(err).Msg("Eat bulk checkin error; Keep on truckin'") err = nil } @@ -75,7 +76,7 @@ LOOP: return err } -func (bc *BulkCheckin) flush(ctx context.Context, sv saved.CRUD) error { +func (bc *BulkCheckin) flush(ctx context.Context) error { start := time.Now() bc.mut.Lock() diff --git a/cmd/fleet/dsl.go b/cmd/fleet/dsl.go deleted file mode 100644 index b11c215b8..000000000 --- a/cmd/fleet/dsl.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - "github.com/elastic/fleet-server/v7/internal/pkg/dsl" - "github.com/elastic/fleet-server/v7/internal/pkg/saved" -) - -const ( - kTmplApiKeyField = "ApiKeyId" - kTmplAgentIdField = "AgentIdList" -) - -var agentActionQueryTmpl = genAgentActionQueryTemplate() - -func genAgentActionQueryTemplate() *dsl.Tmpl { - tmpl := dsl.NewTmpl() - token := tmpl.Bind(kTmplAgentIdField) - - root := saved.NewQuery(AGENT_ACTION_SAVED_OBJECT_TYPE) - - fieldSentAt := saved.ScopeField(AGENT_ACTION_SAVED_OBJECT_TYPE, "sent_at") - fieldAgentId := saved.ScopeField(AGENT_ACTION_SAVED_OBJECT_TYPE, "agent_id") - - root.Query().Bool().Must().Terms(fieldAgentId, token, nil) - root.Query().Bool().MustNot().Exists(fieldSentAt) - - if err := tmpl.Resolve(root); err != nil { - panic(err) - } - - return tmpl -} diff --git a/cmd/fleet/handleCheckin.go b/cmd/fleet/handleCheckin.go index f074eedac..7f57d90db 100644 --- a/cmd/fleet/handleCheckin.go +++ b/cmd/fleet/handleCheckin.go @@ -21,7 +21,6 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/model" "github.com/elastic/fleet-server/v7/internal/pkg/monitor" "github.com/elastic/fleet-server/v7/internal/pkg/policy" - "github.com/elastic/fleet-server/v7/internal/pkg/saved" "github.com/julienschmidt/httprouter" "github.com/rs/zerolog/log" @@ -347,7 +346,7 @@ func findAgentByApiKeyId(ctx context.Context, bulker bulk.Bulk, id string) (*mod // parseMeta compares the agent and the request local_metadata content // and returns fields to update the agent record or nil -func parseMeta(agent *model.Agent, req *CheckinRequest) (fields saved.Fields, err error) { +func parseMeta(agent *model.Agent, req *CheckinRequest) (fields Fields, err error) { // Quick comparison first if bytes.Equal(req.LocalMeta, agent.LocalMetadata) { log.Trace().Msg("Quick comparing local metadata is equal") @@ -355,8 +354,8 @@ func parseMeta(agent *model.Agent, req *CheckinRequest) (fields saved.Fields, er } // Compare local_metadata content and update if different - var reqLocalMeta saved.Fields - var agentLocalMeta saved.Fields + var reqLocalMeta Fields + var agentLocalMeta Fields err = json.Unmarshal(req.LocalMeta, &reqLocalMeta) if err != nil { return nil, err diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index 68d918c3e..15fe89984 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -27,7 +27,6 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/policy" "github.com/elastic/fleet-server/v7/internal/pkg/profile" "github.com/elastic/fleet-server/v7/internal/pkg/reload" - "github.com/elastic/fleet-server/v7/internal/pkg/saved" "github.com/elastic/fleet-server/v7/internal/pkg/signal" "github.com/elastic/fleet-server/v7/internal/pkg/status" @@ -469,7 +468,6 @@ func (f *FleetServer) runServer(ctx context.Context, cfg *config.Config) (err er if err != nil { return err } - sv := saved.NewMgr(bulker, savedObjectKey()) // Replacing to errgroup context g, ctx := errgroup.WithContext(ctx) @@ -512,9 +510,7 @@ func (f *FleetServer) runServer(ctx context.Context, cfg *config.Config) (err er } bc := NewBulkCheckin(bulker) - g.Go(loggedRunFunc(ctx, "Bulk checkin", func(ctx context.Context) error { - return bc.Run(ctx, sv) - })) + g.Go(loggedRunFunc(ctx, "Bulk checkin", bc.Run)) ct := NewCheckinT(f.cfg, f.cache, bc, pm, am, ad, tr, bulker) et, err := NewEnrollerT(&f.cfg.Inputs[0].Server, bulker, f.cache) diff --git a/go.mod b/go.mod index 13460aef5..a7cb815b1 100644 --- a/go.mod +++ b/go.mod @@ -14,12 +14,10 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.1 github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d github.com/julienschmidt/httprouter v1.3.0 - github.com/mitchellh/mapstructure v1.3.3 github.com/rs/xid v1.2.1 github.com/rs/zerolog v1.19.0 github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.6.1 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e ) diff --git a/go.sum b/go.sum index 15f04d599..19430dbad 100644 --- a/go.sum +++ b/go.sum @@ -360,8 +360,6 @@ github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18 github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= diff --git a/internal/pkg/migrate/migrate.go b/internal/pkg/migrate/migrate.go deleted file mode 100644 index f18a4bde0..000000000 --- a/internal/pkg/migrate/migrate.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package migrate - -import ( - "context" - "encoding/json" - "errors" - - "github.com/elastic/fleet-server/v7/internal/pkg/bulk" - "github.com/elastic/fleet-server/v7/internal/pkg/dl" - "github.com/elastic/fleet-server/v7/internal/pkg/es" - "github.com/elastic/fleet-server/v7/internal/pkg/model" - "github.com/elastic/fleet-server/v7/internal/pkg/saved" - "github.com/rs/zerolog" -) - -type enrollmentApiKey struct { - Name string `json:"name"` - Type string `json:"type"` - ApiKey string `json:"api_key" saved:"encrypt"` - ApiKeyId string `json:"api_key_id"` - PolicyId string `json:"policy_id"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - ExpireAt string `json:"expire_at"` - Active bool `json:"active"` -} - -// Data migration -// This is for development only (1 instance of fleet) -// Not safe for multiple instances of fleet -// Initially needed to migrate the enrollment-api-keys that kibana creates -func Migrate(ctx context.Context, log zerolog.Logger, sv saved.CRUD, bulker bulk.Bulk) error { - return MigrateEnrollmentAPIKeys(ctx, log, sv, bulker) -} - -func MigrateEnrollmentAPIKeys(ctx context.Context, log zerolog.Logger, sv saved.CRUD, bulker bulk.Bulk) error { - - // Query all enrollment keys from the new schema - raw, err := dl.RenderAllEnrollmentAPIKeysQuery(1000) - if err != nil { - return err - } - - var recs []model.EnrollmentApiKey - var resHits []es.HitT - res, err := bulker.Search(ctx, []string{dl.FleetEnrollmentAPIKeys}, raw, bulk.WithRefresh()) - if err != nil { - if errors.Is(err, es.ErrIndexNotFound) { - log.Debug().Str("index", dl.FleetEnrollmentAPIKeys).Msg(es.ErrIndexNotFound.Error()) - // Continue with migration if the .fleet-enrollment-api-keys index is not found - err = nil - } else { - return err - } - } else { - resHits = res.Hits - } - - for _, hit := range resHits { - var rec model.EnrollmentApiKey - err := json.Unmarshal(hit.Source, &rec) - if err != nil { - return err - } - recs = append(recs, rec) - } - - // Query enrollment keys from kibana saved objects - query := saved.NewQuery("fleet-enrollment-api-keys") - - hits, err := sv.FindByNode(ctx, query) - if err != nil { - return err - } - - for _, hit := range hits { - var rec enrollmentApiKey - if err := sv.Decode(hit, &rec); err != nil { - return err - } - if _, ok := findExistingEnrollmentAPIKey(recs, rec); !ok { - newRec := translateEnrollmentAPIKey(rec) - b, err := json.Marshal(newRec) - if err != nil { - return err - } - _, err = bulker.Create(ctx, dl.FleetEnrollmentAPIKeys, "", b, bulk.WithRefresh()) - if err != nil { - return err - } - } - } - - return nil -} - -func findExistingEnrollmentAPIKey(hay []model.EnrollmentApiKey, needle enrollmentApiKey) (*model.EnrollmentApiKey, bool) { - for _, rec := range hay { - if rec.ApiKeyId == needle.ApiKeyId { - return &rec, true - } - } - return nil, false -} - -func translateEnrollmentAPIKey(src enrollmentApiKey) model.EnrollmentApiKey { - return model.EnrollmentApiKey{ - Active: src.Active, - ApiKey: src.ApiKey, - ApiKeyId: src.ApiKeyId, - CreatedAt: src.CreatedAt, - ExpireAt: src.ExpireAt, - Name: src.Name, - PolicyId: src.PolicyId, - UpdatedAt: src.UpdatedAt, - } -} diff --git a/internal/pkg/saved/crud.go b/internal/pkg/saved/crud.go deleted file mode 100644 index a3fd2aabf..000000000 --- a/internal/pkg/saved/crud.go +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "context" - "encoding/json" - "time" - - "github.com/elastic/fleet-server/v7/internal/pkg/bulk" - "github.com/elastic/fleet-server/v7/internal/pkg/dsl" - - "github.com/elastic/go-elasticsearch/v8" - "github.com/rs/zerolog/log" -) - -const ( - kIndexKibana = ".kibana*" - kMigrationVersion = "7.9.0" // TODO: bring in during build -) - -type Hit struct { - Id string - Type string - Space string - References []string - UpdatedAt string - Data json.RawMessage -} - -type UpdateT struct { - Id string - Type string - Fields map[string]interface{} -} - -type CRUD interface { - Create(ctx context.Context, ty string, src interface{}, opts ...Option) (id string, err error) - Read(ctx context.Context, ty, id string, dst interface{}, opts ...Option) error - - // AAD or Encrypted fields not supported; you will break your saved object; don't do that. - Update(ctx context.Context, ty, id string, fields map[string]interface{}, opts ...Option) error - MUpdate(ctx context.Context, updates []UpdateT, opts ...Option) error - - FindByField(ctx context.Context, ty string, fields map[string]interface{}) ([]Hit, error) - FindByNode(ctx context.Context, node *dsl.Node) ([]Hit, error) - FindRaw(ctx context.Context, json []byte) ([]Hit, error) - Decode(hit Hit, dst interface{}) error - - Client() *elasticsearch.Client -} - -type mgr struct { - idx bulk.Bulk - key string -} - -func NewMgr(idx bulk.Bulk, key string) CRUD { - return &mgr{idx, key} -} - -func (m *mgr) Client() *elasticsearch.Client { - return m.idx.Client() -} - -func (m *mgr) Create(ctx context.Context, ty string, src interface{}, options ...Option) (id string, err error) { - opts, err := processOpts(options...) - - if err != nil { - return - } - - if err = validateType(ty); err != nil { - return - } - - if id, err = genID(opts); err != nil { - return - } - - var data []byte - if data, err = m.encode(ty, id, opts.Space, src); err != nil { - return - } - - docID := fmtID(ty, id, opts.Space) - - nowStr := time.Now().UTC().Format(time.RFC3339) - - // TODO: hardcoded migration version - var objMap = map[string]interface{}{ - ty: json.RawMessage(data), - "type": ty, - "updated_at": nowStr, - "migrationVersion": map[string]string{ - "config": kMigrationVersion, - }, - "references": opts.References, - } - - if opts.Space != "" { - objMap["namespace"] = opts.Space - } - - var source []byte - if source, err = json.Marshal(objMap); err != nil { - return - } - - bulkOpts := m.makeBulkOpts(opts) - - if opts.Overwrite { - id, err = m.idx.Index(ctx, kIndexKibana, docID, source, bulkOpts...) - } else { - id, err = m.idx.Create(ctx, kIndexKibana, docID, source, bulkOpts...) - } - - log.Trace().Err(err).RawJSON("source", source).Msg("On create") - - return -} - -func (m *mgr) makeBulkOpts(opts optionsT) []bulk.Opt { - var bulkOpts []bulk.Opt - if opts.Refresh { - bulkOpts = append(bulkOpts, bulk.WithRefresh()) - } - return bulkOpts -} - -func (m *mgr) Read(ctx context.Context, ty, id string, dst interface{}, options ...Option) error { - opts, err := processOpts(options...) - if err != nil { - return err - } - - if err := validateType(ty); err != nil { - return err - } - - if err := validateId(id); err != nil { - return err - } - - docId := fmtID(ty, id, opts.Space) - - payload, err := m.idx.Read(ctx, kIndexKibana, docId, bulk.WithRefresh()) - if err != nil { - return err - } - - var tmap map[string]json.RawMessage - if err = json.Unmarshal(payload, &tmap); err != nil { - return err - } - - obj, ok := tmap[ty] - if !ok { - return ErrMalformedSavedObj - } - - return m.decode(ty, id, opts.Space, obj, dst) -} - -// Warning: If you pass encrypted or AAD fields, you broke something. Don't do that. -func (m *mgr) Update(ctx context.Context, ty, id string, fields map[string]interface{}, options ...Option) error { - opts, err := processOpts(options...) - if err != nil { - return err - } - - if err := validateType(ty); err != nil { - return err - } - - if err := validateId(id); err != nil { - return err - } - - docId := fmtID(ty, id, opts.Space) - - timeNow := time.Now().UTC().Format(time.RFC3339) - - source, err := json.Marshal(map[string]interface{}{ - "doc": map[string]interface{}{ - ty: fields, - "updated_at": timeNow, - }, - }) - - if err != nil { - return err - } - - bulkOpts := m.makeBulkOpts(opts) - - return m.idx.Update(ctx, kIndexKibana, docId, source, bulkOpts...) -} - -// Warning: If you pass encrypted or AAD fields, you broke something. Don't do that. -func (m *mgr) MUpdate(ctx context.Context, updates []UpdateT, options ...Option) error { - opts, err := processOpts(options...) - if err != nil { - return err - } - - timeNow := time.Now().UTC().Format(time.RFC3339) - - ops := make([]bulk.BulkOp, 0, len(updates)) - - for _, u := range updates { - - if err := validateType(u.Type); err != nil { - return err - } - - if err := validateId(u.Id); err != nil { - return err - } - - docId := fmtID(u.Type, u.Id, opts.Space) - - source, err := json.Marshal(map[string]interface{}{ - "doc": map[string]interface{}{ - u.Type: u.Fields, - "updated_at": timeNow, - }, - }) - - if err != nil { - return err - } - - ops = append(ops, bulk.BulkOp{ - Id: docId, - Body: source, - Index: kIndexKibana, - }) - } - - bulkOpts := m.makeBulkOpts(opts) - - return m.idx.MUpdate(ctx, ops, bulkOpts...) -} - -// Simple term query; does NOT support find on encrypted field. -func (m *mgr) FindByField(ctx context.Context, ty string, fields map[string]interface{}) ([]Hit, error) { - - query := NewQuery(ty) - mustNode := query.Query().Bool().Must() - for f, v := range fields { - mustNode.Term(ScopeField(ty, f), v, nil) - } - - return m.FindByNode(ctx, query) -} - -func (m *mgr) FindByNode(ctx context.Context, node *dsl.Node) ([]Hit, error) { - body, err := json.Marshal(node) - if err != nil { - return nil, err - } - - return m.FindRaw(ctx, body) -} - -func (m *mgr) FindRaw(ctx context.Context, body []byte) ([]Hit, error) { - - searcResult, err := m.idx.Search(ctx, []string{kIndexKibana}, body) - - if err != nil { - return nil, err - } - - var hits []Hit - - for _, h := range searcResult.Hits { - - o, err := parseId(h.Id) - if err != nil { - return nil, err - } - - // Decode the source, better way to do this? - var src map[string]json.RawMessage - if err := json.Unmarshal(h.Source, &src); err != nil { - return nil, err - } - - var t string - if err := json.Unmarshal(src["type"], &t); err != nil { - return nil, err - } - - var space string - if v, ok := src["namespace"]; ok { - if err := json.Unmarshal(v, &space); err != nil { - return nil, err - } - } - - if t != o.ty { - return nil, ErrTypeMismatch - } - - if space != o.ns { - return nil, ErrSpaceMismatch - } - - var refs []string - if err := json.Unmarshal(src["references"], &refs); err != nil { - return nil, err - } - - var updatedAt string - if err := json.Unmarshal(src["updated_at"], &updatedAt); err != nil { - return nil, err - } - - hits = append(hits, Hit{ - Id: o.id, - Type: t, - Space: space, - References: refs, - UpdatedAt: updatedAt, - Data: src[t], - }) - - } - - return hits, err -} - -func (m *mgr) Decode(hit Hit, dst interface{}) error { - return m.decode(hit.Type, hit.Id, hit.Space, hit.Data, dst) -} diff --git a/internal/pkg/saved/crypto.go b/internal/pkg/saved/crypto.go deleted file mode 100644 index 948db9ead..000000000 --- a/internal/pkg/saved/crypto.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/sha512" - "encoding/base64" - "encoding/json" - "golang.org/x/crypto/pbkdf2" -) - -const ( - tagLen = 16 - keyLengthInBytes = 32 - keyIterations = 10000 -) - -func encryptFields(key, aad []byte, fields Fields) error { - - for k, v := range fields { - ciphertext, err := encrypt(key, aad, v) - - if err != nil { - return err - } - fields[k] = ciphertext - } - - return nil -} - -func decryptFields(key, aad []byte, fields Fields) error { - - for k, v := range fields { - ciphertext, ok := v.(string) - if !ok { - return ErrBadCipherText - } - - v, err := decrypt(key, aad, ciphertext) - - if err != nil { - return err - } - fields[k] = v - } - - return nil -} - -// see: https://github.com/elastic/node-crypto/blob/master/src/crypto.ts#L119 -func encrypt(key, aad []byte, v interface{}) (string, error) { - - plaintext, err := json.Marshal(v) - if err != nil { - return "", err - } - - // Generate random data for iv and salt - nonce, err := newNonce() - if err != nil { - return "", err - } - - dk := deriveKey(key, nonce.salt()) - - block, err := aes.NewCipher(dk) - if err != nil { - return "", err - } - - aesgcm, err := cipher.NewGCMWithTagSize(block, tagLen) - if err != nil { - return "", err - } - - ciphertext := aesgcm.Seal(nil, nonce.iv(), plaintext, aad) - - // Expects binary buffer [salt, iv, tag, encrypted] - // goland slaps the tag on the back of the slice, so we have to reorg a bit - tagOffset := len(ciphertext) - tagLen - - buf := bytes.Buffer{} - buf.Grow(ivLen + saltLen + len(ciphertext)) - // Write salt:iv - buf.Write(nonce.both()) - // Write tag - buf.Write(ciphertext[tagOffset:]) - // Write cipher text - buf.Write(ciphertext[:tagOffset]) - - payload := base64.StdEncoding.EncodeToString(buf.Bytes()) - return payload, nil -} - -func decrypt(key, aad []byte, cipherText string) (interface{}, error) { - - ciphertext, err := base64.StdEncoding.DecodeString(cipherText) - if err != nil { - return nil, err - } - - // expects header [salt, iv, tag, encrypted] - if len(ciphertext) <= saltLen+ivLen+tagLen { - return nil, ErrBadCipherText - } - - tagOffset := saltLen + ivLen - dataOffset := tagOffset + tagLen - - salt := ciphertext[:saltLen] - iv := ciphertext[saltLen:tagOffset] - tag := ciphertext[tagOffset:dataOffset] - data := ciphertext[dataOffset:] - - dk := deriveKey(key, salt) - - block, err := aes.NewCipher(dk) - if err != nil { - return nil, err - } - - aesgcm, err := cipher.NewGCMWithTagSize(block, tagLen) - if err != nil { - return nil, err - } - - // aesgcm expects the tag to be after the ciphertext - buf := bytes.Buffer{} - buf.Grow(len(data) + len(tag)) - buf.Write(data) - buf.Write(tag) - - plaintext, err := aesgcm.Open(nil, iv, buf.Bytes(), aad) - if err != nil { - return nil, err - } - - // plaintext is raw JSON, decode - var v interface{} - err = json.Unmarshal(plaintext, &v) - return v, err -} - -func deriveKey(key, salt []byte) []byte { - - return pbkdf2.Key( - []byte(key), - salt, - keyIterations, - keyLengthInBytes, - sha512.New, - ) -} - -// Emulate Additional Authenticated Data (AAD) generation in Kibana -// Effectively stable_stringify([ {namespace}, type, id, attributesAAD]); -// -func deriveAAD(ty, id, space string, attrs map[string]interface{}) ([]byte, error) { - /* - if len(attrs) == 0 { - log.Debug().Str("type", ty).Str("id", id).Str("space", space).Msg("No AAD; that seems wrong.") - } - */ - - v := []interface{}{space, ty, id, attrs} - - if space == "" { - v = v[1:] - } - - // This MUST be stable; and 1x1 with what javascript stringify is doing. - // Milage may vary; we may have to implement this manually depending on types and formatting. - return json.Marshal(v) -} diff --git a/internal/pkg/saved/encode.go b/internal/pkg/saved/encode.go deleted file mode 100644 index 995a31f8c..000000000 --- a/internal/pkg/saved/encode.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "bytes" - "encoding/json" - "fmt" - "reflect" - "strings" - "unicode" -) - -const ( - TagSaved = "saved" - TagAad = "aad" - TagEncrypt = "encrypt" - TagJSON = "json" -) - -type tagOptions string - -// From golang JSON code -func parseTag(tag string) (string, tagOptions) { - if idx := strings.Index(tag, ","); idx != -1 { - return tag[:idx], tagOptions(tag[idx+1:]) - } - return tag, tagOptions("") -} - -// From golang JSON code -func (o tagOptions) Contains(optionName string) bool { - if len(o) == 0 { - return false - } - s := string(o) - for s != "" { - var next string - i := strings.Index(s, ",") - if i >= 0 { - s, next = s[:i], s[i+1:] - } - if s == optionName { - return true - } - s = next - } - return false -} - -// From golang JSON code -func isValidTag(s string) bool { - if s == "" { - return false - } - for _, c := range s { - switch { - case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): - // Backslash and quote chars are reserved, but - // otherwise any punctuation chars are allowed - // in a tag name. - case !unicode.IsLetter(c) && !unicode.IsDigit(c): - return false - } - } - return true -} - -// From golang JSON code -func isEmptyValue(v reflect.Value) bool { - switch v.Kind() { - case reflect.Array, reflect.Map, reflect.Slice, reflect.String: - return v.Len() == 0 - case reflect.Bool: - return !v.Bool() - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return v.Int() == 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return v.Uint() == 0 - case reflect.Float32, reflect.Float64: - return v.Float() == 0 - case reflect.Interface, reflect.Ptr: - return v.IsNil() - } - return false -} - -func deriveFieldKey(field reflect.StructField) (string, tagOptions) { - - // Use json tag if available, otherwise lowercase name - tag := field.Tag.Get(TagJSON) - key, opts := parseTag(tag) - - if !isValidTag(key) { - key = strings.ToLower(field.Name) - } - - var out bytes.Buffer - json.HTMLEscape(&out, []byte(key)) - - return out.String(), opts -} - -func gatherAAD(src interface{}) (Fields, Fields) { - t := reflect.TypeOf(src) - v := reflect.ValueOf(src) - - if t.Kind() == reflect.Ptr { - v = v.Elem() - t = reflect.TypeOf(v.Interface()) - } - - aad := make(Fields) - encrypt := make(Fields) - - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - - // Get the field tag value - tag := field.Tag.Get(TagSaved) - - switch tag { - case TagAad: - key, _ := deriveFieldKey(field) - aad[key] = v.Field(i).Interface() - case TagEncrypt: - key, _ := deriveFieldKey(field) - encrypt[key] = v.Field(i).Interface() - case "", "-": - default: - panic(fmt.Sprintf("Unknown tag %s:\"%s\"", TagSaved, tag)) - } - } - - return aad, encrypt -} - -func isEncrypted(src interface{}) bool { - t := reflect.TypeOf(src) - - if t.Kind() == reflect.Ptr { - v := reflect.ValueOf(src).Elem().Interface() - t = reflect.TypeOf(v) - } - - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - - // Get the field tag value - tag := field.Tag.Get(TagSaved) - - switch tag { - case TagEncrypt: - return true - case TagAad, "", "-": - default: - panic(fmt.Sprintf("Unknown tag %s:\"%s\"", TagSaved, tag)) - } - } - - return false -} - -func (m *mgr) encode(ty, id, space string, src interface{}) ([]byte, error) { - if !isEncrypted(src) { - return json.Marshal(src) - } - - // scan for aad - aadSet, encryptSet := gatherAAD(src) - - aad, err := deriveAAD(ty, id, space, aadSet) - if err != nil { - return nil, err - } - - if err := encryptFields([]byte(m.key), aad, encryptSet); err != nil { - return nil, err - } - - fields := NewFields(src) - - for k, v := range encryptSet { - fields[k] = v - } - - return json.Marshal(fields) -} - -func (m *mgr) decode(ty, id, space string, data []byte, dst interface{}) error { - - if err := json.Unmarshal(data, dst); err != nil { - return err - } - - if !isEncrypted(dst) { - return nil - } - - fields := NewFields(dst) - - // scan for aad, this will return empty values, but we need the keys - aadSet, encryptSet := gatherAAD(dst) - - // Fix up aadSet with actual values retrieved from JSON - for k, _ := range aadSet { - aadSet[k] = fields[k] - } - - aad, err := deriveAAD(ty, id, space, aadSet) - if err != nil { - return err - } - - // Fix up encryptSet with actual values retrieved from JSON - for k, _ := range encryptSet { - encryptSet[k] = fields[k] - } - - if err := decryptFields([]byte(m.key), aad, encryptSet); err != nil { - return err - } - - // Overlay encrypted values on fields - for k, v := range encryptSet { - fields[k] = v - } - - return fields.MapInterface(dst) -} diff --git a/internal/pkg/saved/errors.go b/internal/pkg/saved/errors.go deleted file mode 100644 index 2344abd4a..000000000 --- a/internal/pkg/saved/errors.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "errors" -) - -var ( - ErrNoType = errors.New("no type") - ErrRead = errors.New("read error") - ErrNoId = errors.New("no id") - ErrAttributeUnknown = errors.New("unknown attribute") - ErrAttributeType = errors.New("wrong attribute type") - ErrBadCipherText = errors.New("bad cipher text") - ErrNotEncrypted = errors.New("attribute not encrypted") - ErrMalformedSavedObj = errors.New("malformed saved object") - ErrMalformedIdentifier = errors.New("malformed saved object identifier") - ErrTypeMismatch = errors.New("type mismatch") - ErrSpaceMismatch = errors.New("namespace mismatch") -) diff --git a/internal/pkg/saved/fields.go b/internal/pkg/saved/fields.go deleted file mode 100644 index 2177aca3c..000000000 --- a/internal/pkg/saved/fields.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "github.com/mitchellh/mapstructure" - "reflect" -) - -type Fields map[string]interface{} - -func NewFields(src interface{}) Fields { - t := reflect.TypeOf(src) - v := reflect.ValueOf(src) - - if t.Kind() == reflect.Ptr { - v = v.Elem() - t = reflect.TypeOf(v.Interface()) - } - - nFields := v.NumField() - - m := make(Fields, nFields) - - for i := 0; i < nFields; i++ { - key, opts := deriveFieldKey(t.Field(i)) - - if key == "-" || (opts.Contains("omitempty") && isEmptyValue(v.Field(i))) { - continue - } - - m[key] = v.Field(i).Interface() - } - - return m -} - -func (f Fields) MapInterface(dst interface{}) error { - - config := &mapstructure.DecoderConfig{ - TagName: TagJSON, - Result: dst, - } - - decoder, err := mapstructure.NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(f) -} diff --git a/internal/pkg/saved/id.go b/internal/pkg/saved/id.go deleted file mode 100644 index 5cb78eaa2..000000000 --- a/internal/pkg/saved/id.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "fmt" - "strings" - - "github.com/gofrs/uuid" -) - -func genID(opts optionsT) (string, error) { - var id string - - if opts.Id != "" { - id = opts.Id - } else if u, err := uuid.NewV4(); err != nil { - return "", err - } else { - id = u.String() - } - - return id, nil -} - -func fmtID(ty, id, space string) string { - - if space != "" { - return fmt.Sprintf("%s:%s:%s", space, ty, id) - } - - return fmt.Sprintf("%s:%s", ty, id) -} - -type objectId struct { - id string - ns string - ty string -} - -// Deconstruct the ID. Expect namespace:type:id -func parseId(id string) (o objectId, err error) { - - tuple := strings.Split(id, ":") - - switch len(tuple) { - case 1: - o.id = tuple[0] - case 2: - o.ty = tuple[0] - o.id = tuple[1] - case 3: - o.ns = tuple[0] - o.ty = tuple[1] - o.id = tuple[2] - default: - err = ErrMalformedIdentifier - } - - return -} diff --git a/internal/pkg/saved/nonce.go b/internal/pkg/saved/nonce.go deleted file mode 100644 index b22124a3f..000000000 --- a/internal/pkg/saved/nonce.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "crypto/rand" -) - -const ( - ivLen = 12 - saltLen = 64 -) - -type nonceT struct { - buf []byte -} - -func newNonce() (nonceT, error) { - n := nonceT{ - buf: make([]byte, saltLen+ivLen), - } - - _, err := rand.Read(n.buf) - return n, err -} - -func (n nonceT) iv() []byte { - return n.buf[saltLen:] -} - -func (n nonceT) salt() []byte { - return n.buf[:saltLen] -} - -func (n nonceT) both() []byte { - return n.buf -} diff --git a/internal/pkg/saved/opts.go b/internal/pkg/saved/opts.go deleted file mode 100644 index 542c25662..000000000 --- a/internal/pkg/saved/opts.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -type optionsT struct { - Id string - Space string - Overwrite bool - Flush bool - Refresh bool - References []string -} - -func (c optionsT) Validate() error { - // TODO: validate Space - // TODO: validate Id - // TODO: validate References - return nil -} - -type Option func(*optionsT) - -func WithId(id string) Option { - return func(opt *optionsT) { - opt.Id = id - } -} - -func WithSpace(space string) Option { - return func(opt *optionsT) { - opt.Space = space - } -} - -func WithOverwrite() Option { - return func(opt *optionsT) { - opt.Overwrite = true - } -} - -func WithFlush() Option { - return func(opt *optionsT) { - opt.Flush = true - } -} - -func WithRefresh() Option { - return func(opt *optionsT) { - opt.Refresh = true - } -} - -func WithRefs(refs []string) Option { - return func(opt *optionsT) { - opt.References = refs - } -} - -func processOpts(options ...Option) (opts optionsT, err error) { - for _, optF := range options { - optF(&opts) - } - - err = opts.Validate() - return -} - -func validateType(ty string) error { - // TODO: check for invalidate runes - if ty == "" { - return ErrNoType - } - return nil -} - -func validateId(id string) error { - // TODO: check for invalidate runes - if id == "" { - return ErrNoId - } - return nil -} diff --git a/internal/pkg/saved/query.go b/internal/pkg/saved/query.go deleted file mode 100644 index 2b39a1094..000000000 --- a/internal/pkg/saved/query.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package saved - -import ( - "fmt" - - "github.com/elastic/fleet-server/v7/internal/pkg/dsl" -) - -func NewQuery(ty string) *dsl.Node { - - root := dsl.NewRoot() - - // Require the type - root.Query().Bool().Must().Term("type", ty, nil) - - return root -} - -func ScopeField(ty, field string) string { - return fmt.Sprintf("%s.%s", ty, field) -} - -type ScopeFuncT func(field string) string - -func ScopeFunc(ty string) ScopeFuncT { - prefix := fmt.Sprintf("%s.", ty) - return func(field string) string { - return prefix + field - } -} - -/* - -1) saved.SearchNode(ctx, dsl.Node) -2) saved.SearchRaw(ctx, []byte) -3) fix policy to support N looksup in parallel -4) multisearch? how return hits? -5) strip out comments... -6) templatize call to get agent id at beginning of program - - - - q.Field(scopedField, value, boost) - -type treeMap map[string]*QueryN -type QueryN struct { - leaf interface{} - tree treeMap - array []*QueryN -} - - -func (q *QueryN) MarshalJSON() ([]byte, error) { - - switch { - case q.leaf != nil: - return json.Marshal(q.leaf) - case q.tree != nil: - return json.Marshal(q.tree) - case q.array != nil: - return json.Marshal(q.array) - } - - return []byte("null"), nil -} - -func (q *QueryN) Query() *QueryN { - if node, ok := q.tree["query"]; ok { - return node - } - - if q.tree == nil { - q.tree = make(map[string]*QueryN) - } - - node := &QueryN{} - q.tree["query"] = node - return node -} - -func (q *QueryN) Bool() *QueryN { - if node, ok := q.tree["bool"]; ok { - return node - } - - if q.tree == nil { - q.tree = make(map[string]*QueryN) - } - - node := &QueryN{} - q.tree["bool"] = node - return node -} - -func (q *QueryN) Must() *QueryN { - if node, ok := q.tree["must"]; ok { - return node - } - - if q.tree == nil { - q.tree = make(map[string]*QueryN) - } - - node := &QueryN{ - array: make([]*QueryN, 0), - } - q.tree["must"] = node - return node -} - -func (q *QueryN) Term() *QueryN { - return q.makeChildNode("term") -} - -func (q *QueryN) makeChildNode(key string) *QueryN { - node := &QueryN{} - if q.array != nil { - tNode := QueryN{ - tree: map[string]*QueryN{key:node}, - } - q.array = append(q.array, &tNode) - - } else { - if q.tree == nil { - q.tree = make(map[string]*QueryN) - } - q.tree[key] = node - } - - return node -} - -func (q *QueryN) Field(field string, value interface{}, boost *float64) { - if q.tree == nil { - q.tree = make(map[string]*QueryN) - } - - var leaf interface{} - - switch boost { - case nil: - leaf = value - default: - leaf = &struct { - Value interface{} `json:"value"` - Boost *float64 `json:"boost,omitempty"` - } { - value, - boost, - } - } - - node := &QueryN{ - leaf: leaf, - } - - q.tree[field] = node -} - -func (q *QueryN) SavedField(ty, field string, value interface{}, boost *float64) { - scopedField := fmt.Sprintf("%s.%s", ty, field) - q.Field(scopedField, value, boost) -} - -type RangeOpt func(treeMap) - -func WithRangeGT(v interface{}) RangeOpt { - return func(tmap treeMap) { - tmap["gt"] = &QueryN{leaf:v} - } -} - -func (q *QueryN) Range(field string, opts ...RangeOpt) { - - fieldNode := &QueryN{ - tree: make(treeMap), - } - - for _, o := range opts { - o(fieldNode.tree) - } - - node := q.makeChildNode("range") - node.tree = map[string]*QueryN{ - field: fieldNode, - } -} - -func (q *QueryN) Size(sz uint64) { - if q.tree == nil { - q.tree = make(treeMap) - } - q.tree["size"] = &QueryN { - leaf: sz, - } -} - -func (q *QueryN) Sort() *QueryN { - n := q.makeChildNode("sort") - n.array = make([]*QueryN, 0) - return n -} - -type SortOrderT string - -const ( - SortAscend SortOrderT = "asc" - SortDescend = "desc" -) - -func (q *QueryN) SortOrder(field string, order SortOrderT) { - if q.array == nil { - panic("Parent should be sort node") - } - - defaultOrder := SortAscend - if field == "_score" { - defaultOrder = SortDescend - } - - if order == defaultOrder { - q.array = append(q.array, &QueryN{leaf:field}) - } else { - n := q.makeChildNode(field) - n.leaf = order - } -} - - -func (q *QueryN) SortOpt(field string, order SortOrder, opts ...SortOpt) { - // TODO -} -*/