From d2a704241e571681b7be15926103d6ba95069722 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 2 Aug 2021 08:59:27 -0400 Subject: [PATCH 1/3] Populate agent.id field in .fleet-agents index --- cmd/fleet/handleCheckin.go | 11 ++++++++--- cmd/fleet/handleEnroll.go | 10 +++++++--- cmd/fleet/userAgent.go | 12 ++++++------ cmd/fleet/userAgent_test.go | 2 +- internal/pkg/checkin/bulk.go | 12 ++++++++++-- internal/pkg/checkin/bulk_test.go | 14 ++++++++++++-- internal/pkg/dl/constants.go | 1 + 7 files changed, 45 insertions(+), 17 deletions(-) diff --git a/cmd/fleet/handleCheckin.go b/cmd/fleet/handleCheckin.go index ff45a3449..5d5d1e379 100644 --- a/cmd/fleet/handleCheckin.go +++ b/cmd/fleet/handleCheckin.go @@ -152,11 +152,16 @@ func (ct *CheckinT) _handleCheckin(zlog zerolog.Logger, w http.ResponseWriter, r return err } - err = validateUserAgent(r, ct.verCon) + ver, err := validateUserAgent(r, ct.verCon) if err != nil { return err } + var newVer string + if ver != agent.Agent.Version { + newVer = ver + } + // Metrics; serenity now. dfunc := cntCheckin.IncStart() defer dfunc() @@ -225,7 +230,7 @@ func (ct *CheckinT) _handleCheckin(zlog zerolog.Logger, w http.ResponseWriter, r defer longPoll.Stop() // Intial update on checkin, and any user fields that might have changed - ct.bc.CheckIn(agent.Id, req.Status, rawMeta, seqno) + ct.bc.CheckIn(agent.Id, req.Status, rawMeta, seqno, newVer) // Initial fetch for pending actions var ( @@ -262,7 +267,7 @@ func (ct *CheckinT) _handleCheckin(zlog zerolog.Logger, w http.ResponseWriter, r zlog.Trace().Msg("fire long poll") break LOOP case <-tick.C: - ct.bc.CheckIn(agent.Id, req.Status, nil, nil) + ct.bc.CheckIn(agent.Id, req.Status, nil, nil, newVer) } } } diff --git a/cmd/fleet/handleEnroll.go b/cmd/fleet/handleEnroll.go index b25dcf874..3b5538fa4 100644 --- a/cmd/fleet/handleEnroll.go +++ b/cmd/fleet/handleEnroll.go @@ -135,7 +135,7 @@ func (et *EnrollerT) handleEnroll(w http.ResponseWriter, r *http.Request) (*Enro return nil, err } - err = validateUserAgent(r, et.verCon) + ver, err := validateUserAgent(r, et.verCon) if err != nil { return nil, err } @@ -167,10 +167,10 @@ func (et *EnrollerT) handleEnroll(w http.ResponseWriter, r *http.Request) (*Enro cntEnroll.bodyIn.Add(readCounter.Count()) - return _enroll(r.Context(), et.bulker, et.cache, *req, *erec) + return _enroll(r.Context(), et.bulker, et.cache, *req, *erec, ver) } -func _enroll(ctx context.Context, bulker bulk.Bulk, c cache.Cache, req EnrollRequest, erec model.EnrollmentApiKey) (*EnrollResponse, error) { +func _enroll(ctx context.Context, bulker bulk.Bulk, c cache.Cache, req EnrollRequest, erec model.EnrollmentApiKey, ver string) (*EnrollResponse, error) { if req.SharedId != "" { // TODO: Support pre-existing install @@ -210,6 +210,10 @@ func _enroll(ctx context.Context, bulker bulk.Bulk, c cache.Cache, req EnrollReq LocalMetadata: localMeta, AccessApiKeyId: accessApiKey.Id, ActionSeqNo: []int64{sqn.UndefinedSeqNo}, + Agent: &model.AgentMetadata{ + Id: agentId, + Version: ver, + }, } err = createFleetAgent(ctx, bulker, agentId, agentData) diff --git a/cmd/fleet/userAgent.go b/cmd/fleet/userAgent.go index 1773be60f..17d46427c 100644 --- a/cmd/fleet/userAgent.go +++ b/cmd/fleet/userAgent.go @@ -57,22 +57,22 @@ func maximizePatch(ver *version.Version) string { // validateUserAgent validates that the User-Agent of the connecting Elastic Agent is valid and that the version is // supported for this Fleet Server. -func validateUserAgent(r *http.Request, verConst version.Constraints) error { +func validateUserAgent(r *http.Request, verConst version.Constraints) (string, error) { userAgent := r.Header.Get("User-Agent") if userAgent == "" { - return ErrInvalidUserAgent + return "", ErrInvalidUserAgent } userAgent = strings.ToLower(userAgent) if !strings.HasPrefix(userAgent, userAgentPrefix) { - return ErrInvalidUserAgent + return "", ErrInvalidUserAgent } verStr := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(userAgent, userAgentPrefix), "-snapshot")) ver, err := version.NewVersion(verStr) if err != nil { - return ErrInvalidUserAgent + return "", ErrInvalidUserAgent } if !verConst.Check(ver) { - return ErrUnsupportedVersion + return "", ErrUnsupportedVersion } - return nil + return strings.TrimPrefix(verStr, "v"), nil } diff --git a/cmd/fleet/userAgent_test.go b/cmd/fleet/userAgent_test.go index e9c8d9926..f6304e508 100644 --- a/cmd/fleet/userAgent_test.go +++ b/cmd/fleet/userAgent_test.go @@ -82,7 +82,7 @@ func TestValidateUserAgent(t *testing.T) { t.Run(tr.userAgent, func(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("User-Agent", tr.userAgent) - res := validateUserAgent(req, tr.verCon) + _, res := validateUserAgent(req, tr.verCon) if tr.err != res { t.Fatalf("err mismatch: %v != %v", tr.err, res) } diff --git a/internal/pkg/checkin/bulk.go b/internal/pkg/checkin/bulk.go index ec9bdc937..17a2c6b93 100644 --- a/internal/pkg/checkin/bulk.go +++ b/internal/pkg/checkin/bulk.go @@ -34,6 +34,7 @@ func WithFlushInterval(d time.Duration) Opt { type extraT struct { meta []byte seqNo sqn.SeqNo + ver string } // Minimize the size of this structure. @@ -94,16 +95,17 @@ func (bc *Bulk) timestamp() string { // WARNING: Bulk will take ownership of fields, // so do not use after passing in. -func (bc *Bulk) CheckIn(id string, status string, meta []byte, seqno sqn.SeqNo) error { +func (bc *Bulk) CheckIn(id string, status string, meta []byte, seqno sqn.SeqNo, newVer string) error { // Separate out the extra data to minimize // the memory footprint of the 90% case of just // updating the timestamp. var extra *extraT - if meta != nil || seqno.IsSet() { + if meta != nil || seqno.IsSet() || newVer != "" { extra = &extraT{ meta: meta, seqNo: seqno, + ver: newVer, } } @@ -192,6 +194,12 @@ func (bc *Bulk) flush(ctx context.Context) error { dl.FieldLastCheckinStatus: pendingData.status, // Set the pending status } + // If the agent version is not empty it needs to be updated + // Assuming the agent can by upgraded keeping the same id, but incrementing the version + if pendingData.extra.ver != "" { + fields[dl.FieldAgentVersion] = pendingData.extra.ver + } + // Update local metadata if provided if pendingData.extra.meta != nil { // Surprise: The json encodeer compacts this raw JSON during diff --git a/internal/pkg/checkin/bulk_test.go b/internal/pkg/checkin/bulk_test.go index 897450242..8a11a1ff3 100644 --- a/internal/pkg/checkin/bulk_test.go +++ b/internal/pkg/checkin/bulk_test.go @@ -43,12 +43,14 @@ func TestBulkSimple(t *testing.T) { bc := NewBulk(&mockBulk) + const ver = "8.0.0" cases := []struct { desc string id string status string meta []byte seqno sqn.SeqNo + ver string }{ { "Simple case", @@ -56,6 +58,7 @@ func TestBulkSimple(t *testing.T) { "online", nil, nil, + "", }, { "Singled field case", @@ -63,6 +66,7 @@ func TestBulkSimple(t *testing.T) { "online", []byte(`{"hey":"now"}`), nil, + "", }, { "Multi field case", @@ -70,6 +74,7 @@ func TestBulkSimple(t *testing.T) { "online", []byte(`{"hey":"now","brown":"cow"}`), nil, + ver, }, { "Multi field nested case", @@ -77,6 +82,7 @@ func TestBulkSimple(t *testing.T) { "online", []byte(`{"hey":"now","wee":{"little":"doggie"}}`), nil, + "", }, { "Simple case with seqNo", @@ -84,6 +90,7 @@ func TestBulkSimple(t *testing.T) { "online", nil, sqn.SeqNo{1, 2, 3, 4}, + ver, }, { "Field case with seqNo", @@ -91,6 +98,7 @@ func TestBulkSimple(t *testing.T) { "online", []byte(`{"uncle":"fester"}`), sqn.SeqNo{5, 6, 7, 8}, + ver, }, { "Unusual status", @@ -98,6 +106,7 @@ func TestBulkSimple(t *testing.T) { "unusual", nil, nil, + "", }, { "Empty status", @@ -105,13 +114,14 @@ func TestBulkSimple(t *testing.T) { "", nil, nil, + "", }, } for _, c := range cases { t.Run(c.desc, func(t *testing.T) { - if err := bc.CheckIn(c.id, c.status, c.meta, c.seqno); err != nil { + if err := bc.CheckIn(c.id, c.status, c.meta, c.seqno, c.ver); err != nil { t.Fatal(err) } @@ -205,7 +215,7 @@ func benchmarkBulk(n int, flush bool, b *testing.B) { for i := 0; i < b.N; i++ { for _, id := range ids { - err := bc.CheckIn(id, "", nil, nil) + err := bc.CheckIn(id, "", nil, nil, "") if err != nil { b.Fatal(err) } diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index 7a1ed13d0..725fad1aa 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -40,6 +40,7 @@ const ( FieldDefaultApiKeyId = "default_api_key_id" FieldPolicyOutputPermissionsHash = "policy_output_permissions_hash" FieldUnenrolledReason = "unenrolled_reason" + FieldAgentVersion = "agent.version" FieldActive = "active" FieldUpdatedAt = "updated_at" From a37078b9e8c9cef5b721ecba90d3a0daf4df898a Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 2 Aug 2021 09:49:04 -0400 Subject: [PATCH 2/3] Address feedback on the draft --- cmd/fleet/userAgent.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/fleet/userAgent.go b/cmd/fleet/userAgent.go index 17d46427c..2304fceab 100644 --- a/cmd/fleet/userAgent.go +++ b/cmd/fleet/userAgent.go @@ -66,7 +66,14 @@ func validateUserAgent(r *http.Request, verConst version.Constraints) (string, e if !strings.HasPrefix(userAgent, userAgentPrefix) { return "", ErrInvalidUserAgent } - verStr := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(userAgent, userAgentPrefix), "-snapshot")) + + // Trim "elastic agent " prefix + s := strings.TrimPrefix(userAgent, userAgentPrefix) + // Trim "-snapshot" suffix + s = strings.TrimSuffix(s, "-snapshot") + // Trim leading and traling spaces + verStr := strings.TrimSpace(s) + ver, err := version.NewVersion(verStr) if err != nil { return "", ErrInvalidUserAgent @@ -74,5 +81,5 @@ func validateUserAgent(r *http.Request, verConst version.Constraints) (string, e if !verConst.Check(ver) { return "", ErrUnsupportedVersion } - return strings.TrimPrefix(verStr, "v"), nil + return ver.String(), nil } From 789e691b066475b98808be37c78ff83c3fb4b8d3 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 2 Aug 2021 22:31:35 -0400 Subject: [PATCH 3/3] Put back the version spaces trimming that was lost in the master changes --- cmd/fleet/userAgent.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/fleet/userAgent.go b/cmd/fleet/userAgent.go index 7d5fbf669..76feb425f 100644 --- a/cmd/fleet/userAgent.go +++ b/cmd/fleet/userAgent.go @@ -73,7 +73,10 @@ func validateUserAgent(r *http.Request, verConst version.Constraints) (string, e // Split the version to accommodate versions with suffixes such as v8.0.0-snapshot v8.0.0-alpha1 verSep := strings.Split(s, "-") - ver, err := version.NewVersion(verSep[0]) + // Trim leading and traling spaces + verStr := strings.TrimSpace(verSep[0]) + + ver, err := version.NewVersion(verStr) if err != nil { return "", ErrInvalidUserAgent }