diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 3e7ef9383b8..9e38e093103 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -67,6 +67,16 @@ between, and it can be very specific or not very specific. > **_NOTE:_** This only filters down the tests based on the platform. It will not execute a tests on a platform unless > the test defines as supporting it. +#### Selecting specific group + +By default, the runner will run all test groups. Each group runs on a dedicated machine instance. When working on groups of tests it's better to limit to a specific +group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"` +environment variable. This variable can take multiple groups with a space between. + +- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group. +- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or +"upgrade-standalone" group. + #### Passing additional go test flags When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`. @@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools` package for helper utilities. +### Test group + +Every `define.Require` must define a `Group` that it belongs too. Each group is executed on a separate instance with all tests with in the same group executed +on the same instance. Placing similar tests in the same group allows those tests to run on its own instance +as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`. + +Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that +are defined the more instances will be provisioned to complete all tests. A balance between a small good set of +groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can +out weight the benefits of creating another group. + ### Test namespaces Every test has access to its own unique namespace (a string value). This namespace can diff --git a/magefile.go b/magefile.go index 6954b7135fe..c48545e6118 100644 --- a/magefile.go +++ b/magefile.go @@ -1773,6 +1773,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche _ = os.MkdirAll(diagDir, 0755) cfg := runner.Config{ +<<<<<<< HEAD AgentVersion: agentVersion, AgentStackVersion: agentStackVersion, BuildDir: agentBuildDir, @@ -1787,6 +1788,24 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche Timestamp: timestamp, TestFlags: goTestFlags, ExtraEnv: extraEnv, +======= + AgentVersion: agentVersion, + StackVersion: agentStackVersion, + BuildDir: agentBuildDir, + GOVersion: goVersion, + RepoDir: repoDir, + DiagnosticsDir: diagDir, + StateDir: ".integration-cache", + Platforms: testPlatforms(), + Groups: testGroups(), + Matrix: matrix, + SingleTest: singleTest, + VerboseMode: mg.Verbose(), + Timestamp: timestamp, + TestFlags: goTestFlags, + ExtraEnv: extraEnv, + BinaryName: binaryName, +>>>>>>> 8a8abd046a (Add ability to split integration tests into different groups (#3544)) } ogcCfg := ogc.Config{ ServiceTokenPath: serviceTokenPath, @@ -1871,6 +1890,20 @@ func testPlatforms() []string { return platforms } +func testGroups() []string { + groupsStr := os.Getenv("TEST_GROUPS") + if groupsStr == "" { + return nil + } + var groups []string + for _, g := range strings.Split(groupsStr, " ") { + if g != "" { + groups = append(groups, g) + } + } + return groups +} + // Pre-requisite: user must have the gcloud CLI installed func authGCP(ctx context.Context) error { // We only need the service account token to exist. diff --git a/pkg/testing/define/batch.go b/pkg/testing/define/batch.go index 4efa9f5ad4c..24c71fca4a3 100644 --- a/pkg/testing/define/batch.go +++ b/pkg/testing/define/batch.go @@ -41,15 +41,16 @@ var defaultOS = []OS{ // Batch is a grouping of tests that all have the same requirements. type Batch struct { + // Group must be set on each test to define which group the tests belongs. + // Tests that are in the same group are executed on the same runner. + Group string `json:"group"` + // OS defines the operating systems this test batch needs. OS OS `json:"os"` // Stack defines the stack required for this batch. Stack *Stack `json:"stack,omitempty"` - // Isolate defines that this batch is isolated to a single test. - Isolate bool `json:"isolate"` - // Tests define the set of packages and tests that do not require sudo // privileges to be performed. Tests []BatchPackageTests `json:"tests"` @@ -177,15 +178,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch } for _, o := range set { var batch Batch - batchIdx := -1 - if !req.Isolate { - batchIdx = findBatchIdx(batches, o, req.Stack) - } + batchIdx := findBatchIdx(batches, req.Group, o, req.Stack) if batchIdx == -1 { // new batch required batch = Batch{ + Group: req.Group, OS: o, - Isolate: req.Isolate, Tests: nil, SudoTests: nil, } @@ -241,10 +239,10 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack return tests } -func findBatchIdx(batches []Batch, os OS, stack *Stack) int { +func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int { for i, b := range batches { - if b.Isolate { - // never add to an isolate batch + if b.Group != group { + // must be in the same group continue } if b.OS.Type != os.Type || b.OS.Arch != os.Arch { diff --git a/pkg/testing/define/batch_test.go b/pkg/testing/define/batch_test.go index 0f9afbf3691..a7e265b5f0e 100644 --- a/pkg/testing/define/batch_test.go +++ b/pkg/testing/define/batch_test.go @@ -93,6 +93,7 @@ func TestBatch(t *testing.T) { } expected := []Batch{ { + Group: Default, OS: OS{ Type: Darwin, Arch: AMD64, @@ -101,6 +102,7 @@ func TestBatch(t *testing.T) { SudoTests: darwinSudoTests, }, { + Group: Default, OS: OS{ Type: Darwin, Arch: ARM64, @@ -109,6 +111,7 @@ func TestBatch(t *testing.T) { SudoTests: darwinSudoTests, }, { + Group: Default, OS: OS{ Type: Linux, Arch: AMD64, @@ -117,6 +120,7 @@ func TestBatch(t *testing.T) { SudoTests: linuxSudoTests, }, { + Group: Default, OS: OS{ Type: Linux, Arch: ARM64, @@ -152,6 +156,7 @@ func TestBatch(t *testing.T) { SudoTests: linuxSudoTests, }, { + Group: Default, OS: OS{ Type: Windows, Arch: AMD64, @@ -160,170 +165,47 @@ func TestBatch(t *testing.T) { SudoTests: windowsSudoTests, }, { + Group: "one", OS: OS{ - Type: Darwin, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Darwin, - Arch: ARM64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: ARM64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Windows, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestAnyIsolate", - }, - }, - }, - }, - }, - { - OS: OS{ - Type: Darwin, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ - { - Name: "TestDarwinIsolate", - }, - }, - }, + Type: Linux, + Arch: ARM64, + Version: "20.04", + Distro: "ubuntu", }, - }, - { - OS: OS{ - Type: Darwin, - Arch: ARM64, + Stack: &Stack{ + Version: "8.8.0", }, - Isolate: true, Tests: []BatchPackageTests{ { Name: pkgName, Tests: []BatchPackageTest{ { - Name: "TestDarwinIsolate", + Name: "TestGroup_One_One", + Stack: true, }, - }, - }, - }, - }, - { - OS: OS{ - Type: Linux, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ { - Name: "TestLinuxIsolate", + Name: "TestGroup_One_Two", + Stack: true, }, }, }, }, }, { + Group: "two", OS: OS{ Type: Linux, Arch: ARM64, }, - Isolate: true, Tests: []BatchPackageTests{ { Name: pkgName, Tests: []BatchPackageTest{ { - Name: "TestLinuxIsolate", + Name: "TestGroup_Two_One", }, - }, - }, - }, - }, - { - OS: OS{ - Type: Windows, - Arch: AMD64, - }, - Isolate: true, - Tests: []BatchPackageTests{ - { - Name: pkgName, - Tests: []BatchPackageTest{ { - Name: "TestWindowsIsolate", + Name: "TestGroup_Two_Two", }, }, }, @@ -344,6 +226,7 @@ var testLinuxLocalTests = []BatchPackageTest{ var testLinuxLocalBatch = []Batch{ { + Group: Default, OS: OS{ Type: "linux", Arch: "amd64", @@ -356,6 +239,7 @@ var testLinuxLocalBatch = []Batch{ }, }, { + Group: Default, OS: OS{ Type: "linux", Arch: "arm64", diff --git a/pkg/testing/define/requirements.go b/pkg/testing/define/requirements.go index 2cfc4a35cb4..c62f874009f 100644 --- a/pkg/testing/define/requirements.go +++ b/pkg/testing/define/requirements.go @@ -11,6 +11,11 @@ import ( "github.com/elastic/elastic-agent/pkg/component" ) +const ( + // Default constant can be used as the default group for tests. + Default = "default" +) + const ( // Darwin is macOS platform Darwin = component.Darwin @@ -82,6 +87,13 @@ type Stack struct { // Requirements defines the testing requirements for the test to run. type Requirements struct { + // Group must be set on each test to define which group the tests belongs to. + // Tests that are in the same group are executed on the same runner. + // + // Useful when tests take a long time to complete and sharding them across multiple + // hosts can improve the total amount of time to complete all the tests. + Group string `json:"group"` + // OS defines the operating systems this test can run on. In the case // multiple are provided the test is ran multiple times one time on each // combination. @@ -97,10 +109,6 @@ type Requirements struct { // when a full test run is performed. Local bool `json:"local"` - // Isolate defines that this test must be isolated to its own dedicated VM and the test - // cannot be shared with other tests. - Isolate bool `json:"isolate"` - // Sudo defines that this test must run under superuser permissions. On Mac and Linux the // test gets executed under sudo and on Windows it gets run under Administrator. Sudo bool `json:"sudo"` @@ -108,6 +116,9 @@ type Requirements struct { // Validate returns an error if not valid. func (r Requirements) Validate() error { + if r.Group == "" { + return errors.New("group is required") + } for i, o := range r.OS { if err := o.Validate(); err != nil { return fmt.Errorf("invalid os %d: %w", i, err) diff --git a/pkg/testing/define/testdata/sample_test.go b/pkg/testing/define/testdata/sample_test.go index 51f9e158de8..01095aceaac 100644 --- a/pkg/testing/define/testdata/sample_test.go +++ b/pkg/testing/define/testdata/sample_test.go @@ -14,24 +14,21 @@ import ( func TestAnyLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, Local: true, }) } func TestAnySudo(t *testing.T) { define.Require(t, define.Requirements{ - Sudo: true, - }) -} - -func TestAnyIsolate(t *testing.T) { - define.Require(t, define.Requirements{ - Isolate: true, + Group: define.Default, + Sudo: true, }) } func TestDarwinLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Darwin, @@ -43,6 +40,7 @@ func TestDarwinLocal(t *testing.T) { func TestDarwinSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Darwin, @@ -52,19 +50,9 @@ func TestDarwinSudo(t *testing.T) { }) } -func TestDarwinIsolate(t *testing.T) { - define.Require(t, define.Requirements{ - OS: []define.OS{ - { - Type: define.Darwin, - }, - }, - Isolate: true, - }) -} - func TestLinuxLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -76,6 +64,7 @@ func TestLinuxLocal(t *testing.T) { func TestLinuxSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -85,52 +74,61 @@ func TestLinuxSudo(t *testing.T) { }) } -func TestLinuxIsolate(t *testing.T) { +func TestWindowsLocal(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Linux, + Type: define.Windows, }, }, - Isolate: true, + Local: true, }) } -func TestWindowsLocal(t *testing.T) { +func TestWindowsSudo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Windows, }, }, - Local: true, + Sudo: true, }) } -func TestWindowsSudo(t *testing.T) { +func TestSpecificCombinationOne(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Windows, + Type: define.Linux, + Arch: define.ARM64, + Distro: "ubuntu", + Version: "20.04", }, }, - Sudo: true, }) } -func TestWindowsIsolate(t *testing.T) { +func TestSpecificCombinationTwo(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { - Type: define.Windows, + Type: define.Linux, + Arch: define.ARM64, + Distro: "ubuntu", + Version: "20.04", }, }, - Isolate: true, }) } -func TestSpecificCombinationOne(t *testing.T) { +func TestSpecificCombinationWithCloud(t *testing.T) { define.Require(t, define.Requirements{ + Group: define.Default, OS: []define.OS{ { Type: define.Linux, @@ -139,11 +137,15 @@ func TestSpecificCombinationOne(t *testing.T) { Version: "20.04", }, }, + Stack: &define.Stack{ + Version: "8.8.0", + }, }) } -func TestSpecificCombinationTwo(t *testing.T) { +func TestGroup_One_One(t *testing.T) { define.Require(t, define.Requirements{ + Group: "one", OS: []define.OS{ { Type: define.Linux, @@ -152,11 +154,15 @@ func TestSpecificCombinationTwo(t *testing.T) { Version: "20.04", }, }, + Stack: &define.Stack{ + Version: "8.8.0", + }, }) } -func TestSpecificCombinationWithCloud(t *testing.T) { +func TestGroup_One_Two(t *testing.T) { define.Require(t, define.Requirements{ + Group: "one", OS: []define.OS{ { Type: define.Linux, @@ -170,3 +176,27 @@ func TestSpecificCombinationWithCloud(t *testing.T) { }, }) } + +func TestGroup_Two_One(t *testing.T) { + define.Require(t, define.Requirements{ + Group: "two", + OS: []define.OS{ + { + Type: define.Linux, + Arch: define.ARM64, + }, + }, + }) +} + +func TestGroup_Two_Two(t *testing.T) { + define.Require(t, define.Requirements{ + Group: "two", + OS: []define.OS{ + { + Type: define.Linux, + Arch: define.ARM64, + }, + }, + }) +} diff --git a/pkg/testing/ogc/provisioner.go b/pkg/testing/ogc/provisioner.go index 74c92eb0b30..853a9a3502e 100644 --- a/pkg/testing/ogc/provisioner.go +++ b/pkg/testing/ogc/provisioner.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "os" - "path" "path/filepath" "strings" "time" @@ -290,16 +289,6 @@ func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout { } else { tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Type, strings.Replace(batch.OS.Version, ".", "-", -1)))) } - if batch.Batch.Isolate { - tags = append(tags, "isolate") - var test define.BatchPackageTests - if len(batch.Batch.SudoTests) > 0 { - test = batch.Batch.SudoTests[0] - } else if len(batch.Batch.Tests) > 0 { - test = batch.Batch.Tests[0] - } - tags = append(tags, fmt.Sprintf("%s-%s", path.Base(test.Name), strings.ToLower(test.Tests[0].Name))) - } los, _ := findOSLayout(batch.OS.OS) return Layout{ Name: batch.ID, diff --git a/pkg/testing/runner/config.go b/pkg/testing/runner/config.go index a65f8a1550e..17a3fa5fc43 100644 --- a/pkg/testing/runner/config.go +++ b/pkg/testing/runner/config.go @@ -27,6 +27,17 @@ type Config struct { // defined in this list. Platforms []string +<<<<<<< HEAD +======= + // BinaryName is the name of the binary package under test, i.e, elastic-agent, metricbeat, etc + // this is used to copy the .tar.gz to the remote host + BinaryName string + + // Groups filters the tests to only run tests that are part of + // the groups defined in this list. + Groups []string + +>>>>>>> 8a8abd046a (Add ability to split integration tests into different groups (#3544)) // Matrix enables matrix testing. This explodes each test to // run on all supported platforms the runner supports. Matrix bool diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index ed9d6d9966f..b25cf52cf22 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -158,11 +158,13 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches var osBatches []OSBatch for _, b := range batches { - lbs, err := createBatches(b, platforms, cfg.Matrix) + lbs, err := createBatches(b, platforms, cfg.Groups, cfg.Matrix) if err != nil { return nil, err } - osBatches = append(osBatches, lbs...) + if lbs != nil { + osBatches = append(osBatches, lbs...) + } } if cfg.SingleTest != "" { osBatches, err = filterSingleTest(osBatches, cfg.SingleTest) @@ -880,8 +882,20 @@ func findBatchByID(id string, batches []OSBatch) (OSBatch, bool) { return OSBatch{}, false } -func createBatches(batch define.Batch, platforms []define.OS, matrix bool) ([]OSBatch, error) { +func batchInGroups(batch define.Batch, groups []string) bool { + for _, g := range groups { + if batch.Group == g { + return true + } + } + return false +} + +func createBatches(batch define.Batch, platforms []define.OS, groups []string, matrix bool) ([]OSBatch, error) { var batches []OSBatch + if len(groups) > 0 && !batchInGroups(batch, groups) { + return nil, nil + } specifics, err := getSupported(batch.OS, platforms) if errors.Is(err, ErrOSNotSupported) { var s SupportedOS @@ -1003,6 +1017,7 @@ func createBatchID(batch OSBatch) string { id += "-" + batch.OS.Distro } id += "-" + strings.Replace(batch.OS.Version, ".", "", -1) +<<<<<<< HEAD if batch.Batch.Isolate { if len(batch.Batch.Tests) > 0 { // only ever has one test in an isolated batch @@ -1013,6 +1028,19 @@ func createBatchID(batch OSBatch) string { id += "-" + batch.Batch.SudoTests[0].Tests[0].Name } } +======= + id += "-" + strings.Replace(batch.Batch.Group, ".", "", -1) + + // The batchID needs to be at most 63 characters long otherwise + // OGC will fail to instantiate the VM. + maxIDLen := 63 + if len(id) > maxIDLen { + hash := fmt.Sprintf("%x", md5.Sum([]byte(id))) + hashLen := utf8.RuneCountInString(hash) + id = id[:maxIDLen-hashLen-1] + "-" + hash + } + +>>>>>>> 8a8abd046a (Add ability to split integration tests into different groups (#3544)) return strings.ToLower(id) } diff --git a/testing/integration/apm_propagation_test.go b/testing/integration/apm_propagation_test.go index 62aa87b0d83..c1c573773c8 100644 --- a/testing/integration/apm_propagation_test.go +++ b/testing/integration/apm_propagation_test.go @@ -53,6 +53,7 @@ agent.monitoring: func TestAPMConfig(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Default, Stack: &define.Stack{}, }) f, err := define.NewFixture(t, define.Version()) diff --git a/testing/integration/beats_serverless_test.go b/testing/integration/beats_serverless_test.go new file mode 100644 index 00000000000..730d1d5075e --- /dev/null +++ b/testing/integration/beats_serverless_test.go @@ -0,0 +1,629 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration + +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "text/template" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/elastic/elastic-agent-libs/mapstr" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools" + "github.com/elastic/elastic-agent/pkg/testing/tools/estools" +) + +type BeatRunner struct { + suite.Suite + requirementsInfo *define.Info + agentFixture *atesting.Fixture + + // connection info + ESHost string + user string + pass string + kibHost string + + testUuid string + testbeatName string + + skipCleanup bool +} + +func TestBeatsServerless(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: Default, + OS: []define.OS{ + {Type: define.Linux}, + }, + Stack: &define.Stack{}, + Local: false, + Sudo: true, + }) + + suite.Run(t, &BeatRunner{requirementsInfo: info}) +} + +func (runner *BeatRunner) SetupSuite() { + runner.skipCleanup = false + + runner.testbeatName = os.Getenv("TEST_BINARY_NAME") + if runner.testbeatName == "" { + runner.T().Fatalf("TEST_BINARY_NAME must be set") + } + if runner.testbeatName == "elastic-agent" { + runner.T().Skipf("tests must be run against a beat, not elastic-agent") + } + + if runner.testbeatName != "filebeat" && runner.testbeatName != "metricbeat" && runner.testbeatName != "auditbeat" && runner.testbeatName != "packetbeat" { + runner.T().Skip("test only supports metricbeat or filebeat") + } + runner.T().Logf("running serverless tests with %s", runner.testbeatName) + + agentFixture, err := define.NewFixtureWithBinary(runner.T(), define.Version(), runner.testbeatName, "/home/ubuntu", atesting.WithRunLength(time.Minute*3), atesting.WithAdditionalArgs([]string{"-E", "output.elasticsearch.allow_older_versions=true"})) + runner.agentFixture = agentFixture + require.NoError(runner.T(), err) + + // the require.* code will fail without these, so assume the values are non-nil + runner.ESHost = os.Getenv("ELASTICSEARCH_HOST") + runner.user = os.Getenv("ELASTICSEARCH_USERNAME") + runner.pass = os.Getenv("ELASTICSEARCH_PASSWORD") + runner.kibHost = os.Getenv("KIBANA_HOST") + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + beatOutConfig := ` +output.elasticsearch: + hosts: ["{{.es_host}}"] + api_key: "{{.key_user}}:{{.key_pass}}" +setup.kibana: + host: {{.kb_host}} +processors: + - add_fields: + target: host + fields: + test-id: {{.test_id}} +{{.beat_cfg}} +` + + mbCfg := ` +metricbeat.config.modules: + path: ${path.config}/modules.d/*.yml +` + + fbCfg := ` +filebeat.modules: + - module: system + syslog: + enabled: true + auth: + enabled: true +filebeat.config.modules: + - modules: system + syslog: + enabled: true + auth: + enabled: true +` + auditbeatCfg := ` +auditbeat.modules: + +- module: file_integrity + paths: + - /bin + - /usr/bin + - /sbin + - /usr/sbin + - /etc +` + + packetbeatCfg := ` +` + + tmpl, err := template.New("config").Parse(beatOutConfig) + require.NoError(runner.T(), err) + + apiResp, err := estools.CreateAPIKey(ctx, runner.requirementsInfo.ESClient, estools.APIKeyRequest{Name: "test-api-key", Expiration: "1d"}) + require.NoError(runner.T(), err) + + // beats likes to add standard ports to URLs that don't have them, and ESS will sometimes return a URL without a port, assuming :443 + // so try to fix that here + fixedKibanaHost := runner.kibHost + parsedKibana, err := url.Parse(runner.kibHost) + require.NoError(runner.T(), err) + if parsedKibana.Port() == "" { + fixedKibanaHost = fmt.Sprintf("%s:443", fixedKibanaHost) + } + + fixedESHost := runner.ESHost + parsedES, err := url.Parse(runner.ESHost) + require.NoError(runner.T(), err) + if parsedES.Port() == "" { + fixedESHost = fmt.Sprintf("%s:443", fixedESHost) + } + + runner.T().Logf("configuring beats with %s / %s", fixedESHost, fixedKibanaHost) + + testUuid, err := uuid.NewV4() + require.NoError(runner.T(), err) + runner.testUuid = testUuid.String() + + additionalCfg := mbCfg + if runner.testbeatName == "filebeat" { + additionalCfg = fbCfg + } else if runner.testbeatName == "auditbeat" { + additionalCfg = auditbeatCfg + } else if runner.testbeatName == "packetbeat" { + additionalCfg = packetbeatCfg + } + + tmpl_map := map[string]string{"es_host": fixedESHost, "key_user": apiResp.Id, "key_pass": apiResp.APIKey, "kb_host": fixedKibanaHost, "test_id": testUuid.String(), "beat_cfg": additionalCfg} + parsedCfg := bytes.Buffer{} + err = tmpl.Execute(&parsedCfg, tmpl_map) + require.NoError(runner.T(), err) + + err = runner.agentFixture.WriteFileToWorkDir(ctx, parsedCfg.String(), fmt.Sprintf("%s.yml", runner.testbeatName)) + require.NoError(runner.T(), err) +} + +// run the beat with default metricsets, ensure no errors in logs + data is ingested +func (runner *BeatRunner) TestRunAndCheckData() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*4) + defer cancel() + + // in case there's already a running template, delete it, forcing the beat to re-install + runner.CleanupTemplates(ctx) + + err := runner.agentFixture.RunBeat(ctx) + require.NoError(runner.T(), err) + + docs, err := estools.GetLatestDocumentMatchingQuery(ctx, runner.requirementsInfo.ESClient, map[string]interface{}{ + "match": map[string]interface{}{ + "host.test-id": runner.testUuid, + }, + }, fmt.Sprintf("*%s*", runner.testbeatName)) + require.NoError(runner.T(), err) + require.NotEmpty(runner.T(), docs.Hits.Hits) +} + +// tests the [beat] setup --dashboards command +func (runner *BeatRunner) TestSetupDashboards() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*3) //dashboards seem to take a while + defer cancel() + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", runner.agentFixture.WorkDir(), "setup", "--dashboards"}) + assert.NoError(runner.T(), err) + runner.T().Logf("got response from dashboard setup: %s", string(resp)) + require.True(runner.T(), strings.Contains(string(resp), "Loaded dashboards")) + + dashList, err := tools.GetDashboards(ctx, runner.requirementsInfo.KibanaClient) + require.NoError(runner.T(), err) + + // interesting hack in cases where we don't have a clean environment + // check to see if any of the dashboards were created recently + found := false + for _, dash := range dashList { + if time.Since(dash.UpdatedAt) < time.Minute*5 { + found = true + break + } + } + require.True(runner.T(), found, fmt.Sprintf("could not find dashboard newer than 5 minutes, out of %d dashboards", len(dashList))) + + runner.Run("export dashboards", runner.SubtestExportDashboards) + // cleanup + if !runner.skipCleanup { + for _, dash := range dashList { + err = tools.DeleteDashboard(ctx, runner.requirementsInfo.KibanaClient, dash.ID) + if err != nil { + runner.T().Logf("WARNING: could not delete dashboards after test: %s", err) + break + } + } + } +} + +// tests the [beat] export dashboard command +func (runner *BeatRunner) SubtestExportDashboards() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + outDir := runner.T().TempDir() + + dashlist, err := tools.GetDashboards(ctx, runner.requirementsInfo.KibanaClient) + require.NoError(runner.T(), err) + require.NotEmpty(runner.T(), dashlist) + + exportOut, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "export", + "dashboard", "--folder", outDir, "--id", dashlist[0].ID}) + + runner.T().Logf("got output: %s", exportOut) + assert.NoError(runner.T(), err) + + inFolder, err := os.ReadDir(filepath.Join(outDir, "/_meta/kibana/8/dashboard")) + require.NoError(runner.T(), err) + runner.T().Logf("got log contents: %#v", inFolder) + require.NotEmpty(runner.T(), inFolder) +} + +// NOTE for the below tests: the testing framework doesn't guarantee a new stack instance each time, +// which means we might be running against a stack where a previous test has already done setup. +// perhaps CI should run `mage integration:clean` first? + +// tests the [beat] setup --pipelines command +func (runner *BeatRunner) TestSetupPipelines() { + if runner.testbeatName != "filebeat" { + runner.T().Skip("pipelines only available on filebeat") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + defer func() { + // cleanup + if !runner.skipCleanup { + err := estools.DeletePipelines(ctx, runner.requirementsInfo.ESClient, "*filebeat*") + if err != nil { + runner.T().Logf("WARNING: could not clean up pipelines: %s", err) + } + } + + }() + + // need to actually enable something that has pipelines + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", runner.agentFixture.WorkDir(), + "setup", "--pipelines", "--modules", "apache", "-M", "apache.error.enabled=true", "-M", "apache.access.enabled=true"}) + assert.NoError(runner.T(), err) + + runner.T().Logf("got response from pipeline setup: %s", string(resp)) + + pipelines, err := estools.GetPipelines(ctx, runner.requirementsInfo.ESClient, "*filebeat*") + require.NoError(runner.T(), err) + require.NotEmpty(runner.T(), pipelines) + +} + +// test beat setup --index-management with ILM disabled +func (runner *BeatRunner) TestIndexManagementNoILM() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + defer func() { + runner.CleanupTemplates(ctx) + }() + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.ilm.enabled=false"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + assert.NoError(runner.T(), err) + // we should not print a warning if we've explicitly disabled ILM + assert.NotContains(runner.T(), string(resp), "not supported") + + tmpls, err := estools.GetIndexTemplatesForPattern(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("*%s*", runner.testbeatName)) + require.NoError(runner.T(), err) + for _, tmpl := range tmpls.IndexTemplates { + runner.T().Logf("got template: %s", tmpl.Name) + } + require.NotEmpty(runner.T(), tmpls.IndexTemplates) + + runner.Run("export templates", runner.SubtestExportTemplates) + runner.Run("export index patterns", runner.SubtestExportIndexPatterns) + +} + +// tests setup with all default settings +func (runner *BeatRunner) TestWithAllDefaults() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + defer func() { + runner.CleanupTemplates(ctx) + }() + + // pre-delete in case something else missed cleanup + runner.CleanupTemplates(ctx) + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + streams, err := estools.GetDataStreamsForPattern(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("%s*", runner.testbeatName)) + require.NoError(runner.T(), err) + + require.NotEmpty(runner.T(), streams.DataStreams) + +} + +// test the setup process with mismatching template and DSL names +func (runner *BeatRunner) TestCustomBadNames() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + defer func() { + runner.CleanupTemplates(ctx) + }() + + resp, err := runner.agentFixture.Exec(ctx, []string{"-e", "--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.dsl.enabled=true", "--E=setup.dsl.data_stream_pattern='custom-bad-name'", "--E=setup.template.name='custom-name'", "--E=setup.template.pattern='custom-name'"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + require.True(runner.T(), strings.Contains(string(resp), "Additional updates & overwrites to this config will not work.")) + +} + +func (runner *BeatRunner) TestOverwriteWithCustomName() { + //an updated policy that has a different value than the default of 7d + updatedPolicy := mapstr.M{ + "data_retention": "1d", + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + defer func() { + runner.CleanupTemplates(ctx) + }() + + lctemp := runner.T().TempDir() + raw, err := json.MarshalIndent(updatedPolicy, "", " ") + require.NoError(runner.T(), err) + + lifecyclePath := filepath.Join(lctemp, "dsl_policy.json") + + err = os.WriteFile(lifecyclePath, raw, 0o744) + require.NoError(runner.T(), err) + + runner.CleanupTemplates(ctx) + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.dsl.enabled=true", "--E=setup.dsl.data_stream_pattern='custom-name'", "--E=setup.template.name='custom-name'", "--E=setup.template.pattern='custom-name'"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + runner.CheckDSLPolicy(ctx, "*custom-name*", "7d") + + resp, err = runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.dsl.enabled=true", "--E=setup.dsl.overwrite=true", "--E=setup.dsl.data_stream_pattern='custom-name'", + "--E=setup.template.name='custom-name'", "--E=setup.template.pattern='custom-name'", fmt.Sprintf("--E=setup.dsl.policy_file=%s", lifecyclePath)}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + runner.CheckDSLPolicy(ctx, "*custom-name*", "1d") + +} + +// TestWithCustomLifecyclePolicy uploads a custom DSL policy +func (runner *BeatRunner) TestWithCustomLifecyclePolicy() { + //create a custom policy file + dslPolicy := mapstr.M{ + "data_retention": "1d", + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + defer func() { + runner.CleanupTemplates(ctx) + }() + + lctemp := runner.T().TempDir() + raw, err := json.MarshalIndent(dslPolicy, "", " ") + require.NoError(runner.T(), err) + + lifecyclePath := filepath.Join(lctemp, "dsl_policy.json") + + err = os.WriteFile(lifecyclePath, raw, 0o744) + require.NoError(runner.T(), err) + + runner.CleanupTemplates(ctx) + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.dsl.enabled=true", fmt.Sprintf("--E=setup.dsl.policy_file=%s", lifecyclePath)}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + runner.CheckDSLPolicy(ctx, fmt.Sprintf("%s*", runner.testbeatName), "1d") + +} + +// tests beat setup --index-management with ILM explicitly set +// On serverless, this should fail. +func (runner *BeatRunner) TestIndexManagementILMEnabledFailure() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + info, err := estools.GetPing(ctx, runner.requirementsInfo.ESClient) + require.NoError(runner.T(), err) + + if info.Version.BuildFlavor != "serverless" { + runner.T().Skip("must run on serverless") + } + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.ilm.enabled=true", "--E=setup.ilm.overwrite=true"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.Error(runner.T(), err) + assert.Contains(runner.T(), string(resp), "error creating") +} + +// tests setup with both ILM and DSL enabled, should fail +func (runner *BeatRunner) TestBothLifecyclesEnabled() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.ilm.enabled=true", "--E=setup.dsl.enabled=true"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.Error(runner.T(), err) +} + +// disable all lifecycle management, ensure it's actually disabled +func (runner *BeatRunner) TestAllLifecyclesDisabled() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + defer func() { + runner.CleanupTemplates(ctx) + }() + + runner.CleanupTemplates(ctx) + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "setup", + "--index-management", + "--E=setup.ilm.enabled=false", "--E=setup.dsl.enabled=false"}) + runner.T().Logf("got response from management setup: %s", string(resp)) + require.NoError(runner.T(), err) + + // make sure we have data streams, but there's no lifecycles + streams, err := estools.GetDataStreamsForPattern(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("*%s*", runner.testbeatName)) + require.NoError(runner.T(), err) + + require.NotEmpty(runner.T(), streams.DataStreams, "found no datastreams") + foundPolicy := false + for _, stream := range streams.DataStreams { + if stream.Lifecycle.DataRetention != "" { + foundPolicy = true + break + } + } + require.False(runner.T(), foundPolicy, "Found a lifecycle policy despite disabling lifecycles. Found: %#v", streams) +} + +// the export command doesn't actually make a network connection, +// so this won't fail +func (runner *BeatRunner) TestExport() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + info, err := estools.GetPing(ctx, runner.requirementsInfo.ESClient) + require.NoError(runner.T(), err) + + if info.Version.BuildFlavor != "serverless" { + runner.T().Skip("must run on serverless") + } + + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "export", "ilm-policy", "--E=setup.ilm.enabled=true"}) + runner.T().Logf("got response from export: %s", string(resp)) + assert.NoError(runner.T(), err) + // check to see if we got a valid output + policy := map[string]interface{}{} + err = json.Unmarshal(resp, &policy) + require.NoError(runner.T(), err) + + require.NotEmpty(runner.T(), policy["policy"]) +} + +// tests beat export with DSL +func (runner *BeatRunner) TestExportDSL() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "export", "ilm-policy", "--E=setup.dsl.enabled=true"}) + runner.T().Logf("got response from export: %s", string(resp)) + assert.NoError(runner.T(), err) + // check to see if we got a valid output + policy := map[string]interface{}{} + err = json.Unmarshal(resp, &policy) + require.NoError(runner.T(), err) + + require.NotEmpty(runner.T(), policy["data_retention"]) +} + +func (runner *BeatRunner) SubtestExportTemplates() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + outDir := runner.T().TempDir() + + _, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "export", + "template", "--dir", outDir}) + assert.NoError(runner.T(), err) + + inFolder, err := os.ReadDir(filepath.Join(outDir, "/template")) + require.NoError(runner.T(), err) + runner.T().Logf("got log contents: %#v", inFolder) + require.NotEmpty(runner.T(), inFolder) +} + +func (runner *BeatRunner) SubtestExportIndexPatterns() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + + rawPattern, err := runner.agentFixture.Exec(ctx, []string{"--path.home", + runner.agentFixture.WorkDir(), + "export", + "index-pattern"}) + assert.NoError(runner.T(), err) + + idxPattern := map[string]interface{}{} + + err = json.Unmarshal(rawPattern, &idxPattern) + require.NoError(runner.T(), err) + require.NotNil(runner.T(), idxPattern["attributes"]) +} + +// CheckDSLPolicy checks if we have a match for the given DSL policy given a template name and policy data_retention +func (runner *BeatRunner) CheckDSLPolicy(ctx context.Context, tmpl string, policy string) { + streams, err := estools.GetDataStreamsForPattern(ctx, runner.requirementsInfo.ESClient, tmpl) + require.NoError(runner.T(), err) + + foundCustom := false + for _, stream := range streams.DataStreams { + if stream.Lifecycle.DataRetention == policy { + foundCustom = true + break + } + } + + require.True(runner.T(), foundCustom, "did not find our lifecycle policy. Found: %#v", streams) +} + +// CleanupTemplates removes any existing index +func (runner *BeatRunner) CleanupTemplates(ctx context.Context) { + if !runner.skipCleanup { + _ = estools.DeleteIndexTemplatesDataStreams(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("%s*", runner.testbeatName)) + _ = estools.DeleteIndexTemplatesDataStreams(ctx, runner.requirementsInfo.ESClient, "*custom-name*") + } +} diff --git a/testing/integration/diagnostics_test.go b/testing/integration/diagnostics_test.go index 00d8d97aed0..c56b89c3e9a 100644 --- a/testing/integration/diagnostics_test.go +++ b/testing/integration/diagnostics_test.go @@ -88,6 +88,7 @@ type componentAndUnitNames struct { func TestDiagnosticsOptionalValues(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: false, }) @@ -113,6 +114,7 @@ func TestDiagnosticsOptionalValues(t *testing.T) { func TestDiagnosticsCommand(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: false, }) diff --git a/testing/integration/endpoint_security_test.go b/testing/integration/endpoint_security_test.go index f3943604c1d..836f4628b36 100644 --- a/testing/integration/endpoint_security_test.go +++ b/testing/integration/endpoint_security_test.go @@ -74,10 +74,10 @@ var protectionTests = []struct { // test automatically. func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -100,10 +100,10 @@ func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) { // but at this point endpoint is already uninstalled. func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -128,10 +128,10 @@ func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) { func TestInstallWithEndpointSecurityAndRemoveEndpointIntegration(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation OS: []define.OS{ {Type: define.Linux}, }, @@ -490,10 +490,10 @@ func installElasticDefendPackage(t *testing.T, info *define.Info, policyID strin // path other than default func TestEndpointSecurityNonDefaultBasePath(t *testing.T) { info := define.Require(t, define.Requirements{ - Stack: &define.Stack{}, - Local: false, // requires Agent installation - Isolate: false, - Sudo: true, // requires Agent installation + Group: Fleet, + Stack: &define.Stack{}, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation }) ctx, cn := context.WithCancel(context.Background()) diff --git a/testing/integration/fake_test.go b/testing/integration/fake_test.go index cb685e4cb4a..2a17931f359 100644 --- a/testing/integration/fake_test.go +++ b/testing/integration/fake_test.go @@ -44,6 +44,7 @@ inputs: func TestFakeComponent(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: true, }) diff --git a/testing/integration/fqdn_test.go b/testing/integration/fqdn_test.go index 9bb1b34de89..4478de0c76b 100644 --- a/testing/integration/fqdn_test.go +++ b/testing/integration/fqdn_test.go @@ -32,6 +32,7 @@ import ( func TestFQDN(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Default, // placed in default only because its skipped OS: []define.OS{ {Type: define.Linux}, }, diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go new file mode 100644 index 00000000000..64d8cd7cf02 --- /dev/null +++ b/testing/integration/groups_test.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration + +package integration + +import "github.com/elastic/elastic-agent/pkg/testing/define" + +const ( + // Default group. + Default = define.Default + + // Fleet group of tests. Used for testing Elastic Agent with Fleet. + Fleet = "fleet" + + // FleetAirgapped group of tests. Used for testing Elastic Agent with Fleet and airgapped. + FleetAirgapped = "fleet-airgapped" + + // Upgrade group of tests. Used for testing upgrades. + Upgrade = "upgrade" +) diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go index e2e9d0206cf..40df66b1d15 100644 --- a/testing/integration/install_test.go +++ b/testing/integration/install_test.go @@ -24,6 +24,7 @@ import ( func TestInstallWithoutBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install`. Sudo: true, @@ -71,6 +72,7 @@ func TestInstallWithoutBasePath(t *testing.T) { func TestInstallWithBasePath(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, // We require sudo for this test to run // `elastic-agent install`. Sudo: true, diff --git a/testing/integration/install_unprivileged_test.go b/testing/integration/install_unprivileged_test.go new file mode 100644 index 00000000000..a2700870df8 --- /dev/null +++ b/testing/integration/install_unprivileged_test.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && !windows + +package integration + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/install" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +func TestInstallUnprivilegedWithoutBasePath(t *testing.T) { + define.Require(t, define.Requirements{ + Group: Default, + // We require sudo for this test to run + // `elastic-agent install` (even though it will + // be installed as non-root). + Sudo: true, + + // It's not safe to run this test locally as it + // installs Elastic Agent. + Local: false, + + // Only supports Linux at the moment. + OS: []define.OS{ + { + Type: define.Linux, + }, + }, + }) + + // Get path to Elastic Agent executable + fixture, err := define.NewFixture(t, define.Version()) + require.NoError(t, err) + + // Prepare the Elastic Agent so the binary is extracted and ready to use. + err = fixture.Prepare(context.Background()) + require.NoError(t, err) + + // Check that default base path is clean + var defaultBasePath string + switch runtime.GOOS { + case "darwin": + defaultBasePath = `/Library` + case "linux": + defaultBasePath = `/opt` + case "windows": + defaultBasePath = `C:\Program Files` + } + + topPath := filepath.Join(defaultBasePath, "Elastic", "Agent") + err = os.RemoveAll(topPath) + require.NoError(t, err, "failed to remove %q. The test requires this path not to exist.") + + // Run `elastic-agent install`. We use `--force` to prevent interactive + // execution. + out, err := fixture.Install(context.Background(), &atesting.InstallOpts{Force: true, Unprivileged: true}) + if err != nil { + t.Logf("install output: %s", out) + require.NoError(t, err) + } + + checkInstallUnprivilegedSuccess(t, topPath) +} + +func TestInstallUnprivilegedWithBasePath(t *testing.T) { + define.Require(t, define.Requirements{ + Group: Default, + // We require sudo for this test to run + // `elastic-agent install` (even though it will + // be installed as non-root). + Sudo: true, + + // It's not safe to run this test locally as it + // installs Elastic Agent. + Local: false, + + // Only supports Linux at the moment. + OS: []define.OS{ + { + Type: define.Linux, + }, + }, + }) + + // Get path to Elastic Agent executable + fixture, err := define.NewFixture(t, define.Version()) + require.NoError(t, err) + + // Prepare the Elastic Agent so the binary is extracted and ready to use. + err = fixture.Prepare(context.Background()) + require.NoError(t, err) + + // Other test `TestInstallWithBasePath` uses a random directory for the base + // path and that works because its running root. When using a base path the + // base needs to be accessible by the `elastic-agent` user that will be + // executing the process, but is not created yet. Using a base that exists + // and is known to be accessible by standard users, ensures this tests + // works correctly and will not hit a permission issue when spawning the + // elastic-agent service. + var basePath string + switch runtime.GOOS { + case define.Linux: + // default is `/opt` + basePath = `/usr` + default: + t.Fatalf("only Linux is supported by this test; should have been skipped") + } + + // Run `elastic-agent install`. We use `--force` to prevent interactive + // execution. + out, err := fixture.Install(context.Background(), &atesting.InstallOpts{ + BasePath: basePath, + Force: true, + Unprivileged: true, + }) + if err != nil { + t.Logf("install output: %s", out) + require.NoError(t, err) + } + + // Check that Agent was installed in the custom base path + topPath := filepath.Join(basePath, "Elastic", "Agent") + checkInstallUnprivilegedSuccess(t, topPath) +} + +func checkInstallUnprivilegedSuccess(t *testing.T, topPath string) { + t.Helper() + + // Check that the elastic-agent user/group exist. + uid, err := install.FindUID("elastic-agent") + require.NoError(t, err) + gid, err := install.FindGID("elastic-agent") + require.NoError(t, err) + + // Path should now exist as well as be owned by the correct user/group. + info, err := os.Stat(topPath) + require.NoError(t, err) + fs, ok := info.Sys().(*syscall.Stat_t) + require.True(t, ok) + require.Equalf(t, fs.Uid, uint32(uid), "%s not owned by elastic-agent user", topPath) + require.Equalf(t, fs.Gid, uint32(gid), "%s not owned by elastic-agent group", topPath) + + // Check that a few expected installed files are present + installedBinPath := filepath.Join(topPath, exeOnWindows("elastic-agent")) + installedDataPath := filepath.Join(topPath, "data") + installMarkerPath := filepath.Join(topPath, ".installed") + _, err = os.Stat(installedBinPath) + require.NoError(t, err) + _, err = os.Stat(installedDataPath) + require.NoError(t, err) + _, err = os.Stat(installMarkerPath) + require.NoError(t, err) + + // Check that the socket is created with the correct permissions. + socketPath := strings.TrimPrefix(paths.ControlSocketUnprivilegedPath, "unix://") + require.Eventuallyf(t, func() bool { + _, err = os.Stat(socketPath) + return err == nil + }, 3*time.Minute, 1*time.Second, "%s socket never created: %s", socketPath, err) + info, err = os.Stat(socketPath) + require.NoError(t, err) + fs, ok = info.Sys().(*syscall.Stat_t) + require.True(t, ok) + require.Equalf(t, fs.Uid, uint32(uid), "%s not owned by elastic-agent user", socketPath) + require.Equalf(t, fs.Gid, uint32(gid), "%s not owned by elastic-agent group", socketPath) + + // Executing `elastic-agent status` as the `elastic-agent` user should work. + var output []byte + require.Eventuallyf(t, func() bool { + cmd := exec.Command("sudo", "-u", "elastic-agent", "elastic-agent", "status") + output, err = cmd.CombinedOutput() + return err == nil + }, 3*time.Minute, 1*time.Second, "status never successful: %s (output: %s)", err, output) + + // Executing `elastic-agent status` as the original user should fail, because that + // user is not in the 'elastic-agent' group. + originalUser := os.Getenv("USER") + if originalUser != "" { + cmd := exec.Command("sudo", "-u", originalUser, "elastic-agent", "status") + output, err := cmd.CombinedOutput() + require.Error(t, err, "running elastic-agent status should have failed: %s", output) + } +} diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go new file mode 100644 index 00000000000..ff5c19f0405 --- /dev/null +++ b/testing/integration/logs_ingestion_test.go @@ -0,0 +1,463 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration + +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/http/httputil" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent/pkg/control/v2/client" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools" + "github.com/elastic/elastic-agent/pkg/testing/tools/check" + "github.com/elastic/elastic-agent/pkg/testing/tools/estools" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" + "github.com/elastic/elastic-transport-go/v8/elastictransport" +) + +func TestLogIngestionFleetManaged(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: Fleet, + Stack: &define.Stack{}, + Local: false, + Sudo: true, + }) + ctx := context.Background() + + agentFixture, err := define.NewFixture(t, define.Version()) + require.NoError(t, err) + + // 1. Create a policy in Fleet with monitoring enabled. + // To ensure there are no conflicts with previous test runs against + // the same ESS stack, we add the current time at the end of the policy + // name. This policy does not contain any integration. + t.Log("Enrolling agent in Fleet with a test policy") + createPolicyReq := kibana.AgentPolicy{ + Name: fmt.Sprintf("test-policy-enroll-%d", time.Now().Unix()), + Namespace: info.Namespace, + Description: "test policy for agent enrollment", + MonitoringEnabled: []kibana.MonitoringEnabledOption{ + kibana.MonitoringEnabledLogs, + kibana.MonitoringEnabledMetrics, + }, + AgentFeatures: []map[string]interface{}{ + { + "name": "test_enroll", + "enabled": true, + }, + }, + } + + installOpts := atesting.InstallOpts{ + NonInteractive: true, + Force: true, + } + + // 2. Install the Elastic-Agent with the policy that + // was just created. + policy, err := tools.InstallAgentWithPolicy( + ctx, + t, + installOpts, + agentFixture, + info.KibanaClient, + createPolicyReq) + require.NoError(t, err) + t.Logf("created policy: %s", policy.ID) + check.ConnectedToFleet(t, agentFixture, 5*time.Minute) + + t.Run("Monitoring logs are shipped", func(t *testing.T) { + testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy) + }) + + t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) { + testFlattenedDatastreamFleetPolicy(t, ctx, info, agentFixture, policy) + }) +} + +func testMonitoringLogsAreShipped( + t *testing.T, + ctx context.Context, + info *define.Info, + agentFixture *atesting.Fixture, + policy kibana.PolicyResponse, +) { + // Stage 1: Make sure metricbeat logs are populated + t.Log("Making sure metricbeat logs are populated") + docs := findESDocs(t, func() (estools.Documents, error) { + return estools.GetLogsForDataset(info.ESClient, "elastic_agent.metricbeat") + }) + t.Logf("metricbeat: Got %d documents", len(docs.Hits.Hits)) + require.NotZero(t, len(docs.Hits.Hits), + "Looking for logs in dataset 'elastic_agent.metricbeat'") + + // Stage 2: make sure all components are healthy + t.Log("Making sure all components are healthy") + status, err := agentFixture.ExecStatus(ctx) + require.NoError(t, err, + "could not get agent status to verify all components are healthy") + for _, c := range status.Components { + assert.Equalf(t, client.Healthy, client.State(c.State), + "component %s: want %s, got %s", + c.Name, client.Healthy, client.State(c.State)) + } + + // Stage 3: Make sure there are no errors in logs + t.Log("Making sure there are no error logs") + docs = queryESDocs(t, func() (estools.Documents, error) { + return estools.CheckForErrorsInLogs(info.ESClient, info.Namespace, []string{ + // acceptable error messages (include reason) + "Error dialing dial tcp 127.0.0.1:9200: connect: connection refused", // beat is running default config before its config gets updated + "Global configuration artifact is not available", // Endpoint: failed to load user artifact due to connectivity issues + "Failed to download artifact", + "Failed to initialize artifact", + "Failed to apply initial policy from on disk configuration", + "elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-id\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the openstack metadata to not work + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/hostname\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/placement/availability-zone\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work + }) + }) + t.Logf("error logs: Got %d documents", len(docs.Hits.Hits)) + for _, doc := range docs.Hits.Hits { + t.Logf("%#v", doc.Source) + } + require.Empty(t, docs.Hits.Hits) + + // Stage 4: Make sure we have message confirming central management is running + t.Log("Making sure we have message confirming central management is running") + docs = findESDocs(t, func() (estools.Documents, error) { + return estools.FindMatchingLogLines(info.ESClient, info.Namespace, + "Parsed configuration and determined agent is managed by Fleet") + }) + require.NotZero(t, len(docs.Hits.Hits)) + + // Stage 5: verify logs from the monitoring components are not sent to the output + t.Log("Check monitoring logs") + hostname, err := os.Hostname() + if err != nil { + t.Fatalf("could not get hostname to filter Agent: %s", err) + } + + agentID, err := fleettools.GetAgentIDByHostname(info.KibanaClient, policy.ID, hostname) + require.NoError(t, err, "could not get Agent ID by hostname") + t.Logf("Agent ID: %q", agentID) + + // We cannot search for `component.id` because at the moment of writing + // this field is not mapped. There is an issue for that: + // https://github.com/elastic/integrations/issues/6545 + // TODO: use runtime fields while the above issue is not resolved. + docs = findESDocs(t, func() (estools.Documents, error) { + return estools.GetLogsForAgentID(info.ESClient, agentID) + }) + require.NoError(t, err, "could not get logs from Agent ID: %q, err: %s", + agentID, err) + + monRegExp := regexp.MustCompile(".*-monitoring$") + for i, d := range docs.Hits.Hits { + // Lazy way to navigate a map[string]any: convert to JSON then + // decode into a struct. + jsonData, err := json.Marshal(d.Source) + if err != nil { + t.Fatalf("could not encode document source as JSON: %s", err) + } + + doc := ESDocument{} + if err := json.Unmarshal(jsonData, &doc); err != nil { + t.Fatalf("could not unmarshal document source: %s", err) + } + + if monRegExp.MatchString(doc.Component.ID) { + t.Errorf("[%d] Document on index %q with 'component.id': %q "+ + "and 'elastic_agent.id': %q. 'elastic_agent.id' must not "+ + "end in '-monitoring'\n", + i, d.Index, doc.Component.ID, doc.ElasticAgent.ID) + } + } +} + +// queryESDocs runs `findFn` until it returns no error. Zero documents returned +// is considered a success. +func queryESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents { + var docs estools.Documents + require.Eventually( + t, + func() bool { + var err error + docs, err = findFn() + if err != nil { + t.Logf("got an error querying ES, retrying. Error: %s", err) + } + return err == nil + }, + 3*time.Minute, + 15*time.Second, + ) + + return docs +} + +// findESDocs runs `findFn` until at least one document is returned and there is no error +func findESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents { + var docs estools.Documents + require.Eventually( + t, + func() bool { + var err error + docs, err = findFn() + if err != nil { + t.Logf("got an error querying ES, retrying. Error: %s", err) + return false + } + + return docs.Hits.Total.Value != 0 + }, + 3*time.Minute, + 15*time.Second, + ) + + return docs +} + +func testFlattenedDatastreamFleetPolicy( + t *testing.T, + ctx context.Context, + info *define.Info, + agentFixture *atesting.Fixture, + policy kibana.PolicyResponse, +) { + dsType := "logs" + dsNamespace := cleanString(fmt.Sprintf("%snamespace%d", t.Name(), rand.Uint64())) + dsDataset := cleanString(fmt.Sprintf("%s-dataset", t.Name())) + numEvents := 60 + + tempDir := t.TempDir() + logFilePath := filepath.Join(tempDir, "log.log") + generateLogFile(t, logFilePath, 2*time.Millisecond, numEvents) + + agentFixture, err := define.NewFixture(t, define.Version()) + if err != nil { + t.Fatalf("could not create new fixture: %s", err) + } + + // 1. Prepare a request to add an integration to the policy + tmpl, err := template.New(t.Name() + "custom-log-policy").Parse(policyJSON) + if err != nil { + t.Fatalf("cannot parse template: %s", err) + } + + // The time here ensures there are no conflicts with the integration name + // in Fleet. + agentPolicyBuilder := strings.Builder{} + err = tmpl.Execute(&agentPolicyBuilder, policyVars{ + Name: "Log-Input-" + t.Name() + "-" + time.Now().Format(time.RFC3339), + PolicyID: policy.ID, + LogFilePath: logFilePath, + Namespace: dsNamespace, + Dataset: dsDataset, + }) + if err != nil { + t.Fatalf("could not render template: %s", err) + } + // We keep a copy of the policy for debugging prurposes + agentPolicy := agentPolicyBuilder.String() + + // 2. Call Kibana to create the policy. + // Docs: https://www.elastic.co/guide/en/fleet/current/fleet-api-docs.html#create-integration-policy-api + resp, err := info.KibanaClient.Connection.Send( + http.MethodPost, + "/api/fleet/package_policies", + nil, + nil, + bytes.NewBufferString(agentPolicy)) + if err != nil { + t.Fatalf("could not execute request to Kibana/Fleet: %s", err) + } + if resp.StatusCode != http.StatusOK { + // On error dump the whole request response so we can easily spot + // what went wrong. + t.Errorf("received a non 200-OK when adding package to policy. "+ + "Status code: %d", resp.StatusCode) + respDump, err := httputil.DumpResponse(resp, true) + if err != nil { + t.Fatalf("could not dump error response from Kibana: %s", err) + } + // Make debugging as easy as possible + t.Log("================================================================================") + t.Log("Kibana error response:") + t.Log(string(respDump)) + t.Log("================================================================================") + t.Log("Rendered policy:") + t.Log(agentPolicy) + t.Log("================================================================================") + t.FailNow() + } + + require.Eventually( + t, + ensureDocumentsInES(t, ctx, info.ESClient, dsType, dsDataset, dsNamespace, numEvents), + 120*time.Second, + time.Second, + "could not get all expected documents form ES") +} + +// ensureDocumentsInES asserts the documents were ingested into the correct +// datastream +func ensureDocumentsInES( + t *testing.T, + ctx context.Context, + esClient elastictransport.Interface, + dsType, dsDataset, dsNamespace string, + numEvents int, +) func() bool { + + f := func() bool { + t.Helper() + + docs, err := estools.GetLogsForDatastream(ctx, esClient, dsType, dsDataset, dsNamespace) + if err != nil { + t.Logf("error quering ES, will retry later: %s", err) + } + + if docs.Hits.Total.Value == numEvents { + return true + } + + return false + + } + + return f +} + +// generateLogFile generates a log file by appending new lines every tick +// the lines are composed by the test name and the current time in RFC3339Nano +// This function spans a new goroutine and does not block +func generateLogFile(t *testing.T, fullPath string, tick time.Duration, events int) { + t.Helper() + f, err := os.Create(fullPath) + if err != nil { + t.Fatalf("could not create file '%s: %s", fullPath, err) + } + + go func() { + t.Helper() + ticker := time.NewTicker(tick) + t.Cleanup(ticker.Stop) + + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + + defer func() { + if err := f.Close(); err != nil { + t.Errorf("could not close log file '%s': %s", fullPath, err) + } + }() + + i := 0 + for { + select { + case <-done: + return + case now := <-ticker.C: + i++ + _, err := fmt.Fprintln(f, t.Name(), "Iteration: ", i, now.Format(time.RFC3339Nano)) + if err != nil { + // The Go compiler does not allow me to call t.Fatalf from a non-test + // goroutine, t.Errorf is our only option + t.Errorf("could not write data to log file '%s': %s", fullPath, err) + return + } + // make sure log lines are synced as quickly as possible + if err := f.Sync(); err != nil { + t.Errorf("could not sync file '%s': %s", fullPath, err) + } + if i == events { + return + } + } + } + }() +} + +func cleanString(s string) string { + return nonAlphanumericRegex.ReplaceAllString(strings.ToLower(s), "") +} + +var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9 ]+`) + +var policyJSON = ` +{ + "policy_id": "{{.PolicyID}}", + "package": { + "name": "log", + "version": "2.3.0" + }, + "name": "{{.Name}}", + "namespace": "{{.Namespace}}", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "{{.LogFilePath | js}}" {{/* we need to escape windows paths */}} + ], + "data_stream.dataset": "{{.Dataset}}" + } + } + } + } + } +}` + +type policyVars struct { + Name string + PolicyID string + LogFilePath string + Namespace string + Dataset string +} + +type ESDocument struct { + ElasticAgent ElasticAgent `json:"elastic_agent"` + Component Component `json:"component"` + Host Host `json:"host"` +} +type ElasticAgent struct { + ID string `json:"id"` + Version string `json:"version"` + Snapshot bool `json:"snapshot"` +} +type Component struct { + Binary string `json:"binary"` + ID string `json:"id"` +} +type Host struct { + Hostname string `json:"hostname"` +} diff --git a/testing/integration/package_version_test.go b/testing/integration/package_version_test.go index 2b8ca6ebe90..b2d560526f5 100644 --- a/testing/integration/package_version_test.go +++ b/testing/integration/package_version_test.go @@ -22,6 +22,7 @@ import ( func TestPackageVersion(t *testing.T) { define.Require(t, define.Requirements{ + Group: Default, Local: true, }) diff --git a/testing/integration/proxy_url_test.go b/testing/integration/proxy_url_test.go index e95469405be..c9ae99a248a 100644 --- a/testing/integration/proxy_url_test.go +++ b/testing/integration/proxy_url_test.go @@ -91,6 +91,7 @@ func TearDownTest(t *testing.T, p *ProxyURL) { func TestProxyURL_EnrollProxyAndNoProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -135,6 +136,7 @@ func TestProxyURL_EnrollProxyAndNoProxyInThePolicy(t *testing.T) { func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -180,6 +182,7 @@ func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) { func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -239,6 +242,7 @@ func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) { func TestProxyURL_NoEnrollProxyAndProxyInThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) @@ -302,6 +306,7 @@ func TestProxyURL_NoEnrollProxyAndProxyInThePolicy(t *testing.T) { func TestProxyURL_RemoveProxyFromThePolicy(t *testing.T) { _ = define.Require(t, define.Requirements{ + Group: Fleet, Local: false, Sudo: true, }) diff --git a/testing/integration/upgrade_broken_package_test.go b/testing/integration/upgrade_broken_package_test.go index a9f090d132b..635480c0f36 100644 --- a/testing/integration/upgrade_broken_package_test.go +++ b/testing/integration/upgrade_broken_package_test.go @@ -25,6 +25,7 @@ import ( func TestUpgradeBrokenPackageVersion(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_downgrade_test.go b/testing/integration/upgrade_downgrade_test.go index 0fa42552340..9067fcda790 100644 --- a/testing/integration/upgrade_downgrade_test.go +++ b/testing/integration/upgrade_downgrade_test.go @@ -23,6 +23,7 @@ import ( func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 8ced575fc63..b63faf35741 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -31,6 +31,7 @@ import ( // versions as the standalone tests already perform those tests and would be redundant. func TestFleetManagedUpgrade(t *testing.T) { info := define.Require(t, define.Requirements{ + Group: Fleet, Stack: &define.Stack{}, Local: false, // requires Agent installation Sudo: true, // requires Agent installation @@ -74,7 +75,98 @@ func TestFleetManagedUpgrade(t *testing.T) { testUpgradeFleetManagedElasticAgent(ctx, t, info, startFixture, endFixture) } +<<<<<<< HEAD func testUpgradeFleetManagedElasticAgent(ctx context.Context, t *testing.T, info *define.Info, startFixture *atesting.Fixture, endFixture *atesting.Fixture) { +======= +func TestFleetAirGappedUpgrade(t *testing.T) { + stack := define.Require(t, define.Requirements{ + Group: FleetAirgapped, + Stack: &define.Stack{}, + // The test uses iptables to simulate the air-gaped environment. + OS: []define.OS{{Type: define.Linux}}, + Local: false, // Needed as the test requires Agent installation + Sudo: true, // Needed as the test uses iptables and installs the Agent + }) + + ctx, _ := testcontext.WithDeadline( + t, context.Background(), time.Now().Add(10*time.Minute)) + + artifactAPI := tools.NewArtifactAPIClient() + latest, err := artifactAPI.GetLatestSnapshotVersion(ctx, t) + require.NoError(t, err, "could not fetch latest version from artifacts API") + + // We need to prepare it first because it'll download the artifact, and it + // has to happen before we block the artifacts API IPs. + // The test does not need a fixture, but testUpgradeFleetManagedElasticAgent + // uses it to get some information about the agent version. + upgradeTo, err := atesting.NewFixture( + t, + latest.String(), + atesting.WithFetcher(atesting.ArtifactFetcher()), + ) + require.NoError(t, err) + err = upgradeTo.Prepare(ctx) + require.NoError(t, err) + + s := newArtifactsServer(ctx, t, latest.String()) + host := "artifacts.elastic.co" + simulateAirGapedEnvironment(t, host) + + rctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodGet, "https://"+host, nil) + _, err = http.DefaultClient.Do(req) + if !(errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, os.ErrDeadlineExceeded)) { + t.Fatalf( + "request to %q should have failed, iptables rules should have blocked it", + host) + } + + _, err = stack.ESClient.Info() + require.NoErrorf(t, err, + "failed to interact with ES after blocking %q through iptables", host) + _, body, err := stack.KibanaClient.Request(http.MethodGet, "/api/features", + nil, nil, nil) + require.NoErrorf(t, err, + "failed to interact with Kibana after blocking %q through iptables. "+ + "It should not affect the connection to the stack. Host: %s, response body: %s", + stack.KibanaClient.URL, host, body) + + fixture, err := define.NewFixture(t, define.Version()) + require.NoError(t, err) + err = fixture.Prepare(ctx) + require.NoError(t, err) + + t.Logf("Testing Elastic Agent upgrade from %s to %s with Fleet...", + define.Version(), latest) + + downloadSource := kibana.DownloadSource{ + Name: "local-air-gaped-" + uuid.NewString(), + Host: s.URL + "/downloads/beats/elastic-agent/", + IsDefault: false, // other tests reuse the stack, let's not mess things up + } + t.Logf("creating download source %q, using %q.", + downloadSource.Name, downloadSource.Host) + src, err := stack.KibanaClient.CreateDownloadSource(ctx, downloadSource) + require.NoError(t, err, "could not create download source") + + policy := defaultPolicy() + policy.DownloadSourceID = src.Item.ID + + testUpgradeFleetManagedElasticAgent(ctx, t, stack, fixture, upgradeTo, policy) +} + +func testUpgradeFleetManagedElasticAgent( + ctx context.Context, + t *testing.T, + info *define.Info, + startFixture *atesting.Fixture, + endFixture *atesting.Fixture, + policy kibana.AgentPolicy) { + kibClient := info.KibanaClient + +>>>>>>> 8a8abd046a (Add ability to split integration tests into different groups (#3544)) startVersionInfo, err := startFixture.ExecVersion(ctx) require.NoError(t, err) startParsedVersion, err := version.ParseVersion(startVersionInfo.Binary.String()) diff --git a/testing/integration/upgrade_gpg_test.go b/testing/integration/upgrade_gpg_test.go index 56d4378d147..a2399e85d6f 100644 --- a/testing/integration/upgrade_gpg_test.go +++ b/testing/integration/upgrade_gpg_test.go @@ -23,6 +23,7 @@ import ( func TestStandaloneUpgradeWithGPGFallback(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) @@ -78,6 +79,7 @@ func TestStandaloneUpgradeWithGPGFallback(t *testing.T) { func TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_rollback_test.go b/testing/integration/upgrade_rollback_test.go index 3226ae92a51..4b83675e040 100644 --- a/testing/integration/upgrade_rollback_test.go +++ b/testing/integration/upgrade_rollback_test.go @@ -31,6 +31,7 @@ import ( // that the Agent is rolled back to the previous version. func TestStandaloneUpgradeRollback(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) @@ -128,6 +129,7 @@ inputs: // rolled back to the previous version. func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_inprogress.go b/testing/integration/upgrade_standalone_inprogress.go index adc52cf0872..53b2c50fbed 100644 --- a/testing/integration/upgrade_standalone_inprogress.go +++ b/testing/integration/upgrade_standalone_inprogress.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build integration + package integration import ( @@ -24,6 +26,7 @@ import ( // the second upgrade. func TestStandaloneUpgradeFailsWhenUpgradeIsInProgress(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_retry_test.go b/testing/integration/upgrade_standalone_retry_test.go index 00e74b4cef0..4273452cc50 100644 --- a/testing/integration/upgrade_standalone_retry_test.go +++ b/testing/integration/upgrade_standalone_retry_test.go @@ -26,6 +26,7 @@ import ( func TestStandaloneUpgradeRetryDownload(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_standalone_test.go b/testing/integration/upgrade_standalone_test.go index f1d97bd12b5..d79a7f18da3 100644 --- a/testing/integration/upgrade_standalone_test.go +++ b/testing/integration/upgrade_standalone_test.go @@ -21,6 +21,7 @@ import ( func TestStandaloneUpgrade(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) diff --git a/testing/integration/upgrade_uninstall_test.go b/testing/integration/upgrade_uninstall_test.go index a26a3bb6d1e..da3b1cae17d 100644 --- a/testing/integration/upgrade_uninstall_test.go +++ b/testing/integration/upgrade_uninstall_test.go @@ -24,6 +24,7 @@ import ( func TestStandaloneUpgradeUninstallKillWatcher(t *testing.T) { define.Require(t, define.Requirements{ + Group: Upgrade, Local: false, // requires Agent installation Sudo: true, // requires Agent installation })