diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml index dcce7b4553..aa350a4a49 100644 --- a/.ci/.e2e-tests.yaml +++ b/.ci/.e2e-tests.yaml @@ -15,12 +15,16 @@ SUITES: - "ubuntu-18.04" scenarios: - name: "Fleet" + pullRequestFilter: " && ~debian" tags: "fleet_mode_agent" - name: "Endpoint Integration" + pullRequestFilter: " && ~debian" tags: "agent_endpoint_integration" - name: "Stand-alone" + pullRequestFilter: " && ~ubi8" tags: "stand_alone_agent" - name: "Backend Processes" + pullRequestFilter: " && ~debian" tags: "backend_processes" - suite: "metricbeat" platforms: @@ -45,6 +49,7 @@ SUITES: - name: "Metricbeat" tags: "metricbeat" - name: "MySQL" + pullRequestFilter: " && ~old_versions" tags: "integrations && mysql" - name: "Oracle" tags: "integrations && oracle" diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8c7d5f2a4c..a549ab9bc8 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -250,12 +250,13 @@ def checkTestSuite(Map parallelTasks = [:], Map item = [:]) { def suite = item.suite def platforms = item.platforms item.scenarios.each { scenario -> + def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : '' def tags = scenario.tags def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ] if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) { platforms.each { platform -> log(level: 'INFO', text: "Adding ${platform}:${suite}:${tags} test suite to the build execution") - parallelTasks["${platform}_${suite}_${tags}"] = generateFunctionalTestStep(platform: "${platform}", suite: "${suite}", tags: "${tags}") + parallelTasks["${platform}_${suite}_${tags}"] = generateFunctionalTestStep(platform: "${platform}", suite: "${suite}", tags: "${tags}", pullRequestFilter: "${pullRequestFilter}") } } else { log(level: 'WARN', text: "The ${suite}:${tags} test suite won't be executed because there are no modified files") @@ -307,6 +308,7 @@ def generateFunctionalTestStep(Map args = [:]){ def platform = args.get('platform') def suite = args.get('suite') def tags = args.get('tags') + def pullRequestFilter = args.get('pullRequestFilter')?.trim() ? args.get('pullRequestFilter') : '' // We will decide whether to include the nightly tests in the execution at CI time, only. // On the other hand, the developers can use the TAGS environment variable locally. @@ -317,6 +319,10 @@ def generateFunctionalTestStep(Map args = [:]){ } tags += excludeNightlyTag + if (isPR() || isUpstreamTrigger(filter: 'PR-')) { + tags += pullRequestFilter + } + return { node("${platform} && immutable && docker") { try { diff --git a/e2e/_suites/fleet/features/agent_endpoint_integration.feature b/e2e/_suites/fleet/features/agent_endpoint_integration.feature index 0f692c1ad8..d74de99922 100644 --- a/e2e/_suites/fleet/features/agent_endpoint_integration.feature +++ b/e2e/_suites/fleet/features/agent_endpoint_integration.feature @@ -3,36 +3,84 @@ Feature: Agent Endpoint Integration Scenarios for Agent to deploy Endpoint and sending data to Fleet and Elasticsearch. @deploy-endpoint-with-agent -Scenario: Adding the Endpoint Integration to an Agent makes the host to show in Security App - Given a "centos" agent is deployed to Fleet with "tar" installer +Scenario Outline: Adding the Endpoint Integration to an Agent makes the host to show in Security App + Given a "" agent is deployed to Fleet with "tar" installer And the agent is listed in Fleet as "online" When the "Endpoint Security" integration is "added" in the policy Then the "Endpoint Security" datasource is shown in the policy as added And the host name is shown in the Administration view in the Security App as "online" +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | + @endpoint-policy-check -Scenario: Deploying an Endpoint makes policies to appear in the Security App - When an Endpoint is successfully deployed with a "centos" Agent using "tar" installer +Scenario Outline: Deploying an Endpoint makes policies to appear in the Security App + When an Endpoint is successfully deployed with a "" Agent using "tar" installer Then the policy response will be shown in the Security App +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | + @set-policy-and-check-changes -Scenario: Changing an Agent policy is reflected in the Security App - Given an Endpoint is successfully deployed with a "centos" Agent using "tar" installer +Scenario Outline: Changing an Agent policy is reflected in the Security App + Given an Endpoint is successfully deployed with a "" Agent using "tar" installer When the policy is updated to have "malware" in "detect" mode Then the policy will reflect the change in the Security App +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | + @deploy-endpoint-then-unenroll-agent -Scenario: Un-enrolling Elastic Agent stops Elastic Endpoint - Given an Endpoint is successfully deployed with a "centos" Agent using "tar" installer +Scenario Outline: Un-enrolling Elastic Agent stops Elastic Endpoint + Given an Endpoint is successfully deployed with a "" Agent using "tar" installer When the agent is un-enrolled Then the agent is listed in Fleet as "inactive" And the host name is not shown in the Administration view in the Security App - And the "elastic-endpoint" process is in the "stopped" state on the host + +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | @deploy-endpoint-then-remove-it-from-policy -Scenario: Removing Endpoint from Agent policy stops the connected Endpoint - Given an Endpoint is successfully deployed with a "centos" Agent using "tar" installer +Scenario Outline: Removing Endpoint from Agent policy stops the connected Endpoint + Given an Endpoint is successfully deployed with a "" Agent using "tar" installer When the "Endpoint Security" integration is "removed" in the policy Then the agent is listed in Fleet as "online" But the host name is not shown in the Administration view in the Security App - And the "elastic-endpoint" process is in the "stopped" state on the host + +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | diff --git a/e2e/_suites/fleet/features/backend_processes.feature b/e2e/_suites/fleet/features/backend_processes.feature index 88ae79bf9b..5e61d0e9f3 100644 --- a/e2e/_suites/fleet/features/backend_processes.feature +++ b/e2e/_suites/fleet/features/backend_processes.feature @@ -8,9 +8,15 @@ Scenario Outline: Deploying the agent When the "elastic-agent" process is in the "started" state on the host Then the "filebeat" process is in the "started" state on the host And the "metricbeat" process is in the "started" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @enroll @@ -19,9 +25,15 @@ Scenario Outline: Deploying the agent with enroll and then run on rpm and d When the "elastic-agent" process is in the "started" state on the host Then the "filebeat" process is in the "started" state on the host And the "metricbeat" process is in the "started" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @stop-agent @@ -30,9 +42,15 @@ Scenario Outline: Stopping the agent stops backend processes When the "elastic-agent" process is "stopped" on the host Then the "filebeat" process is in the "stopped" state on the host And the "metricbeat" process is in the "stopped" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @restart-agent @@ -41,9 +59,15 @@ Scenario Outline: Restarting the installed agent When the "elastic-agent" process is "restarted" on the host Then the "filebeat" process is in the "started" state on the host And the "metricbeat" process is in the "started" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @restart-host @@ -53,9 +77,15 @@ Scenario Outline: Restarting the host with persistent agent restarts backen Then the "elastic-agent" process is in the "started" state on the host And the "filebeat" process is in the "started" state on the host And the "metricbeat" process is in the "started" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @unenroll @@ -65,9 +95,15 @@ Scenario Outline: Un-enrolling the agent stops backend processes Then the "elastic-agent" process is in the "started" state on the host And the "filebeat" process is in the "stopped" state on the host And the "metricbeat" process is in the "stopped" state on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @reenroll @@ -77,9 +113,15 @@ Scenario Outline: Re-enrolling the agent starts the elastic-agent process And the "elastic-agent" process is "stopped" on the host When the agent is re-enrolled on the host Then the "elastic-agent" process is "started" on the host -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @uninstall-host @@ -89,7 +131,45 @@ Scenario Outline: Un-installing the installed agent Then the "elastic-agent" process is in the "stopped" state on the host And the "filebeat" process is in the "stopped" state on the host And the "metricbeat" process is in the "stopped" state on the host -Examples: + +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | + +@deploy-endpoint-then-unenroll-agent +Scenario Outline: Un-enrolling Elastic Agent stops Elastic Endpoint + Given an Endpoint is successfully deployed with a "" Agent using "tar" installer + When the agent is un-enrolled + Then the "elastic-endpoint" process is in the "stopped" state on the host + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | +| debian | + +@deploy-endpoint-then-remove-it-from-policy +Scenario Outline: Removing Endpoint from Agent policy stops the connected Endpoint + Given an Endpoint is successfully deployed with a "" Agent using "tar" installer + When the "Endpoint Security" integration is "removed" in the policy + Then the "elastic-endpoint" process is in the "stopped" state on the host + +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | | debian | diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 26d77eeddd..31eab5853e 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -8,9 +8,15 @@ Scenario Outline: Deploying the agent When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" And system package dashboards are listed in Fleet -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @enroll @@ -19,9 +25,15 @@ Scenario Outline: Deploying the agent with enroll and then run on rpm and d When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" And system package dashboards are listed in Fleet -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | # @upgrade-agent @@ -32,18 +44,26 @@ Scenario Outline: Upgrading the installed agent And the "elastic-agent" process is "restarted" on the host When agent is upgraded to version "latest" Then agent is in version "latest" -Examples: -| os | -| debian | + +@debian +Examples: Debian +| os | +| debian | @restart-agent Scenario Outline: Restarting the installed agent Given a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is "restarted" on the host Then the agent is listed in Fleet as "online" -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @unenroll @@ -51,9 +71,15 @@ Scenario Outline: Un-enrolling the agent deactivates the agent Given a "" agent is deployed to Fleet with "tar" installer When the agent is un-enrolled Then the agent is listed in Fleet as "inactive" -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @reenroll @@ -64,9 +90,15 @@ Scenario Outline: Re-enrolling the agent activates the agent in Fleet And the agent is re-enrolled on the host When the "elastic-agent" process is "started" on the host Then the agent is listed in Fleet as "online" -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @revoke-token @@ -74,9 +106,15 @@ Scenario Outline: Revoking the enrollment token for the agent Given a "" agent is deployed to Fleet with "tar" installer When the enrollment token is revoked Then an attempt to enroll a new agent fails -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | @uninstall-host @@ -85,7 +123,13 @@ Scenario Outline: Un-installing the installed agent When the "elastic-agent" process is "uninstalled" on the host Then the file system Agent folder is empty And the agent is listed in Fleet as "offline" -Examples: + +@centos +Examples: Centos | os | | centos | + +@debian +Examples: Debian +| os | | debian | diff --git a/e2e/_suites/fleet/features/stand_alone_agent.feature b/e2e/_suites/fleet/features/stand_alone_agent.feature index bdc78beeec..5842cd57cb 100644 --- a/e2e/_suites/fleet/features/stand_alone_agent.feature +++ b/e2e/_suites/fleet/features/stand_alone_agent.feature @@ -9,18 +9,30 @@ Scenario Outline: Starting the agent starts backend processes When a "" stand-alone agent is deployed Then the "filebeat" process is in the "started" state on the host And the "metricbeat" process is in the "started" state on the host -Examples: + +@default +Examples: default | image | | default | + +@ubi8 +Examples: Ubi8 +| image | | ubi8 | @deploy-stand-alone Scenario Outline: Deploying a stand-alone agent When a "" stand-alone agent is deployed Then there is new data in the index from agent -Examples: + +@default +Examples: default | image | | default | + +@ubi8 +Examples: Ubi8 +| image | | ubi8 | @stop-agent @@ -28,7 +40,13 @@ Scenario Outline: Stopping the agent container stops data going into ES Given a "" stand-alone agent is deployed When the "elastic-agent" docker container is stopped Then there is no new data in the index after agent shuts down -Examples: + +@default +Examples: default | image | | default | + +@ubi8 +Examples: Ubi8 +| image | | ubi8 | diff --git a/e2e/_suites/metricbeat/features/integrations.feature b/e2e/_suites/metricbeat/features/integrations.feature index ea6afe2c09..dbde408369 100644 --- a/e2e/_suites/metricbeat/features/integrations.feature +++ b/e2e/_suites/metricbeat/features/integrations.feature @@ -92,20 +92,24 @@ Examples: vSphere | integration | version | | vsphere | latest | -@variants +@mysql Scenario Outline: -- sends metrics to Elasticsearch without errors Given "" v, variant of "", is running for metricbeat When metricbeat is installed and configured for "", variant of the "" module Then there are "" events in the index And there are no errors in the index -@mysql -Examples: MySQL +@latest_versions +Examples: MySQL (latest versions) | integration | variant | version | -| mysql | MariaDB | 10.2.23 | -| mysql | MariaDB | 10.3.14 | | mysql | MariaDB | 10.4.4 | -| mysql | MySQL | 5.7.12 | | mysql | MySQL | 8.0.13 | +| mysql | MySQL | 5.7.12 | + +@old_versions +Examples: MySQL (old versions) +| integration | variant | version | +| mysql | MariaDB | 10.2.23 | +| mysql | MariaDB | 10.3.14 | | mysql | Percona | 5.7.24 | | mysql | Percona | 8.0.13-4 |