Remove Python system tests (#6555)

.ci/scripts/linux-test.sh

@@ -11,10 +11,10 @@ cleanup() {
 }
 trap cleanup EXIT
 
-make update apm-server docker-system-tests
+make update apm-server
 
 SYSTEM_TESTS_XUNIT_PATH="$(pwd)/build"
-# TODO(axw) make this part of the "system-tests" target
+# TODO(axw) make this a Makefile target
 # TODO(mdelapenya) meanwhile there is no 'DefaultGoTestSystemArgs' at beats' GoTest implementation, this command is reproducing what Beats should provide: a gotestsum representation for system tests.
 #    In the same hand, the system tests should be tagged with "// +build systemtests"
 (cd systemtest && gotestsum --no-color -f standard-quiet --junitfile "${SYSTEM_TESTS_XUNIT_PATH}/TEST-go-system_tests.xml" --jsonfile "${SYSTEM_TESTS_XUNIT_PATH}/TEST-go-system_tests.out.json" -- -tags  systemtests ./...)

.ci/scripts/windows-build.ps1

@@ -40,8 +40,6 @@ exec { go get github.com/docker/libcompose }
 
 if (Test-Path "build") { Remove-Item -Recurse -Force build }
 New-Item -ItemType directory -Path build\coverage | Out-Null
-New-Item -ItemType directory -Path build\system-tests | Out-Null
-New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
 choco install python -y -r --no-progress --version 3.8.1.20200110
 refreshenv

.ci/scripts/windows-test.ps1

@@ -47,8 +47,6 @@ exec { go get github.com/docker/libcompose }
 
 if (Test-Path "build") { Remove-Item -Recurse -Force build }
 New-Item -ItemType directory -Path build\coverage | Out-Null
-New-Item -ItemType directory -Path build\system-tests | Out-Null
-New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
 echo "Building $env:beat"
 exec { mage build } "Build FAILURE"

Makefile

@@ -63,14 +63,6 @@ clean: $(MAGE)
 # Checks/tests.
 ##############################################################################
 
-# SYSTEM_TEST_TARGET is passed to nosetests in "system-tests".
-#
-# This may be overridden to specify which tests to run.
-SYSTEM_TEST_TARGET?=./tests/system
-
-# PYTEST_OPTIONS is passed to pytest in "system-tests".
-PYTEST_OPTIONS?=--timeout=90 --durations=20 --junit-xml=build/TEST-system.xml
-
 .PHONY: check-full
 check-full: update check golint staticcheck check-docker-compose
 
@@ -86,33 +78,11 @@ check: $(MAGE) check-fmt check-headers check-package
 bench:
 	@$(GO) test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
 
-.PHONY: system-tests
-system-tests: $(PYTHON_BIN) apm-server.test
-	INTEGRATION_TESTS=1 TZ=UTC $(PYTHON_BIN)/pytest $(PYTEST_OPTIONS) $(SYSTEM_TEST_TARGET)
-
-.PHONY: docker-system-tests
-docker-system-tests: export SYSTEM_TEST_TARGET:=$(SYSTEM_TEST_TARGET)
-docker-system-tests: docker-compose.override.yml
-	docker-compose build
-	docker-compose run --rm -T beat make system-tests
-
-# docker-compose.override.yml holds overrides for docker-compose.yml.
-#
-# Create this to ensure the UID used inside docker-compose is the same
-# as the current user on the host, so files are created with the same
-# privileges.
-#
-# Note that this target is intentionally non-.PHONY, so that users can
-# modify the resulting file without it being overwritten. To recreate
-# the file, remove it.
-docker-compose.override.yml:
-	printf "version: '2.3'\nservices:\n beat:\n  build:\n   args: [UID=%d]" $(shell id -u) > $@
-
 ##############################################################################
 # Rules for updating config files, fields.yml, etc.
 ##############################################################################
 
-update: fields go-generate add-headers copy-docs build-package notice $(MAGE)
+update: fields go-generate add-headers build-package notice $(MAGE)
 	@$(MAGE) update
 	@go mod download all # make sure go.sum is complete
 
@@ -153,25 +123,14 @@ get-version:
 ##############################################################################
 
 .PHONY: docs
-docs: copy-docs
+docs:
 	@rm -rf build/html_docs
 	sh script/build_apm_docs.sh apm-server docs/index.asciidoc build
 
 .PHONY: update-beats-docs
 update-beats-docs: $(PYTHON)
 	@$(PYTHON) script/copy-docs.py
 
-.PHONY: copy-docs
-copy-docs:
-	@mkdir -p docs/data/intake-api/generated/sourcemap
-	@cp testdata/intake-v2/events.ndjson docs/data/intake-api/generated/
-	@cp testdata/intake-v3/rum_events.ndjson docs/data/intake-api/generated/rum_v3_events.ndjson
-	@cp testdata/sourcemap/bundle.js.map docs/data/intake-api/generated/sourcemap/
-	@mkdir -p docs/data/elasticsearch/generated/
-	@cp tests/system/error.approved.json docs/data/elasticsearch/generated/errors.json
-	@cp tests/system/transaction.approved.json docs/data/elasticsearch/generated/transactions.json
-	@cp tests/system/spans.approved.json docs/data/elasticsearch/generated/spans.json
-
 ##############################################################################
 # Beats synchronisation.
 ##############################################################################

TESTING.md

@@ -7,41 +7,8 @@ The tests are built on top of the [Beats Test Framework](https://github.com/elas
 
 To run the unit tests, you can use `make test` or simply `go test ./...`. The unit tests do not require any external services.
 
-The APM Server "system tests" run the APM Server in various scenarios, and require an Elastic Stack to be running.
-To run the system tests locally, first start Elasticsearch and Kibana, e.g.:
-
-```
-docker-compose up -d
-make system-tests
-```
-
-You can alternatively run the system tests entirely within Docker:
-
-```
-make docker-system-tests
-```
-
-### Developing Tests
-
-While developing new tests or troubleshooting test failures, it is handy to run tests from outside of docker, for
-example from within an editor, while still allowing all dependencies to run in containers.  To accomplish this:
-
-* Run `docker-compose up -d` to start docker containers for the Elastic Stack.
-* Run tests with `make system-tests`, e.g.:
-
-```
-make system-tests SYSTEM_TEST_TARGET=./tests/system/test_integration.py:SourcemappingIntegrationTest.test_backend_error
-```
-
-* Or run the dockerised version of the tests with `make docker-system-tests`, e.g.:
-
-```
-make docker-system-tests SYSTEM_TEST_TARGET=./tests/system/test_integration.py:SourcemappingIntegrationTest.test_backend_error
-```
-
-Elasticsearch diagnostics may be enabled by setting `DIAGNOSTIC_INTERVAL`.
-`DIAGNOSTIC_INTERVAL=1` will dump hot threads and task lists every second while tests are running
-to `build/system-tests/run/$test_name/diagnostics/`.
+The APM Server "system tests" run the APM Server in various scenarios, with the Elastic Stack running inside Docker containers.
+To run the system tests locally, you can run `go test` inside the systemtest directory.
 
 ## Snapshot-Testing
 Some tests make use of the concept of _snapshot_ or _approvals testing_. If running tests leads to changed snapshots, you can use the `approvals` tool to update the snapshots.

docker-compose.yml

@@ -1,28 +1,5 @@
 version: '2.3'
 services:
-  beat:
-    build:
-      context: .
-      dockerfile: tests/Dockerfile
-    depends_on:
-      - proxy_dep
-    volumes:
-      - .:/src/apm-server/
-    working_dir: /src/apm-server
-    command: make
-    environment:
-      BEAT_STRICT_PERMS: "${BEAT_STRICT_PERMS:-false}"
-      ES_HOST: elasticsearch
-      ES_PORT: "9200"
-      ES_USER: "${ES_USER:-apm_server_user}"
-      ES_SUPERUSER_USER: "${ES_SUPERUSER_USER:-admin}"
-      ES_SUPERUSER_PASS: "${ES_SUPERUSER_PASS:-changeme}"
-      KIBANA_HOST: kibana
-      KIBANA_PORT: "5601"
-      KIBANA_USER: "${BEAT_KIBANA_USER:-apm_user_ro}"
-      KIBANA_PASS: "${BEAT_KIBANA_PASS:-changeme}"
-      SYSTEM_TEST_TARGET: "${SYSTEM_TEST_TARGET:-}"
-
   # This is a proxy used to block beats until all services are healthy.
   # See: https://github.com/docker/compose/issues/4369
   proxy_dep: