Introduce Go system tests (#3790) (#4151)

* systemtest: introduce new system testing framework

New system testing framework, written in Go.

The goal here is to simplify the development and
maintenance of apm-server system tests, consolidating
on Go as the language of choice. The framework
provides abstractions for:

 - building and starting apm-server with a specified configuration
 - querying apm-server metrics via /debug/vars
 - parsing and iterating over apm-server logs
 - querying Elasticsearch, with optional return conditions

* tests/system: remove old API Key command tests

* tests/system: remove old logging tests

Closes #3353

* tests/system: remove old Jaeger tests

* tests/system: remove old onboarding test

* tests/system: reinstate invalidate function

* Add TODO about testing OSS build

* Add "extra" args _after_ converting config to JSON

* Add constants for Kibana

And fix env var for KIBANA_PORT

* systemtests: API Key test improvements

- cleanup apm_server_user API Keys
- test validity of API Keys with authenticate requests

.gitignore

@@ -30,3 +30,4 @@ html_docs
 /x-pack/apm-server/logs
 /docker-compose.override.yml
 /config.mk
+/systemtest/logs

script/jenkins/linux-test.sh

@@ -12,3 +12,6 @@ cleanup() {
 trap cleanup EXIT
 
 make update docker-system-tests
+
+# TODO(axw) make this part of the "system-tests" target
+(cd systemtest && go test -v ./...)

systemtest/apikeycmd_test.go

@@ -0,0 +1,161 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package systemtest_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-server/systemtest"
+	"github.com/elastic/apm-server/systemtest/apmservertest"
+	"github.com/elastic/apm-server/systemtest/estest"
+)
+
+func apiKeyCommand(subcommand string, args ...string) *apmservertest.ServerCmd {
+	cfg := apmservertest.DefaultConfig()
+	cfgargs, err := cfg.Args()
+	if err != nil {
+		panic(err)
+	}
+
+	var esargs []string
+	for i := 1; i < len(cfgargs); i += 2 {
+		if !strings.HasPrefix(cfgargs[i], "output.elasticsearch") {
+			continue
+		}
+		esargs = append(esargs, "-E", cfgargs[i])
+	}
+
+	userargs := args
+	args = append([]string{subcommand}, esargs...)
+	args = append(args, userargs...)
+	return apmservertest.ServerCommand("apikey", args...)
+}
+
+func TestAPIKeyCreate(t *testing.T) {
+	systemtest.InvalidateAPIKeys(t)
+	defer systemtest.InvalidateAPIKeys(t)
+
+	cmd := apiKeyCommand("create", "--name", t.Name(), "--json")
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err)
+
+	attrs := decodeJSONMap(t, bytes.NewReader(out))
+	assert.Equal(t, t.Name(), attrs["name"])
+	assert.Contains(t, attrs, "id")
+	assert.Contains(t, attrs, "api_key")
+	assert.Contains(t, attrs, "credentials")
+
+	es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string))
+	assertAuthenticateSucceeds(t, es)
+}
+
+func TestAPIKeyCreateExpiration(t *testing.T) {
+	systemtest.InvalidateAPIKeys(t)
+	defer systemtest.InvalidateAPIKeys(t)
+
+	cmd := apiKeyCommand("create", "--name", t.Name(), "--json", "--expiration=1d")
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err)
+
+	attrs := decodeJSONMap(t, bytes.NewReader(out))
+	assert.Contains(t, attrs, "expiration")
+}
+
+func TestAPIKeyInvalidateName(t *testing.T) {
+	systemtest.InvalidateAPIKeys(t)
+	defer systemtest.InvalidateAPIKeys(t)
+
+	var clients []*estest.Client
+	for i := 0; i < 2; i++ {
+		cmd := apiKeyCommand("create", "--name", t.Name(), "--json")
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err)
+
+		attrs := decodeJSONMap(t, bytes.NewReader(out))
+		es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string))
+		assertAuthenticateSucceeds(t, es)
+		clients = append(clients, es)
+	}
+
+	cmd := apiKeyCommand("invalidate", "--name", t.Name(), "--json")
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err)
+
+	result := decodeJSONMap(t, bytes.NewReader(out))
+	assert.Len(t, result["invalidated_api_keys"], 2)
+	assert.Equal(t, float64(0), result["error_count"])
+
+	for _, es := range clients {
+		assertAuthenticateFails(t, es)
+	}
+}
+
+func TestAPIKeyInvalidateID(t *testing.T) {
+	systemtest.InvalidateAPIKeys(t)
+	defer systemtest.InvalidateAPIKeys(t)
+
+	cmd := apiKeyCommand("create", "--json")
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err)
+	attrs := decodeJSONMap(t, bytes.NewReader(out))
+
+	es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string))
+	assertAuthenticateSucceeds(t, es)
+
+	cmd = apiKeyCommand("invalidate", "--json", "--id", attrs["id"].(string))
+	out, err = cmd.CombinedOutput()
+	require.NoError(t, err)
+	result := decodeJSONMap(t, bytes.NewReader(out))
+
+	assert.Equal(t, []interface{}{attrs["id"]}, result["invalidated_api_keys"])
+	assert.Equal(t, float64(0), result["error_count"])
+	assertAuthenticateFails(t, es)
+}
+
+func assertAuthenticateSucceeds(t testing.TB, es *estest.Client) *esapi.Response {
+	t.Helper()
+	resp, err := es.Security.Authenticate()
+	require.NoError(t, err)
+	assert.False(t, resp.IsError())
+	return resp
+}
+
+func assertAuthenticateFails(t testing.TB, es *estest.Client) *esapi.Response {
+	t.Helper()
+	resp, err := es.Security.Authenticate()
+	require.NoError(t, err)
+	assert.True(t, resp.IsError())
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+	return resp
+}
+
+func decodeJSONMap(t *testing.T, r io.Reader) map[string]interface{} {
+	var m map[string]interface{}
+	err := json.NewDecoder(r).Decode(&m)
+	require.NoError(t, err)
+	return m
+}

systemtest/apmservertest/command.go

@@ -0,0 +1,187 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmservertest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// TODO(axw): add support for building/running the OSS apm-server.
+
+// ServerCommand returns a ServerCmd (wrapping os/exec) for running
+// apm-server with args.
+func ServerCommand(subcommand string, args ...string) *ServerCmd {
+	binary, buildErr := buildServer()
+	if buildErr != nil {
+		// Dummy command; Start etc. will return the build error.
+		binary = "/usr/bin/false"
+	}
+	args = append([]string{subcommand}, args...)
+	cmd := exec.Command(binary, args...)
+	cmd.SysProcAttr = serverCommandSysProcAttr
+	return &ServerCmd{
+		Cmd:        cmd,
+		buildError: buildErr,
+	}
+}
+
+// ServerCmd wraps an os/exec.Cmd, taking care of building apm-server
+// and cleaning up on close.
+type ServerCmd struct {
+	*exec.Cmd
+	buildError error
+	tempdir    string
+}
+
+// Run runs the apm-server command, and waits for it to exit.
+func (c *ServerCmd) Run() error {
+	if err := c.Start(); err != nil {
+		return err
+	}
+	return c.Wait()
+}
+
+// Output runs the apm-server command, waiting for it to exit
+// and returning its stdout.
+func (c *ServerCmd) Output() ([]byte, error) {
+	if err := c.prestart(); err != nil {
+		return nil, err
+	}
+	defer c.cleanup()
+	return c.Cmd.Output()
+}
+
+// CombinedOutput runs the apm-server command, waiting for it to exit
+// and returning its combined stdout/stderr.
+func (c *ServerCmd) CombinedOutput() ([]byte, error) {
+	if err := c.prestart(); err != nil {
+		return nil, err
+	}
+	defer c.cleanup()
+	return c.Cmd.CombinedOutput()
+}
+
+// Start starts the apm-server command, and returns immediately.
+func (c *ServerCmd) Start() error {
+	if err := c.prestart(); err != nil {
+		return err
+	}
+	if err := c.Cmd.Start(); err != nil {
+		c.cleanup()
+		return err
+	}
+	return nil
+}
+
+// Wait waits for the previously started apm-server command to exit.
+func (c *ServerCmd) Wait() error {
+	defer c.cleanup()
+	return c.Cmd.Wait()
+}
+
+func (c *ServerCmd) prestart() error {
+	if c.buildError != nil {
+		return c.buildError
+	}
+	if c.Dir == "" {
+		if err := c.createTempDir(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *ServerCmd) createTempDir() error {
+	tempdir, err := ioutil.TempDir("", "apm-server-systemtest")
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filepath.Join(tempdir, "apm-server.yml"), nil, 0644); err != nil {
+		os.RemoveAll(tempdir)
+		return err
+	}
+
+	// Symlink ingest/pipeline/definition.json into the temporary directory.
+	pipelineDir := filepath.Join(tempdir, "ingest", "pipeline")
+	if err := os.MkdirAll(pipelineDir, 0755); err != nil {
+		os.RemoveAll(tempdir)
+		return err
+	}
+	pipelineDefinitionFile := filepath.Join(filepath.Dir(c.Cmd.Path), "ingest", "pipeline", "definition.json")
+	pipelineDefinitionSymlink := filepath.Join(pipelineDir, "definition.json")
+	if err := os.Symlink(pipelineDefinitionFile, pipelineDefinitionSymlink); err != nil {
+		if !os.IsExist(err) {
+			os.RemoveAll(tempdir)
+			return err
+		}
+	}
+
+	c.tempdir = tempdir
+	c.Dir = tempdir
+	return nil
+}
+
+func (c *ServerCmd) cleanup() {
+	if c.tempdir != "" {
+		os.RemoveAll(c.tempdir)
+	}
+}
+
+// buildServer builds the apm-server binary, returning its absolute path.
+func buildServer() (string, error) {
+	apmServerBinaryMu.Lock()
+	defer apmServerBinaryMu.Unlock()
+	if apmServerBinary != "" {
+		return apmServerBinary, nil
+	}
+
+	// Build apm-server binary in the repo root.
+	output, err := exec.Command("go", "list", "-m", "-f={{.Dir}}/..").Output()
+	if err != nil {
+		return "", err
+	}
+	repoRoot := filepath.Clean(strings.TrimSpace(string(output)))
+	abspath := filepath.Join(repoRoot, "apm-server")
+	if runtime.GOOS == "windows" {
+		abspath += ".exe"
+	}
+
+	log.Println("Building apm-server...")
+	cmd := exec.Command("go", "build", "-o", abspath, "./x-pack/apm-server")
+	cmd.Dir = repoRoot
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return "", err
+	}
+	log.Println("Built", abspath)
+	apmServerBinary = abspath
+	return apmServerBinary, nil
+}
+
+var (
+	apmServerBinaryMu sync.Mutex
+	apmServerBinary   string
+)