[tests] Add support for resources.ExternalResources (#3811)

This commit adds an interface for adding external (i.e. not M3)
resources that can be used in integration tests. An
implementation of the interface that allows users to spin up
a docker-backed Prometheus is added as well. This also functions
as a working example for how the interface can be implemented to
support other external resources.

src/integration/resources/docker/common.go

@@ -43,8 +43,12 @@ type dockerResourceOptions struct {
 	containerName    string
 	image            dockerImage
 	portList         []int
-	mounts           []string
-	iOpts            instrument.Options
+	// mounts creates mounts in the container that map back to a resource
+	// on the host system.
+	mounts []string
+	// tmpfsMounts creates mounts to the container's temporary file system
+	tmpfsMounts []string
+	iOpts       instrument.Options
 }
 
 // NB: this will fill unset fields with given default values.
@@ -70,6 +74,10 @@ func (o dockerResourceOptions) withDefaults(
 		o.portList = defaultOpts.portList
 	}
 
+	if len(o.tmpfsMounts) == 0 {
+		o.tmpfsMounts = defaultOpts.tmpfsMounts
+	}
+
 	if len(o.mounts) == 0 {
 		o.mounts = defaultOpts.mounts
 	}

src/integration/resources/docker/config/prometheus.yml

@@ -0,0 +1,48 @@
+# my global config
+global:
+  external_labels:
+    role: "remote"
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'coordinator'
+    static_configs:
+      - targets: ['coordinator01:7203']
+
+  - job_name: 'dbnode'
+    static_configs:
+      - targets: ['dbnode01:9004']
+
+remote_read:
+  - url: http://coordinator01:7201/api/v1/prom/remote/read
+
+remote_write:
+  - url: http://coordinator01:7201/api/v1/prom/remote/write
+    write_relabel_configs:
+      - target_label: metrics_storage
+        replacement: m3db_remote

src/integration/resources/docker/coordinator.go

@@ -58,7 +58,7 @@ func newDockerHTTPCoordinator(
 	opts dockerResourceOptions,
 ) (resources.Coordinator, error) {
 	opts = opts.withDefaults(defaultCoordinatorOptions)
-	opts.mounts = []string{"/etc/m3coordinator/"}
+	opts.tmpfsMounts = []string{"/etc/m3coordinator/"}
 
 	resource, err := newDockerResource(pool, opts)
 	if err != nil {

src/integration/resources/docker/docker_resource.go

@@ -65,9 +65,10 @@ func newDockerResource(
 	opts := exposePorts(newOptions(containerName), portList)
 
 	hostConfigOpts := func(c *dc.HostConfig) {
+		c.AutoRemove = true
 		c.NetworkMode = networkName
-		mounts := make([]dc.HostMount, 0, len(resourceOpts.mounts))
-		for _, m := range resourceOpts.mounts {
+		mounts := make([]dc.HostMount, 0, len(resourceOpts.tmpfsMounts))
+		for _, m := range resourceOpts.tmpfsMounts {
 			mounts = append(mounts, dc.HostMount{
 				Target: m,
 				Type:   string(mount.TypeTmpfs),
@@ -89,6 +90,7 @@ func newDockerResource(
 		}
 	} else {
 		opts = useImage(opts, image)
+		opts.Mounts = resourceOpts.mounts
 		imageWithTag := fmt.Sprintf("%v:%v", image.name, image.tag)
 		logger.Info("running container with options",
 			zap.String("image", imageWithTag), zap.Any("options", opts))

src/integration/resources/docker/prometheus.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2021  Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package docker
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ory/dockertest/v3"
+
+	"github.com/m3db/m3/src/integration/resources"
+	"github.com/m3db/m3/src/x/instrument"
+)
+
+type prometheus struct {
+	pool      *dockertest.Pool
+	pathToCfg string
+	iOpts     instrument.Options
+
+	resource *dockerResource
+}
+
+// PrometheusOptions contains the options for
+// spinning up docker container running Prometheus
+type PrometheusOptions struct {
+	// Pool is the connection to the docker API
+	Pool *dockertest.Pool
+	// PathToCfg contains the path to the prometheus.yml configuration
+	// file to be used on startup.
+	PathToCfg string
+	// InstrumentOptions are the instrument.Options to use when
+	// creating the resource.
+	InstrumentOptions instrument.Options
+}
+
+// NewPrometheus creates a new docker-backed Prometheus
+// that implements the resources.ExternalResources interface.
+func NewPrometheus(opts PrometheusOptions) resources.ExternalResources {
+	if opts.InstrumentOptions == nil {
+		opts.InstrumentOptions = instrument.NewOptions()
+	}
+	return &prometheus{
+		pool:      opts.Pool,
+		pathToCfg: opts.PathToCfg,
+		iOpts:     opts.InstrumentOptions,
+	}
+}
+
+func (p *prometheus) Setup() error {
+	if p.resource != nil {
+		return errors.New("prometheus already setup. must close resource " +
+			"before attempting to setup again")
+	}
+
+	if err := setupNetwork(p.pool); err != nil {
+		return err
+	}
+
+	res, err := newDockerResource(p.pool, dockerResourceOptions{
+		containerName: "prometheus",
+		image: dockerImage{
+			name: "prom/prometheus",
+			tag:  "latest",
+		},
+		portList: []int{9090},
+		mounts: []string{
+			fmt.Sprintf("%s:/etc/prometheus/prometheus.yml", p.pathToCfg),
+		},
+		iOpts: p.iOpts,
+	})
+	if err != nil {
+		return err
+	}
+
+	p.resource = res
+
+	return nil
+}
+
+func (p *prometheus) Close() error {
+	if p.resource.closed {
+		return errClosed
+	}
+
+	if err := p.resource.close(); err != nil {
+		return err
+	}
+
+	p.resource = nil
+
+	return nil
+}

src/integration/resources/docker/prometheus_test.go

@@ -0,0 +1,75 @@
+// +build integration_v2
+// Copyright (c) 2021  Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package docker
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"path"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v3"
+	"github.com/ory/dockertest/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPrometheus(t *testing.T) {
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err)
+
+	_, filename, _, _ := runtime.Caller(0)
+	prom := NewPrometheus(PrometheusOptions{
+		Pool:      pool,
+		PathToCfg: path.Join(path.Dir(filename), "config/prometheus.yml"),
+	})
+	require.NoError(t, prom.Setup())
+	defer func() {
+		assert.NoError(t, prom.Close())
+	}()
+
+	bo := backoff.NewConstantBackOff(1 * time.Second)
+	var (
+		attempts    = 0
+		maxAttempts = 5
+	)
+	err = backoff.Retry(func() error {
+		req, err := http.NewRequestWithContext(context.Background(), "GET", "http://0.0.0.0:9090/-/ready", nil)
+		require.NoError(t, err)
+
+		client := http.Client{}
+		res, _ := client.Do(req)
+		if res != nil && res.StatusCode == 200 {
+			return nil
+		}
+
+		attempts++
+		if attempts >= maxAttempts {
+			return backoff.Permanent(errors.New("prometheus not ready"))
+		}
+		return errors.New("retryable error")
+	}, bo)
+	require.NoError(t, err)
+}

src/integration/resources/types.go

@@ -152,6 +152,19 @@ type M3Resources interface {
 	Coordinator() Coordinator
 }
 
+// ExternalResources represents an external (i.e. non-M3)
+// resource that we'd like to be able to spin up for an
+// integration test.
+type ExternalResources interface {
+	// Setup sets up the external resource so that it's ready
+	// for use.
+	Setup() error
+
+	// Close stops and cleans up all the resources associated with
+	// the external resource.
+	Close() error
+}
+
 // ClusterOptions represents a set of options for a cluster setup.
 type ClusterOptions struct {
 	ReplicationFactor  int32