diff --git a/src/integration/resources/docker/common.go b/src/integration/resources/docker/common.go index 87127d0b01..4e8e76dbaf 100644 --- a/src/integration/resources/docker/common.go +++ b/src/integration/resources/docker/common.go @@ -43,8 +43,12 @@ type dockerResourceOptions struct { containerName string image dockerImage portList []int - mounts []string - iOpts instrument.Options + // mounts creates mounts in the container that map back to a resource + // on the host system. + mounts []string + // tmpfsMounts creates mounts to the container's temporary file system + tmpfsMounts []string + iOpts instrument.Options } // NB: this will fill unset fields with given default values. @@ -70,6 +74,10 @@ func (o dockerResourceOptions) withDefaults( o.portList = defaultOpts.portList } + if len(o.tmpfsMounts) == 0 { + o.tmpfsMounts = defaultOpts.tmpfsMounts + } + if len(o.mounts) == 0 { o.mounts = defaultOpts.mounts } diff --git a/src/integration/resources/docker/config/prometheus.yml b/src/integration/resources/docker/config/prometheus.yml new file mode 100644 index 0000000000..c11e4d7239 --- /dev/null +++ b/src/integration/resources/docker/config/prometheus.yml @@ -0,0 +1,48 @@ +# my global config +global: + external_labels: + role: "remote" + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'coordinator' + static_configs: + - targets: ['coordinator01:7203'] + + - job_name: 'dbnode' + static_configs: + - targets: ['dbnode01:9004'] + +remote_read: + - url: http://coordinator01:7201/api/v1/prom/remote/read + +remote_write: + - url: http://coordinator01:7201/api/v1/prom/remote/write + write_relabel_configs: + - target_label: metrics_storage + replacement: m3db_remote diff --git a/src/integration/resources/docker/coordinator.go b/src/integration/resources/docker/coordinator.go index 6bbb18ead2..4b822209fc 100644 --- a/src/integration/resources/docker/coordinator.go +++ b/src/integration/resources/docker/coordinator.go @@ -58,7 +58,7 @@ func newDockerHTTPCoordinator( opts dockerResourceOptions, ) (resources.Coordinator, error) { opts = opts.withDefaults(defaultCoordinatorOptions) - opts.mounts = []string{"/etc/m3coordinator/"} + opts.tmpfsMounts = []string{"/etc/m3coordinator/"} resource, err := newDockerResource(pool, opts) if err != nil { diff --git a/src/integration/resources/docker/docker_resource.go b/src/integration/resources/docker/docker_resource.go index 1cf964557e..1f026d248e 100644 --- a/src/integration/resources/docker/docker_resource.go +++ b/src/integration/resources/docker/docker_resource.go @@ -65,9 +65,10 @@ func newDockerResource( opts := exposePorts(newOptions(containerName), portList) hostConfigOpts := func(c *dc.HostConfig) { + c.AutoRemove = true c.NetworkMode = networkName - mounts := make([]dc.HostMount, 0, len(resourceOpts.mounts)) - for _, m := range resourceOpts.mounts { + mounts := make([]dc.HostMount, 0, len(resourceOpts.tmpfsMounts)) + for _, m := range resourceOpts.tmpfsMounts { mounts = append(mounts, dc.HostMount{ Target: m, Type: string(mount.TypeTmpfs), @@ -89,6 +90,7 @@ func newDockerResource( } } else { opts = useImage(opts, image) + opts.Mounts = resourceOpts.mounts imageWithTag := fmt.Sprintf("%v:%v", image.name, image.tag) logger.Info("running container with options", zap.String("image", imageWithTag), zap.Any("options", opts)) diff --git a/src/integration/resources/docker/prometheus.go b/src/integration/resources/docker/prometheus.go new file mode 100644 index 0000000000..80e5ac0fec --- /dev/null +++ b/src/integration/resources/docker/prometheus.go @@ -0,0 +1,110 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package docker + +import ( + "errors" + "fmt" + + "github.com/ory/dockertest/v3" + + "github.com/m3db/m3/src/integration/resources" + "github.com/m3db/m3/src/x/instrument" +) + +type prometheus struct { + pool *dockertest.Pool + pathToCfg string + iOpts instrument.Options + + resource *dockerResource +} + +// PrometheusOptions contains the options for +// spinning up docker container running Prometheus +type PrometheusOptions struct { + // Pool is the connection to the docker API + Pool *dockertest.Pool + // PathToCfg contains the path to the prometheus.yml configuration + // file to be used on startup. + PathToCfg string + // InstrumentOptions are the instrument.Options to use when + // creating the resource. + InstrumentOptions instrument.Options +} + +// NewPrometheus creates a new docker-backed Prometheus +// that implements the resources.ExternalResources interface. +func NewPrometheus(opts PrometheusOptions) resources.ExternalResources { + if opts.InstrumentOptions == nil { + opts.InstrumentOptions = instrument.NewOptions() + } + return &prometheus{ + pool: opts.Pool, + pathToCfg: opts.PathToCfg, + iOpts: opts.InstrumentOptions, + } +} + +func (p *prometheus) Setup() error { + if p.resource != nil { + return errors.New("prometheus already setup. must close resource " + + "before attempting to setup again") + } + + if err := setupNetwork(p.pool); err != nil { + return err + } + + res, err := newDockerResource(p.pool, dockerResourceOptions{ + containerName: "prometheus", + image: dockerImage{ + name: "prom/prometheus", + tag: "latest", + }, + portList: []int{9090}, + mounts: []string{ + fmt.Sprintf("%s:/etc/prometheus/prometheus.yml", p.pathToCfg), + }, + iOpts: p.iOpts, + }) + if err != nil { + return err + } + + p.resource = res + + return nil +} + +func (p *prometheus) Close() error { + if p.resource.closed { + return errClosed + } + + if err := p.resource.close(); err != nil { + return err + } + + p.resource = nil + + return nil +} diff --git a/src/integration/resources/docker/prometheus_test.go b/src/integration/resources/docker/prometheus_test.go new file mode 100644 index 0000000000..5e05862867 --- /dev/null +++ b/src/integration/resources/docker/prometheus_test.go @@ -0,0 +1,75 @@ +// +build integration_v2 +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package docker + +import ( + "context" + "errors" + "net/http" + "path" + "runtime" + "testing" + "time" + + "github.com/cenkalti/backoff/v3" + "github.com/ory/dockertest/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewPrometheus(t *testing.T) { + pool, err := dockertest.NewPool("") + require.NoError(t, err) + + _, filename, _, _ := runtime.Caller(0) + prom := NewPrometheus(PrometheusOptions{ + Pool: pool, + PathToCfg: path.Join(path.Dir(filename), "config/prometheus.yml"), + }) + require.NoError(t, prom.Setup()) + defer func() { + assert.NoError(t, prom.Close()) + }() + + bo := backoff.NewConstantBackOff(1 * time.Second) + var ( + attempts = 0 + maxAttempts = 5 + ) + err = backoff.Retry(func() error { + req, err := http.NewRequestWithContext(context.Background(), "GET", "http://0.0.0.0:9090/-/ready", nil) + require.NoError(t, err) + + client := http.Client{} + res, _ := client.Do(req) + if res != nil && res.StatusCode == 200 { + return nil + } + + attempts++ + if attempts >= maxAttempts { + return backoff.Permanent(errors.New("prometheus not ready")) + } + return errors.New("retryable error") + }, bo) + require.NoError(t, err) +} diff --git a/src/integration/resources/types.go b/src/integration/resources/types.go index a97617ccf6..69649b90dc 100644 --- a/src/integration/resources/types.go +++ b/src/integration/resources/types.go @@ -152,6 +152,19 @@ type M3Resources interface { Coordinator() Coordinator } +// ExternalResources represents an external (i.e. non-M3) +// resource that we'd like to be able to spin up for an +// integration test. +type ExternalResources interface { + // Setup sets up the external resource so that it's ready + // for use. + Setup() error + + // Close stops and cleans up all the resources associated with + // the external resource. + Close() error +} + // ClusterOptions represents a set of options for a cluster setup. type ClusterOptions struct { ReplicationFactor int32