Dockerize m3aggregator and m3collector and integrate for clustered downsampling

docker/m3aggregator/Dockerfile

@@ -0,0 +1,29 @@
+# stage 1: build
+FROM golang:1.10-alpine AS builder
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+# Install Glide
+RUN apk add --update glide git make bash
+
+# Add source code
+RUN mkdir -p /go/src/github.com/m3db/m3
+ADD . /go/src/github.com/m3db/m3
+
+# Build m3dbnode binary
+RUN cd /go/src/github.com/m3db/m3/ && \
+    git submodule update --init      && \
+    make m3aggregator-linux-amd64
+
+# stage 2: lightweight "release"
+FROM alpine:latest
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+EXPOSE 5000/tcp 6000/tcp 60001/tcp 7203/tcp 9000-9004/tcp
+
+RUN apk add --no-cache curl jq
+
+COPY --from=builder /go/src/github.com/m3db/m3/bin/m3aggregator /bin/
+COPY --from=builder /go/src/github.com/m3db/m3/src/aggregator/config/m3aggregator.yml /etc/m3aggregator/m3aggregator.yml
+
+ENTRYPOINT [ "/bin/m3aggregator" ]
+CMD [ "-f", "/etc/m3aggregator/m3aggregator.yml" ]

docker/m3collector/Dockerfile

@@ -0,0 +1,29 @@
+# stage 1: build
+FROM golang:1.10-alpine AS builder
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+# Install Glide
+RUN apk add --update glide git make bash
+
+# Add source code
+RUN mkdir -p /go/src/github.com/m3db/m3
+ADD . /go/src/github.com/m3db/m3
+
+# Build m3dbnode binary
+RUN cd /go/src/github.com/m3db/m3/ && \
+    git submodule update --init      && \
+    make m3collector-linux-amd64
+
+# stage 2: lightweight "release"
+FROM alpine:latest
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+EXPOSE 7206-7207/tcp
+
+RUN apk add --no-cache curl jq
+
+COPY --from=builder /go/src/github.com/m3db/m3/bin/m3collector /bin/
+COPY --from=builder /go/src/github.com/m3db/m3/src/collector/config/m3collector.yml /etc/m3collector/m3collector.yml
+
+ENTRYPOINT [ "/bin/m3collector" ]
+CMD [ "-f", "/etc/m3collector/m3collector.yml" ]

scripts/development/m3_stack/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       dockerfile: ./docker/m3dbnode/Dockerfile
     image: m3dbnode01:latest
     volumes:
-      - "./dbnode_config.yml:/etc/m3dbnode/m3dbnode.yml"
+      - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
     environment:
       - M3DB_HOST_ID=m3db_seed
   m3db_data01:
@@ -19,7 +19,7 @@ services:
       dockerfile: ./docker/m3dbnode/Dockerfile
     image: m3dbnode02:latest
     volumes:
-      - "./dbnode_config.yml:/etc/m3dbnode/m3dbnode.yml"
+      - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
     environment:
       - M3DB_HOST_ID=m3db_data01
   m3db_data02:
@@ -30,10 +30,21 @@ services:
       dockerfile: ./docker/m3dbnode/Dockerfile
     image: m3dbnode03:latest
     volumes:
-      - "./dbnode_config.yml:/etc/m3dbnode/m3dbnode.yml"
+      - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
     environment:
       - M3DB_HOST_ID=m3db_data02
-  coordinator01:
+  m3aggregator01:
+    networks:
+      - backend
+    build:
+      context: ../../../
+      dockerfile: ./docker/m3aggregator/Dockerfile
+    image: m3aggregator01:latest
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator01
+  m3coordinator01:
     expose:
       - "7201"
       - "7203"
@@ -49,7 +60,22 @@ services:
       dockerfile: ./docker/m3coordinator/Dockerfile
     image: m3coordinator01:latest
     volumes:
-      - "./:/etc/m3coordinator/"
+      - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+  m3collector01:
+    expose:
+      - "7206"
+      - "7207"
+    ports:
+      - "0.0.0.0:7206:7206"
+      - "0.0.0.0:7207:7207"
+    networks:
+      - backend
+    build:
+      context: ../../../
+      dockerfile: ./docker/m3collector/Dockerfile
+    image: m3collector01:latest
+    volumes:
+      - "./m3collector.yml:/etc/m3collector/m3collector.yml"
   prometheus01:
     expose:
       - "9090"
@@ -60,7 +86,7 @@ services:
     image: prom/prometheus:latest
     volumes:
       - "./:/etc/prometheus/"
-  grafana2:
+  grafana:
     build:
       context: ../../../
       dockerfile: ./docker/grafana/Dockerfile

src/aggregator/config/m3aggregator.yaml → scripts/development/m3_stack/m3aggregator.yml

@@ -1,36 +1,20 @@
-# Copyright (c) 2017 Uber Technologies, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
 logging:
   level: info
 
 metrics:
   scope:
-    reportingInterval: 1s
-  m3:
-    hostPort: 127.0.0.1:5000  # local collector host port for m3 metrics
-    service: m3aggregator
-    env: test
-    includeHost: true
-  samplingRate: 0.01
-  extended: moderate
+    prefix: m3aggregator
+  prometheus:
+    onError: none
+    handlerPath: /metrics
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+http:
+  listenAddress: 0.0.0.0:6001
+  readTimeout: 60s
+  writeTimeout: 60s
 
 rawtcp:
   listenAddress: 0.0.0.0:6000
@@ -85,23 +69,35 @@ rawtcp:
         low: 0.001
         high: 0.002
 
-http:
-  listenAddress: 0.0.0.0:6001
-  readTimeout: 45s
-  writeTimeout: 45s
+kvClient:
+  etcd:
+    env: default_env
+    zone: embedded
+    service: m3aggregator
+    cacheDir: /var/lib/m3kv
+    etcdClusters:
+      - zone: embedded
+        endpoints:
+          - m3db_seed:2379
 
 runtimeOptions:
   kvConfig:
-    namespace: /m3aggregator
-    environment: test
+    environment: default_env
+    zone: embedded
   writeValuesPerMetricLimitPerSecondKey: write-values-per-metric-limit-per-second
   writeValuesPerMetricLimitPerSecond: 0
   writeNewMetricLimitClusterPerSecondKey: write-new-metric-limit-cluster-per-second
   writeNewMetricLimitClusterPerSecond: 0
   writeNewMetricNoLimitWarmupDuration: 0
 
 aggregator:
-  metricPrefix: stats.
+  hostID:
+    resolver: environment
+    envVarName: M3AGGREGATOR_HOST_ID
+  metricPrefix: ""
+  counterPrefix: ""
+  timerPrefix: ""
+  gaugePrefix: ""
   aggregationTypes:
     counterTransformFnType: empty
     timerTransformFnType: suffix
@@ -131,14 +127,13 @@ aggregator:
           capacity: 64
   client:
     placementKV:
-      zone: testZone
-      environment: testEnvironment
-      namespace: testNamespace
+      namespace: /placement
+      zone: embedded
+      environment: default_env
     placementWatcher:
-      key: testWatchKey
+      key: m3aggregator
       initWatchTimeout: 15s
     hashType: murmur32
-    shardCutoverWarmupDuration: 10m
     shardCutoffLingerDuration: 1m
     encoder:
       initBufferSize: 100
@@ -166,19 +161,20 @@ aggregator:
       maxReconnectDuration: 1m
   placementManager:
     kvConfig:
-      namespace: /m3aggregator
-      environment: test
+      namespace: /placement
+      environment: default_env
+      zone: embedded
     placementWatcher:
-      key: placement
+      key: m3aggregator
       initWatchTimeout: 10s
   hashType: murmur32
   bufferDurationBeforeShardCutover: 10m
   bufferDurationAfterShardCutoff: 10m
   resignTimeout: 1m
   flushTimesManager:
     kvConfig:
-      namespace: /m3aggregator
-      environment: test
+      environment: default_env
+      zone: embedded
     flushTimesKeyFmt: shardset/%d/flush
     flushTimesPersistRetrier:
       initialBackoff: 100ms
@@ -192,7 +188,8 @@ aggregator:
       ttlSeconds: 10
     serviceID:
       name: m3aggregator
-      environment: test
+      environment: default_env
+      zone: embedded
     electionKeyFmt: shardset/%d/lock
     campaignRetrier:
       initialBackoff: 100ms
@@ -233,19 +230,47 @@ aggregator:
     maxBufferSize: 5m
     forcedFlushWindowSize: 10s
   flush:
+    writer:
+      maxBufferSize: 1440
+      encodingTimeSamplingRate: 0.01
+      bufferedEncoderPool:
+        size: 16384
     handlers:
-      - staticBackend:
-          type: logging
-      - staticBackend:
-          type: blackhole
+      - dynamicBackend:
+          name: m3msg
+          hashType: murmur32
+          totalShards: 64
+          producer:
+            buffer:
+              maxBufferSize: 1000000000 # max buffer before m3msg start dropping data.
+            writer:
+              topicName: aggregated_metrics
+              topicServiceOverride:
+                zone: embedded
+                environment: default_env
+              messageRetry:
+                initialBackoff: 1m
+                maxBackoff: 2m
+              messageQueueNewWritesScanInterval: 1s
+              ackErrorRetry:
+                initialBackoff: 2s
+                maxBackoff: 10s
+              connection:
+                dialTimeout: 5s
+                writeTimeout: 5s
+                retry:
+                  initialBackoff: 1s
+                  maxBackoff: 10s
+                flushInterval: 1s
+                writeBufferSize: 16384
+                readBufferSize: 256
   forwarding:
     maxSingleDelay: 5s
   entryTTL: 6h
   entryCheckInterval: 10m
   maxTimerBatchSizePerWrite: 140
   defaultStoragePolicies:
     - 10s:2d
-    - 1m:40d
   maxNumCachedSourceSets: 2
   discardNaNAggregatedValues: true
   entryPool: