Add Graphite ingestion

Makefile

@@ -200,6 +200,8 @@ docker-integration-test:
 	@./scripts/docker-integration-tests/setup.sh
 	@./scripts/docker-integration-tests/simple/test.sh
 	@./scripts/docker-integration-tests/prometheus/test.sh
+	# TODO(rartoul): Re-enable once the query P.R lands and we can fix this test.
+	# @./scripts/docker-integration-tests/carbon/test.sh
 
 .PHONY: site-build
 site-build:

scripts/development/m3_stack/docker-compose.yml

@@ -48,10 +48,12 @@ services:
     expose:
       - "7201"
       - "7203"
+      - "7204"
       - "7507"
     ports:
       - "0.0.0.0:7201:7201"
       - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
       - "0.0.0.0:7507:7507"
     networks:
       - backend

scripts/development/m3_stack/m3coordinator.yml

@@ -67,3 +67,6 @@ ingest:
         jitter: true
     handler:
       protobufEnabled: false
+
+carbon:
+  enabled: true

scripts/development/m3_stack/start_m3.sh

@@ -9,7 +9,7 @@ if [[ "$FORCE_BUILD" = true ]] ; then
     DOCKER_ARGS="--build -d --renew-anon-volumes"
 fi
 
-echo "Bringing up nodes in the backgorund with docker compose, remember to run ./stop.sh when done"
+echo "Bringing up nodes in the background with docker compose, remember to run ./stop.sh when done"
 docker-compose -f docker-compose.yml up $DOCKER_ARGS m3coordinator01
 docker-compose -f docker-compose.yml up $DOCKER_ARGS m3db_seed
 docker-compose -f docker-compose.yml up $DOCKER_ARGS prometheus01

scripts/docker-integration-tests/carbon/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.5"
+services:
+  dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9000-9004:9000-9004"
+      - "0.0.0.0:2379-2380:2379-2380"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+  coordinator01:
+    expose:
+      - "7201"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:7201:7201"
+      - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./:/etc/m3coordinator/"
+networks:
+  backend:

scripts/docker-integration-tests/carbon/m3coordinator.yml

@@ -0,0 +1,54 @@
+listenAddress:
+  type: "config"
+  value: "0.0.0.0:7201"
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 15s
+      - namespace: unagg
+        type: unaggregated
+        retention: 10m
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - dbnode01:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+      writeTimeout: 10s
+      fetchTimeout: 15s
+      connectTimeout: 20s
+      writeRetry:
+        initialBackoff: 500ms
+        backoffFactor: 3
+        maxRetries: 2
+        jitter: true
+      fetchRetry:
+        initialBackoff: 500ms
+        backoffFactor: 2
+        maxRetries: 3
+        jitter: true
+      backgroundHealthCheckFailLimit: 4
+      backgroundHealthCheckFailThrottleFactor: 0.5
+
+carbon:
+  enabled: true

scripts/docker-integration-tests/carbon/test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -xe
+
+source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/common.sh
+REVISION=$(git rev-parse HEAD)
+COMPOSE_FILE=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/carbon/docker-compose.yml
+export REVISION
+
+echo "Run m3dbnode and m3coordinator containers"
+docker-compose -f ${COMPOSE_FILE} up -d dbnode01
+docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+
+# think of this as a defer func() in golang
+function defer {
+  docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
+}
+trap defer EXIT
+
+setup_single_m3db_node
+
+echo "Writing out a carbon metric"
+echo "foo.bar.baz 1 `date +%s`" | nc 0.0.0.0 7204
+
+echo "Attempting to read carbon metric back"
+function read_carbon {
+  end=$(date +%s)
+  start=$(($end-300))
+  RESPONSE=$(curl -sSfg "http://localhost:7201/api/v1/query_range?start=$start&end=$end&step=10&query={__graphite0__='foo',__graphite1__='bar',__graphite2__='baz'}")
+  echo "$RESPONSE" | jq '.data.result[0].values[][1]=="1"' | grep -q "true"
+  return $?
+}
+ATTEMPTS=10 TIMEOUT=1 retry_with_backoff read_carbon

scripts/docker-integration-tests/common.sh

@@ -36,3 +36,99 @@ function retry_with_backoff {
 
   return $exitCode
 }
+
+function setup_single_m3db_node {
+  wait_for_db_init
+}
+
+function wait_for_db_init {
+  echo "Sleeping for a bit to ensure db up"
+  sleep 15 # TODO Replace sleeps with logic to determine when to proceed
+
+  echo "Adding namespace"
+  curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
+    "name": "agg",
+    "options": {
+      "bootstrapEnabled": true,
+      "flushEnabled": true,
+      "writesToCommitLog": true,
+      "cleanupEnabled": true,
+      "snapshotEnabled": true,
+      "repairEnabled": false,
+      "retentionOptions": {
+        "retentionPeriodDuration": "48h",
+        "blockSizeDuration": "2h",
+        "bufferFutureDuration": "10m",
+        "bufferPastDuration": "10m",
+        "blockDataExpiry": true,
+        "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
+      },
+      "indexOptions": {
+        "enabled": true,
+        "blockSizeDuration": "2h"
+      }
+    }
+  }'
+
+  echo "Sleep until namespace is init'd"
+  ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]'
+
+  curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
+    "name": "unagg",
+    "options": {
+      "bootstrapEnabled": true,
+      "flushEnabled": true,
+      "writesToCommitLog": true,
+      "cleanupEnabled": true,
+      "snapshotEnabled": true,
+      "repairEnabled": false,
+      "retentionOptions": {
+        "retentionPeriodDuration": "48h",
+        "blockSizeDuration": "2h",
+        "bufferFutureDuration": "10m",
+        "bufferPastDuration": "10m",
+        "blockDataExpiry": true,
+        "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
+      },
+      "indexOptions": {
+        "enabled": true,
+        "blockSizeDuration": "2h"
+      }
+    }
+  }'
+
+  echo "Sleep until namespace is init'd"
+  ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'
+
+  echo "Placement initialization"
+  curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{
+      "num_shards": 64,
+      "replication_factor": 1,
+      "instances": [
+          {
+              "id": "m3db_local",
+              "isolation_group": "rack-a",
+              "zone": "embedded",
+              "weight": 1024,
+              "endpoint": "dbnode01:9000",
+              "hostname": "dbnode01",
+              "port": 9000
+          }
+      ]
+  }'
+
+  echo "Sleep until placement is init'd"
+  ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]'
+
+  echo "Sleep until bootstrapped"
+  ATTEMPTS=10 TIMEOUT=2 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:9002/health | jq .bootstrapped)" == true ]'
+
+  echo "Waiting until shards are marked as available"
+  ATTEMPTS=10 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]'
+}
+

scripts/docker-integration-tests/prometheus/test.sh

@@ -17,95 +17,7 @@ function defer {
 }
 trap defer EXIT
 
-
-echo "Sleeping for a bit to ensure db up"
-sleep 15 # TODO Replace sleeps with logic to determine when to proceed
-
-echo "Adding namespace"
-curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
-  "name": "agg",
-  "options": {
-    "bootstrapEnabled": true,
-    "flushEnabled": true,
-    "writesToCommitLog": true,
-    "cleanupEnabled": true,
-    "snapshotEnabled": true,
-    "repairEnabled": false,
-    "retentionOptions": {
-      "retentionPeriodNanos": 172800000000000,
-      "blockSizeNanos": 7200000000000,
-      "bufferFutureNanos": 600000000000,
-      "bufferPastNanos": 600000000000,
-      "blockDataExpiry": true,
-      "blockDataExpiryAfterNotAccessPeriodNanos": 300000000000
-    },
-    "indexOptions": {
-      "enabled": true,
-      "blockSizeNanos": 7200000000000
-    }
-  }
-}'
-
-echo "Sleep until namespace is init'd"
-ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]'
-
-curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
-  "name": "unagg",
-  "options": {
-    "bootstrapEnabled": true,
-    "flushEnabled": true,
-    "writesToCommitLog": true,
-    "cleanupEnabled": true,
-    "snapshotEnabled": true,
-    "repairEnabled": false,
-    "retentionOptions": {
-      "retentionPeriodNanos": 172800000000000,
-      "blockSizeNanos": 7200000000000,
-      "bufferFutureNanos": 600000000000,
-      "bufferPastNanos": 600000000000,
-      "blockDataExpiry": true,
-      "blockDataExpiryAfterNotAccessPeriodNanos": 300000000000
-    },
-    "indexOptions": {
-      "enabled": true,
-      "blockSizeNanos": 7200000000000
-    }
-  }
-}'
-
-echo "Sleep until namespace is init'd"
-ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'
-
-echo "Placement initialization"
-curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{
-    "num_shards": 64,
-    "replication_factor": 1,
-    "instances": [
-        {
-            "id": "m3db_local",
-            "isolation_group": "rack-a",
-            "zone": "embedded",
-            "weight": 1024,
-            "endpoint": "dbnode01:9000",
-            "hostname": "dbnode01",
-            "port": 9000
-        }
-    ]
-}'
-
-echo "Sleep until placement is init'd"
-ATTEMPTS=4 TIMEOUT=1 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]'
-
-echo "Sleep until bootstrapped"
-ATTEMPTS=10 TIMEOUT=2 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:9002/health | jq .bootstrapped)" == true ]'
-
-echo "Waiting until shards are marked as available"
-ATTEMPTS=10 TIMEOUT=1 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]'
+setup_single_m3db_node
 
 echo "Start Prometheus containers"
 docker-compose -f ${COMPOSE_FILE} up -d prometheus01