prepare 5.9.0 release

.circleci/config.yml

@@ -54,6 +54,9 @@ defaults:
               go tool cover -html=$CIRCLE_ARTIFACTS/coverage.txt -o $CIRCLE_ARTIFACTS/coverage.html
             fi
           when: always
+      - run:
+          name: Run Prometheus endpoint test
+          command: ./scripts/test-prometheus-endpoint.sh
       - run:
           name: lint
           command: |

CHANGELOG.md

@@ -3,10 +3,11 @@
 All notable changes to the LaunchDarkly Relay will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org).
 
 ## [5.8.2] - 2019-11-07
+### Changed:
+- Updated to a newer version of the OpenCensus Prometheus exporter. (Thanks, [mightyguava](https://github.com/launchdarkly/ld-relay/pull/70!))
 ### Fixed:
 - When using a persistent feature store (Redis, etc.), if multiple clients request flags at the same time when the flag data is not in the cache, Relay will coalesce these requests so only a single database query is done.
 
-
 ## [5.8.1] - 2019-11-05
 ### Fixed:
 - The `README` incorrectly referred to an environment variable as `DYNAMODB_ENABLED` when it is really `USE_DYNAMODB`. (Thanks, [estraph](https://github.com/launchdarkly/ld-relay/pull/79)!)

README.md

@@ -120,8 +120,11 @@ Property in file    | Environment var    | Type    | Default | Description
 ------------------- | ------------------ | :-----: | :------ | -----------
 `enabled`           | `USE_DYNAMODB`     | Boolean | `false` | Enables DynamoDB.
 `tableName`         | `DYNAMODB_TABLE`   | String  |         | The DynamoDB table name, if you are using the same table for all environments. Otherwise, omit this and specify it in each environment section. (Note, credentials and region are controlled by the usual AWS environment variables and/or local AWS configuration files.)
+`url`               | `DYNAMODB_URL`     | String  |         | The service endpoint if you are using a local DynamoDB instance instead of the regular service.
 `localTtl`          | `CACHE_TTL`        | Number  | `30000`     | Length of time (in milliseconds) that database items can be cached in memory.
 
+The AWS credentials and region for DynamoDB are not part of the Relay configuration; they should be set using either the standard AWS environment variables or a local AWS configuration file, as documented for [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html).
+
 ### File section: `[Consul]`
 
 Property in file | Environment var | Type    | Default     | Description
@@ -211,8 +214,8 @@ Property in file | Environment var          | Type    | Default | Description
 Property in file | Environment var     | Type    | Default | Description
 ---------------- | ------------------- | :-----: | :------ | -----------
 `enabled`        | `USE_PROMETHEUS`    | Boolean | `false` | If true, enables exporting traces to Prometheus.
-`port`           | `PROMETHEUS_PORT`   | Number  | `8031`  | The port that the Relay Proxy will listen to `/metrics` on.
-`prefix`         | `PROMETHEUS_PREFIX` | String  |         |  The metrics prefix to be used by Prometheus.
+`port`           | `PROMETHEUS_PORT`   | Number  | `8031`  | The port that the Relay Proxy will provide the `/metrics` endpoint on.
+`prefix`         | `PROMETHEUS_PREFIX` | String  |         | The metrics prefix to be used by Prometheus.
 
 ### File section: `[Proxy]`
 

config.go

@@ -99,6 +99,7 @@ type ConsulConfig struct {
 type DynamoDBConfig struct {
 	Enabled   bool
 	TableName string
+	Url       string
 	LocalTtl  int
 }
 
@@ -227,7 +228,7 @@ var DefaultConfig = Config{
 		BaseUri:               defaultBaseURI,
 		StreamUri:             defaultStreamURI,
 		HeartbeatIntervalSecs: defaultHeartbeatIntervalSecs,
-		Port: defaultPort,
+		Port:                  defaultPort,
 	},
 	Events: events.Config{
 		Capacity:          defaultEventCapacity,
@@ -402,6 +403,7 @@ func LoadConfigFromEnvironment(c *Config) error {
 	maybeSetFromEnvBool(&c.DynamoDB.Enabled, "USE_DYNAMODB")
 	if c.DynamoDB.Enabled {
 		maybeSetFromEnv(&c.DynamoDB.TableName, "DYNAMODB_TABLE")
+		maybeSetFromEnv(&c.DynamoDB.Url, "DYNAMODB_URL")
 		maybeSetFromEnvInt(&c.DynamoDB.LocalTtl, "CACHE_TTL", &errs)
 	}
 

config_test.go

@@ -266,12 +266,14 @@ func TestConfigFromEnvironmentWithDynamoDB(t *testing.T) {
 			c.DynamoDB = DynamoDBConfig{
 				Enabled:   true,
 				TableName: "table",
+				Url:       "http://localhost:8000",
 				LocalTtl:  3000,
 			}
 		},
 		map[string]string{
 			"USE_DYNAMODB":   "1",
 			"DYNAMODB_TABLE": "table",
+			"DYNAMODB_URL":   "http://localhost:8000",
 			"CACHE_TTL":      "3000",
 		},
 	)

internal/metrics/prometheus.go

@@ -21,7 +21,11 @@ func registerPrometheusExporter(options ExporterOptions) error {
 		port = o.Port
 	}
 
-	exporter, err := prometheus.NewExporter(prometheus.Options{Namespace: o.Prefix})
+	logPrometheusError := func(e error) {
+		logging.Error.Printf("Prometheus exporter error: %s", e)
+	}
+
+	exporter, err := prometheus.NewExporter(prometheus.Options{Namespace: o.Prefix, OnError: logPrometheusError})
 	if err != nil {
 		return err
 	}

relay.go

@@ -14,6 +14,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	redigo "github.com/garyburd/redigo/redis"
 	"github.com/gorilla/mux"
 	"github.com/gregjones/httpcache"
@@ -538,8 +540,20 @@ func createFeatureStore(c Config, envConfig *EnvConfig) (ld.FeatureStoreFactory,
 			return nil, errors.New("TableName property must be specified for DynamoDB, either globally or per environment")
 		}
 		logging.Info.Printf("Using DynamoDB feature store: %s with prefix: %s", tableName, envConfig.Prefix)
-		return lddynamodb.NewDynamoDBFeatureStoreFactory(tableName, lddynamodb.Prefix(envConfig.Prefix),
-			lddynamodb.CacheTTL(time.Duration(c.DynamoDB.LocalTtl)*time.Millisecond), lddynamodb.Logger(logging.Info))
+		options := []lddynamodb.FeatureStoreOption{
+			lddynamodb.Prefix(envConfig.Prefix),
+			lddynamodb.CacheTTL(time.Duration(c.DynamoDB.LocalTtl) * time.Millisecond),
+			lddynamodb.Logger(logging.Info),
+		}
+		if c.DynamoDB.Url != "" {
+			awsOptions := session.Options{
+				Config: aws.Config{
+					Endpoint: aws.String(c.DynamoDB.Url),
+				},
+			}
+			options = append(options, lddynamodb.SessionOptions(awsOptions))
+		}
+		return lddynamodb.NewDynamoDBFeatureStoreFactory(tableName, options...)
 	}
 	return ld.NewInMemoryFeatureStoreFactory(), nil
 }

scripts/test-prometheus-endpoint.sh

@@ -0,0 +1,105 @@
+# !/bin/bash
+
+# This script performs a smoke test for Relay data export to Prometheus. It does not run Prometheus,
+# but starts Relay with Prometheus export enabled and then verifies that a metric is available from
+# Relay's /metrics endpoint.
+
+set -e
+
+FAKE_LD_PORT=8100
+RELAY_PORT=8101
+RELAY_METRICS_PORT=8102
+
+# Start a fake LaunchDarkly service endpoint that simply returns a single empty stream put event -
+# that's enough to ensure that the SDK client can initialize. The nc command will close the socket
+# after responding, so the SDK will go into "reconnecting" mode, but that shouldn't affect our test.
+TEMP_FILE_STREAM_DATA=$(mktemp)
+cat >${TEMP_FILE_STREAM_DATA} <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/event-stream
+
+event: put
+data: {"data":{}}
+
+
+EOF
+
+if [ "$OSTYPE" == "linux-gnu" ]; then
+  nc -l -p ${FAKE_LD_PORT} <"${TEMP_FILE_STREAM_DATA}" >/dev/null &
+else
+  nc -l ${FAKE_LD_PORT} <"${TEMP_FILE_STREAM_DATA}" >/dev/null &
+fi
+
+go build ./cmd/ld-relay
+
+# start Relay as a background process
+echo "starting Relay"
+PORT=${RELAY_PORT} \
+  STREAM_URI=http://localhost:${FAKE_LD_PORT} \
+  LD_LOG_LEVEL="debug" \
+  LD_LOG_LEVEL_test="none" \
+  LD_ENV_test="fake-sdk-key" \
+  LD_CLIENT_SIDE_ID_test="fake-env-id" \
+  USE_PROMETHEUS=1 \
+  PROMETHEUS_PORT=${RELAY_METRICS_PORT} \
+  ./ld-relay --from-env &
+RELAY_PID=$!
+trap "kill ${RELAY_PID}" exit
+
+# wait for relay to start
+echo
+echo "waiting for Relay /status endpoint"
+fail_count=0
+max_attempts=10
+while true; do
+  curl --silent --show-error --fail http://localhost:${RELAY_PORT}/status >/dev/null && break
+  let "fail_count += 1"
+  if [[ ${fail_count} -gt ${max_attempts} ]]; then
+    echo "Relay did not start after ${max_attempts} seconds; failing"
+    exit 1
+  fi
+  echo "...Relay not ready, retrying"
+  sleep 1
+done
+echo "Relay started"
+
+# make an SDK endpoint request, causing request metric to be incremented (we don't care about the output)
+echo
+echo "querying Relay SDK endpoint to generate a metric"
+fail_count=0
+max_attempts=10
+while true; do
+  curl --fail --silent -X REPORT -H 'content-type:application/json' --data '{"key":"test-user"}' \
+    http://localhost:${RELAY_PORT}/sdk/evalx/fake-env-id/user >/dev/null && break
+  let "fail_count += 1"
+  if [[ ${fail_count} -gt ${max_attempts} ]]; then
+    echo "SDK client did not start after ${max_attempts} seconds; failing"
+    exit 1
+  fi
+  echo "...client not ready, retrying"
+  sleep 1
+done
+echo "OK"
+
+# hit the Prometheus exporter endpoint - allow a couple of retries since there can be a lag for the data
+echo
+echo "querying Relay /metrics endpoint"
+TEMP_FILE_METRICS=$(mktemp)
+fail_count=0
+max_attempts=10
+while true; do
+  curl --fail --silent http://localhost:${RELAY_METRICS_PORT}/metrics >${TEMP_FILE_METRICS} # endpoint should return 200 even if there's no data
+  grep 'env="test",method="REPORT",platformCategory="browser",route="_sdk_evalx_{envId}_user"' <${TEMP_FILE_METRICS} && break
+  let "fail_count += 1"
+  if [[ $fail_count -gt ${max_attempts} ]]; then
+    echo "metrics did not show up after ${max_attempts} seconds; failing"
+    echo
+    echo "last output from metrics endpoint was:"
+    cat "${TEMP_FILE_METRICS}"
+    exit 1
+  fi
+  echo "...got no data, retrying"
+  sleep 1
+done
+echo "got metrics data from endpoint:"
+cat "${TEMP_FILE_METRICS}"