Add instruction and example to run dockerized awsbeats w/ Kinesis Dat…

…a Streams

This should be enhanced to cover firehose and other beats. This is just a starting point :)

.gitignore

@@ -17,4 +17,4 @@ coverage.txt
 vendor
 target
 .vagrant
-example/*
+example/firehose/log/*

Makefile

@@ -1,8 +1,12 @@
 .PHONY: all
 
-GO_VERSION=$(shell go version | cut -d ' ' -f 3,4 | sed -e 's/ /-/g' | sed -e 's/\//-/g')
+GO_VERSION=$(shell go version | cut -d ' ' -f 3 | sed -e 's/ /-/g' | sed -e 's/\//-/g' | sed -e 's/^go//g')
+GO_PLATFORM ?= $(go version | cut -d ' ' -f 4 | sed -e 's/ /-/g' | sed -e 's/\//-/g')
 BEATS_VERSION ?= "master"
+BEATS_TAG ?= $(shell echo ${BEATS_VERSION} | sed 's/\([[:digit:]]*\(\.[[:digit:]]*\)\)/v\1/')
 AWSBEATS_VERSION ?= "1-snapshot"
+DOCKER_IMAGE ?= s12v/awsbeats
+DOCKER_TAG ?= canary
 
 all: test beats build
 
@@ -16,18 +20,27 @@ format:
 	test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -w {} + | tee /dev/stderr)"
 
 build: format
+	@echo "Building the plugin with: BEATS_VERSION=$(BEATS_VERSION) GO_VERSION=$(GO_VERSION) GO_PLATFORM=$(GO_PLATFORM)"
+	@cd "$$GOPATH/src/github.com/elastic/beats/filebeat" && \
+	git checkout $(BEATS_TAG) && \
+	cd "$(CURDIR)"
 	go build -buildmode=plugin ./plugins/kinesis
 	@mkdir -p "$(CURDIR)/target"
-	@mv kinesis.so "$(CURDIR)/target/kinesis-$(AWSBEATS_VERSION)-$(BEATS_VERSION)-$(GO_VERSION).so"
+	@mv kinesis.so "$(CURDIR)/target/kinesis-$(AWSBEATS_VERSION)-$(BEATS_VERSION)-go$(GO_VERSION)-$(GO_PLATFORM).so"
+	@find "$(CURDIR)"/target/
 
 beats:
 ifdef BEATS_VERSION
 	@echo "Building filebeats:$(BEATS_VERSION)..."
 	@mkdir -p "$(CURDIR)/target"
 	@cd "$$GOPATH/src/github.com/elastic/beats/filebeat" &&\
-	git checkout $(BEATS_VERSION) &&\
+	git checkout $(BEATS_TAG) &&\
 	make &&\
-	mv filebeat "$(CURDIR)/target/filebeat-$(BEATS_VERSION)-$(GO_VERSION)"
+	mv filebeat "$(CURDIR)/target/filebeat-$(BEATS_VERSION)-go$(GO_VERSION)-$(GO_PLATFORM)"
 else
 	$(error BEATS_VERSION is undefined)
 endif
+
+.PHONY: dockerimage
+dockerimage:
+	docker build --build-arg AWSBEATS_VERSION=$(AWSBEATS_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BEATS_VERSION=$(BEATS_VERSION) -t $(DOCKER_IMAGE):$(DOCKER_TAG) .

README.md

@@ -55,6 +55,31 @@ filebeat-v6.1.3-go1.10rc1-linux-amd64
 kinesis.so-1-snapshot-v6.1.3-go1.10rc1-linux-amd64
 ```
 
+## Running in a docker container
+
+To build a docker image for awsbeats, run:
+
+```
+make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 GOPATH=$HOME/go
+```
+
+The resulting docker image is tagged `s12v/awsbeats:canary`.  It contains a custom build of filebeat and the plugin, along with all the relevant files from the official filebeat docker image.
+
+To try running it, provide AWS credentials via e.g. envvars and run `hack/dockerized-filebeat`:
+
+```
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+
+hack/containerized-filebeat
+```
+
+Emit some line-delimited json log messages:
+
+```
+hack/emit-ndjson-logs
+```
+
 ## Output buffering
 
 TODO

example/streams/filebeat.yml

@@ -0,0 +1,28 @@
+#filebeat.config:
+#  prospectors:
+#    path: ${path.config}/prospectors.d/*.yml
+#    reload.enabled: false
+#  modules:
+#    path: ${path.config}/modules.d/*.yml
+#    reload.enabled: false
+
+filebeat.prospectors:
+  - input_type: log
+    paths:
+    - /mnt/log/*.log
+    json.keys_under_root: true
+    json.add_error_key: true
+
+processors:
+- add_cloud_metadata:
+
+output:
+  streams:
+    region: ap-northeast-1
+    stream_name: test1
+    partition_key: mykey
+
+queue.mem:
+  events: 4096
+  flush.min_events: 5
+  flush.timeout: 3s

hack/containerized-filebeat

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+rm -rf logs/*
+
+for stream in foo; do
+  for i in $(seq 5); do
+    echo ${stream}{i} >> ${stream}.log
+  done
+done
+
+docker run \
+  --rm \
+  -v $(pwd)/logs:/mnt/log/ \
+  -v $(pwd)/filebeat.yml:/etc/filebeat/filebeat.yml \
+  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+  s12v/awsbeats:canary \
+  filebeat \
+  --plugin kinesis.so \
+  -e \
+  -d='*' \
+  -c /etc/filebeat/filebeat.yml \
+  --strict.perms=false
+
+# Note that `strict.perms` seems to be required due to https://discuss.elastic.co/t/volume-mapped-filebeat-yml-permissions-from-docker-on-windows-host/91893/2
+
+# In another terminal, emit some log messages:
+#
+#  for i in $(seq 5); do echo '{"mykey":"foo'${i}'"}' >> logs/foo.log; done
+

hack/emit-ndjson-logs

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+for i in $(seq 5); do
+  echo '{"mykey":"mykey4","myvalue":"'$i'"}' | tee -a logs/foo.log
+done