feat(docker): Heartbeat + Awsbeats

Ref s12v#30

Dockerfile

@@ -19,14 +19,13 @@ ARG BEATS_VERSION=${BEATS_VERSION:-6.1.2}
 ARG GO_PLATFORM=${GO_PLATFORM:-linux-amd64}
 ARG AWSBEATS_VERSION=${AWSBEATS_VERSION:-1-snapshot}
 ARG BEAT_NAME=${BEAT_NAME:-filebeat}
-RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+RUN curl --verbose --fail https://raw.githubusercontent.com/golang/dep/master/install.sh -o install.sh && sh install.sh && rm install.sh
 RUN go get github.com/elastic/beats || true
 RUN /go/bin/dep ensure
 # You need to enable CGO on both the plugin and the beat.
 # Otherwise, for example, filebeat w/ CGO fails to load the plugin w/o CGO, emitting an error like:
 #   Exiting: plugin.Open("kinesis"): plugin was built with a different version of package net
 RUN CGO_ENABLED=1 GOOS=linux make build
-
 FROM golang:${GO_VERSION} AS beats
 
 LABEL maintainr "Yusuke KUOKA <[email protected]>"

Makefile

@@ -86,6 +86,10 @@ auditbeat-image:
 filebeat-image:
 	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=filebeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'
 
+.PHONY: heartbeat-image
+heartbeat-image:
+	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=heartbeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'
+
 .PHONY: metricbeat-image
 metricbeat-image:
 	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=metricbeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'

README.md

@@ -122,6 +122,14 @@ make auditbeat-image
 hack/containerized-auditbeat
 ```
 
+### heartbeat
+
+```
+make heartbeat-image
+
+hack/containerized-heartbeat
+```
+
 ## Running awsbeats on a Kubernetes cluster
 
 ### Filebeat

example/heartbeat/heartbeat.yml

@@ -0,0 +1,41 @@
+heartbeat.monitors:
+- type: icmp
+  schedule: '*/5 * * * * * *'
+  hosts: ["localhost"]
+  ipv4: true
+  timeout: 16s
+  wait: 1s
+- type: tcp
+  schedule: '@every 5s'
+  hosts: ["myhost:7"]  # default TCP Echo Protocol
+  check.send: "Check"
+  check.receive: "Check"
+- type: http
+  schedule: '@every 5s'
+  urls: ["http://localhost:80/service/status"]
+  check.response.status: 200
+
+heartbeat.scheduler:
+  limit: 10
+
+processors:
+- add_cloud_metadata:
+# Match originating pod enrich apm events with metadata from Kubernetes
+# See https://github.com/elastic/apm-server/issues/349 for more details
+#- add_kubernetes_metadata:
+#    indexers:
+#      - ip_port:
+#    matchers:
+#      - fields:
+#          lookup_fields: ["context.system.ip"]
+
+output:
+  streams:
+    region: ap-northeast-1
+    stream_name: kuokatest1
+    partition_key_provider: xid
+
+queue.mem:
+  events: 4096
+  flush.min_events: 5
+  flush.timeout: 3s

hack/containerized-heartbeat

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+docker run \
+  --rm \
+  -v $(pwd)/logs:/mnt/log/  \
+  -v $(pwd)/example/heartbeat/data:/var/lib/beat \
+  -v $(pwd)/example/heartbeat:/etc/heartbeat \
+  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+  s12v/awsbeats:heartbeat-canary \
+  heartbeat \
+  --plugin kinesis.so \
+  -e \
+  -d '*' \
+  -c /etc/heartbeat/heartbeat.yml \
+  --strict.perms=false
+
+# Note that `strict.perms` seems to be required due to https://discuss.elastic.co/t/volume-mapped-filebeat-yml-permissions-from-docker-on-windows-host/91893/2
+
+# --pid=host, AUDIT_CONTROL, and AUDIT_READ are reequired in order to avoid the following startup error
+# 2018-06-13T03:23:30.026ZDEBUG[file_integrity]file_integrity/metricset.go:86Initialized the file event reader. Running as euid=0
+# 2018-06-13T03:23:30.026ZERRORinstance/beat.go:667Exiting: 1 error: 1 error: failed to create audit client: failed to get audit status: operation not permitted
+# Exiting: 1 error: 1 error: failed to create audit client: failed to get audit status: operation not permitted
+#
+# See https://github.com/elastic/beats-docker/issues/25 for more info