Instructions to deploy Auditbeat, Heartbeat, APM Server on Kubernetes (…

…#36)

* feat(docker): Heartbeat + Awsbeats

Ref #30

* Deploy APM Server + Awsbeats to Kubernetes

Resolves #34

* Deploy Heartbeat + Awsbeats to Kubernetes

Resolves #30

* Deploy Auditbeat + Awsbeats to Kubernetes

* feat(docker): Metricbeat + Awsbeats

* Cleans up the README part about building docker images

.gitignore

@@ -18,3 +18,5 @@ vendor
 target
 .vagrant
 example/firehose/log/*
+
+*~

Dockerfile

@@ -19,14 +19,13 @@ ARG BEATS_VERSION=${BEATS_VERSION:-6.1.2}
 ARG GO_PLATFORM=${GO_PLATFORM:-linux-amd64}
 ARG AWSBEATS_VERSION=${AWSBEATS_VERSION:-1-snapshot}
 ARG BEAT_NAME=${BEAT_NAME:-filebeat}
-RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+RUN curl --verbose --fail https://raw.githubusercontent.com/golang/dep/master/install.sh -o install.sh && sh install.sh && rm install.sh
 RUN go get github.com/elastic/beats || true
 RUN /go/bin/dep ensure
 # You need to enable CGO on both the plugin and the beat.
 # Otherwise, for example, filebeat w/ CGO fails to load the plugin w/o CGO, emitting an error like:
 #   Exiting: plugin.Open("kinesis"): plugin was built with a different version of package net
 RUN CGO_ENABLED=1 GOOS=linux make build
-
 FROM golang:${GO_VERSION} AS beats
 
 LABEL maintainr "Yusuke KUOKA <[email protected]>"

Makefile

@@ -86,6 +86,10 @@ auditbeat-image:
 filebeat-image:
 	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=filebeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'
 
+.PHONY: heartbeat-image
+heartbeat-image:
+	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=heartbeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'
+
 .PHONY: metricbeat-image
 metricbeat-image:
 	@bash -c 'make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=metricbeat AWSBEATS_VERSION=$(ref=$(git rev-parse HEAD); ref=${ref:0:7}; echo $ref) GOPATH=$HOME/go'

README.md

@@ -4,7 +4,7 @@
 # AWS Beats
 
 Experimental [Beat](https://github.com/elastic/beats) output plugin.
-Tested with Filebeat and Metricbeat. Supports AWS Kinesis Data Streams and Data Firehose.
+Tested with Filebeat, Metricbeat, Auditbeat, Heartbeat, APM Server. Supports AWS Kinesis Data Streams and Data Firehose.
 
 __NOTE: Beat and the plugin should be built using the same Golang version.__
 
@@ -68,7 +68,7 @@ To build a docker image for awsbeats, run `make dockerimage`.
 ### filebeat
 
 ```
-make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 GOPATH=$HOME/go
+make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=filebeat GOPATH=$HOME/go
 ```
 
 There is also a convenient make target `filebeat-image` with sane defaults:
@@ -96,14 +96,10 @@ hack/emit-ndjson-logs
 
 ### metricbeat
 
-**metricbeat**:
-
 ```
-make dockerimage BEATS_VERSION=6.2.4 GO_VERSION=1.10.2 BEAT_NAME=metricbeat GOPATH=$HOME/go
-
-# Or:
-
 make metricbeat-image
+
+hack/containerized-metricbeat
 ```
 
 ### apm-server
@@ -122,6 +118,14 @@ make auditbeat-image
 hack/containerized-auditbeat
 ```
 
+### heartbeat
+
+```
+make heartbeat-image
+
+hack/containerized-heartbeat
+```
+
 ## Running awsbeats on a Kubernetes cluster
 
 ### Filebeat
@@ -157,6 +161,102 @@ helm upgrade --install filebeat ./charts/stable/filebeat \
   --set rbac.enabled=true
 ```
 
+### APM Server
+
+```
+cat << EOS > values.yaml
+image:
+  repository: kubeaws/awsbeats
+  tag: apm-server-canary
+  pullPolicy: Always
+
+plugins:
+  - kinesis.so
+
+config:
+  output.file:
+    enabled: false
+  output.streams:
+    enabled: true
+    region: ap-northeast-1
+    stream_name: test1
+    partition_key: mykey
+EOS
+
+# No need to do this once stable/apm-server is merged
+# See https://github.com/kubernetes/charts/pull/6058
+git clone [email protected]:mumoshu/charts.git charts
+git checkout apm-server
+
+helm upgrade --install apm-server ./charts/stable/apm-server \
+  -f values.yaml \
+  --set rbac.enabled=true
+```
+
+### Auditbeat
+
+```
+cat << EOS > values.yaml
+image:
+  repository: kubeaws/awsbeats
+  tag: auditbeat-canary
+  pullPolicy: Always
+
+plugins:
+  - kinesis.so
+
+config:
+  output.file:
+    enabled: false
+  output.streams:
+    enabled: true
+    region: ap-northeast-1
+    stream_name: test1
+    partition_key: mykey
+EOS
+
+# No need to do this once stable/auditbeat is merged
+# See https://github.com/kubernetes/charts/pull/6089
+git clone [email protected]:mumoshu/charts.git charts
+git checkout auditbeat
+
+helm upgrade --install auditbeat ./charts/stable/auditbeat \
+  -f values.yaml \
+  --set rbac.enabled=true
+```
+
+### Heartbeat
+
+```
+cat << EOS > values.yaml
+image:
+  repository: kubeaws/awsbeats
+  tag: heartbeat-canary
+  pullPolicy: Always
+
+plugins:
+  - kinesis.so
+
+config:
+  output.file:
+    enabled: false
+  output.streams:
+    enabled: true
+    region: ap-northeast-1
+    stream_name: test1
+    partition_key: mykey
+EOS
+
+# No need to do this once stable/heartbeat is merged
+# See https://github.com/kubernetes/charts/pull/5766
+git clone [email protected]:mumoshu/charts.git charts
+git checkout heartbeat
+
+helm upgrade --install heartbeat ./charts/stable/heartbeat \
+  -f values.yaml \
+  --set rbac.enabled=true
+```
+
 ### Metricbeat
 
 Edit the official Kubernetes manifests to use:

example/heartbeat/heartbeat.yml

@@ -0,0 +1,41 @@
+heartbeat.monitors:
+- type: icmp
+  schedule: '*/5 * * * * * *'
+  hosts: ["localhost"]
+  ipv4: true
+  timeout: 16s
+  wait: 1s
+- type: tcp
+  schedule: '@every 5s'
+  hosts: ["myhost:7"]  # default TCP Echo Protocol
+  check.send: "Check"
+  check.receive: "Check"
+- type: http
+  schedule: '@every 5s'
+  urls: ["http://localhost:80/service/status"]
+  check.response.status: 200
+
+heartbeat.scheduler:
+  limit: 10
+
+processors:
+- add_cloud_metadata:
+# Match originating pod enrich apm events with metadata from Kubernetes
+# See https://github.com/elastic/apm-server/issues/349 for more details
+#- add_kubernetes_metadata:
+#    indexers:
+#      - ip_port:
+#    matchers:
+#      - fields:
+#          lookup_fields: ["context.system.ip"]
+
+output:
+  streams:
+    region: ap-northeast-1
+    stream_name: kuokatest1
+    partition_key_provider: xid
+
+queue.mem:
+  events: 4096
+  flush.min_events: 5
+  flush.timeout: 3s

example/metricbeat/metricbeat.yml

@@ -0,0 +1,64 @@
+metricbeat.config.modules:
+  # Mounted `metricbeat-daemonset-modules` configmap:
+  path: ${path.config}/modules.d/*.yml
+  # Reload module configs as they change:
+  reload.enabled: false
+
+metricbeat.autodiscover:
+  providers:
+  - type: docker
+    templates:
+    - condition:
+        equals.docker.container.image: etcd
+      config:
+      - module: etcd
+        metricsets: ["leader", "self", "store"]
+        period: 10s
+        hosts: ["${host}:2379"]
+
+metricbeat.modules:
+- module: system
+  period: 10s
+  metricsets:
+  - cpu
+  - filesystem
+  - fsstat
+  - load
+  - memory
+  - network
+  - process
+  - process_summary
+  - uptime
+  - core
+  - diskio
+  - socket
+  processes: ['.*']
+  process.include_top_n:
+    by_cpu: 5      # include top 5 processes by CPU
+    by_memory: 5   # include top 5 processes by memory
+- module: system
+  period: 1m
+  metricsets:
+  - filesystem
+  - fsstat
+  processors:
+  - drop_event.when.regexp:
+      system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
+- module: docker
+  metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
+  hosts: ["unix:///var/run/docker.sock"]
+  period: 10s
+
+processors:
+- add_cloud_metadata:
+
+output.streams:
+  region: ap-northeast-1
+  stream_name: test1
+  # Use https://github.com/rs/xid to generate a unique ID per event
+  partition_key_provider: xid
+
+queue.mem:
+  events: 4096
+  flush.min_events: 5
+  flush.timeout: 3s

hack/containerized-heartbeat

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+docker run \
+  --rm \
+  -v $(pwd)/logs:/mnt/log/  \
+  -v $(pwd)/example/heartbeat/data:/var/lib/beat \
+  -v $(pwd)/example/heartbeat:/etc/heartbeat \
+  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+  s12v/awsbeats:heartbeat-canary \
+  heartbeat \
+  --plugin kinesis.so \
+  -e \
+  -d '*' \
+  -c /etc/heartbeat/heartbeat.yml \
+  --strict.perms=false
+
+# Note that `strict.perms` seems to be required due to https://discuss.elastic.co/t/volume-mapped-filebeat-yml-permissions-from-docker-on-windows-host/91893/2
+
+# --pid=host, AUDIT_CONTROL, and AUDIT_READ are reequired in order to avoid the following startup error
+# 2018-06-13T03:23:30.026ZDEBUG[file_integrity]file_integrity/metricset.go:86Initialized the file event reader. Running as euid=0
+# 2018-06-13T03:23:30.026ZERRORinstance/beat.go:667Exiting: 1 error: 1 error: failed to create audit client: failed to get audit status: operation not permitted
+# Exiting: 1 error: 1 error: failed to create audit client: failed to get audit status: operation not permitted
+#
+# See https://github.com/elastic/beats-docker/issues/25 for more info

hack/containerized-metricbeat

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# https://discuss.elastic.co/t/docker-module-cant-connect-to-var-run-docker-sock/86835
+docker run \
+  --rm \
+  -u root \
+  --volume=/proc:/hostfs/proc:ro \
+  --volume=/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro \
+  --volume=/:/hostfs:ro \
+  --net=host \
+  -v $(pwd)/logs:/mnt/log/ \
+  -v $(pwd)/example/metricbeat:/etc/metricbeat \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+  -it \
+  s12v/awsbeats:metricbeat-canary \
+  metricbeat \
+  --plugin kinesis.so \
+  -e \
+  -d '*' \
+  -c /etc/metricbeat/metricbeat.yml \
+  --strict.perms=false \
+  -system.hostfs=/hostfs
+
+# Note that `strict.perms` seems to be required due to https://discuss.elastic.co/t/volume-mapped-filebeat-yml-permissions-from-docker-on-windows-host/91893/2