Add an example of autodiscover to deployment manifests (#8029) (#8538)

Autodiscover is not enabled by default (yet) in our example manifests. This change includes examples on how to enable it for Filebeat and Metricbeat. This should ease the task for users who are willing to use autodiscover. (cherry picked from commit 37b1e63)
elastic · Oct 4, 2018 · 7ae10cd · 7ae10cd
1 parent 8bc83d1
commit 7ae10cd
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 8 deletions.
diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml
@@ -19,6 +19,12 @@ data:
         # Reload module configs as they change:
         reload.enabled: false
 
+    # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+    #filebeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 

diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml
@@ -19,6 +19,12 @@ data:
         # Reload module configs as they change:
         reload.enabled: false
 
+    # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+    #filebeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 

diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: metricbeat-config
+  name: metricbeat-daemonset-config
   namespace: kube-system
   labels:
     k8s-app: metricbeat
@@ -14,6 +14,13 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
+    # To enable hints based autodiscover uncomment this:
+    #metricbeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      host: ${NODE_NAME}
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 
@@ -68,6 +75,7 @@ data:
         - container
         - volume
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["localhost:10255"]
 ---
 # Deploy a Metricbeat instance per node for node metrics retrieval
@@ -109,6 +117,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -146,7 +158,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-daemonset-config
       - name: modules
         configMap:
           defaultMode: 0600
@@ -158,6 +170,32 @@ spec:
 ---
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: metricbeat-deployment-config
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat
+data:
+  metricbeat.yml: |-
+    metricbeat.config.modules:
+      # Mounted `metricbeat-daemonset-modules` configmap:
+      path: ${path.config}/modules.d/*.yml
+      # Reload module configs as they change:
+      reload.enabled: false
+
+    processors:
+      - add_cloud_metadata:
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      username: ${ELASTICSEARCH_USERNAME}
+      password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: metricbeat-deployment-modules
   namespace: kube-system
@@ -176,6 +214,7 @@ data:
         # Uncomment this to get k8s events:
         #- event
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["kube-state-metrics:8080"]
 ---
 # Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
@@ -215,6 +254,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -235,7 +278,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-deployment-config
       - name: modules
         configMap:
           defaultMode: 0600
@@ -270,12 +313,12 @@ rules:
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:
-  - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["apps"]
   resources:
   - statefulsets
+  - deployments
   verbs: ["get", "list", "watch"]
 ---
 apiVersion: v1

diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: metricbeat-config
+  name: metricbeat-daemonset-config
   namespace: kube-system
   labels:
     k8s-app: metricbeat
@@ -14,6 +14,13 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
+    # To enable hints based autodiscover uncomment this:
+    #metricbeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      host: ${NODE_NAME}
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 
@@ -68,4 +75,5 @@ data:
         - container
         - volume
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["localhost:10255"]
diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml
@@ -37,6 +37,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -74,7 +78,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-daemonset-config
       - name: modules
         configMap:
           defaultMode: 0600

diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml
@@ -1,5 +1,31 @@
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: metricbeat-deployment-config
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat
+data:
+  metricbeat.yml: |-
+    metricbeat.config.modules:
+      # Mounted `metricbeat-daemonset-modules` configmap:
+      path: ${path.config}/modules.d/*.yml
+      # Reload module configs as they change:
+      reload.enabled: false
+
+    processors:
+      - add_cloud_metadata:
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      username: ${ELASTICSEARCH_USERNAME}
+      password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: metricbeat-deployment-modules
   namespace: kube-system
@@ -18,4 +44,5 @@ data:
         # Uncomment this to get k8s events:
         #- event
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["kube-state-metrics:8080"]
diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml
@@ -35,6 +35,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -55,7 +59,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-deployment-config
       - name: modules
         configMap:
           defaultMode: 0600

diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml
@@ -14,10 +14,10 @@ rules:
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:
-  - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["apps"]
   resources:
   - statefulsets
+  - deployments
   verbs: ["get", "list", "watch"]