Cherry-pick #8029 to 6.x: Add an example of autodiscover to deployment manifests

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -19,6 +19,12 @@ data:
         # Reload module configs as they change:
         reload.enabled: false
 
+    # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+    #filebeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 

deploy/kubernetes/filebeat/filebeat-configmap.yaml

@@ -19,6 +19,12 @@ data:
         # Reload module configs as they change:
         reload.enabled: false
 
+    # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+    #filebeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: metricbeat-config
+  name: metricbeat-daemonset-config
   namespace: kube-system
   labels:
     k8s-app: metricbeat
@@ -14,6 +14,13 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
+    # To enable hints based autodiscover uncomment this:
+    #metricbeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      host: ${NODE_NAME}
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 
@@ -68,6 +75,7 @@ data:
         - container
         - volume
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["localhost:10255"]
 ---
 # Deploy a Metricbeat instance per node for node metrics retrieval
@@ -109,6 +117,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -146,7 +158,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-daemonset-config
       - name: modules
         configMap:
           defaultMode: 0600
@@ -158,6 +170,32 @@ spec:
 ---
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: metricbeat-deployment-config
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat
+data:
+  metricbeat.yml: |-
+    metricbeat.config.modules:
+      # Mounted `metricbeat-daemonset-modules` configmap:
+      path: ${path.config}/modules.d/*.yml
+      # Reload module configs as they change:
+      reload.enabled: false
+
+    processors:
+      - add_cloud_metadata:
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      username: ${ELASTICSEARCH_USERNAME}
+      password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: metricbeat-deployment-modules
   namespace: kube-system
@@ -176,6 +214,7 @@ data:
         # Uncomment this to get k8s events:
         #- event
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["kube-state-metrics:8080"]
 ---
 # Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
@@ -215,6 +254,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -235,7 +278,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-deployment-config
       - name: modules
         configMap:
           defaultMode: 0600
@@ -270,12 +313,12 @@ rules:
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:
-  - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["apps"]
   resources:
   - statefulsets
+  - deployments
   verbs: ["get", "list", "watch"]
 ---
 apiVersion: v1

deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: metricbeat-config
+  name: metricbeat-daemonset-config
   namespace: kube-system
   labels:
     k8s-app: metricbeat
@@ -14,6 +14,13 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
+    # To enable hints based autodiscover uncomment this:
+    #metricbeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      host: ${NODE_NAME}
+    #      hints.enabled: true
+
     processors:
       - add_cloud_metadata:
 
@@ -68,4 +75,5 @@ data:
         - container
         - volume
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["localhost:10255"]

deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml

@@ -37,6 +37,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -74,7 +78,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-daemonset-config
       - name: modules
         configMap:
           defaultMode: 0600

deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml

@@ -1,5 +1,31 @@
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: metricbeat-deployment-config
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat
+data:
+  metricbeat.yml: |-
+    metricbeat.config.modules:
+      # Mounted `metricbeat-daemonset-modules` configmap:
+      path: ${path.config}/modules.d/*.yml
+      # Reload module configs as they change:
+      reload.enabled: false
+
+    processors:
+      - add_cloud_metadata:
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      username: ${ELASTICSEARCH_USERNAME}
+      password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: metricbeat-deployment-modules
   namespace: kube-system
@@ -18,4 +44,5 @@ data:
         # Uncomment this to get k8s events:
         #- event
       period: 10s
+      host: ${NODE_NAME}
       hosts: ["kube-state-metrics:8080"]

deploy/kubernetes/metricbeat/metricbeat-deployment.yaml

@@ -35,6 +35,10 @@ spec:
           value:
         - name: ELASTIC_CLOUD_AUTH
           value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
         resources:
@@ -55,7 +59,7 @@ spec:
       - name: config
         configMap:
           defaultMode: 0600
-          name: metricbeat-config
+          name: metricbeat-deployment-config
       - name: modules
         configMap:
           defaultMode: 0600

deploy/kubernetes/metricbeat/metricbeat-role.yaml

@@ -14,10 +14,10 @@ rules:
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:
-  - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["apps"]
   resources:
   - statefulsets
+  - deployments
   verbs: ["get", "list", "watch"]