diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 21799acfee2..9bbe3fd391a 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -14,12 +14,47 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true + metricbeat.autodiscover: + providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Uncomment the following to enable leader election provider that handles + # singleton instance configuration across the Daemonset Pods of the whole cluster + # in order to monitor some unique data sources, like kube-state-metrics. + # When enabling this remember to also delete the Deployment or just set the replicas of the + # Deployment to 0. + #- type: kubernetes + # scope: cluster + # node: ${NODE_NAME} + # unique: true + # # identifier: + # templates: + # - config: + # - module: kubernetes + # hosts: ["kube-state-metrics:8080"] + # period: 10s + # add_metadata: true + # metricsets: + # - state_node + # - state_deployment + # - state_replicaset + # - state_pod + # - state_container + # - state_cronjob + # - state_resourcequota + # - state_statefulset + # # Uncomment this to get k8s events: + # #- event + # - module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s processors: - add_cloud_metadata: @@ -258,6 +293,8 @@ metadata: labels: k8s-app: metricbeat spec: + # Set to 0 if using leader election provider with the Daemonset + replicas: 1 selector: matchLabels: k8s-app: metricbeat diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index a244dda551a..3b0da9cc503 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -14,12 +14,47 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true + metricbeat.autodiscover: + providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Uncomment the following to enable leader election provider that handles + # singleton instance configuration across the Daemonset Pods of the whole cluster + # in order to monitor some unique data sources, like kube-state-metrics. + # When enabling this remember to also delete the Deployment or just set the replicas of the + # Deployment to 0. + #- type: kubernetes + # scope: cluster + # node: ${NODE_NAME} + # unique: true + # # identifier: + # templates: + # - config: + # - module: kubernetes + # hosts: ["kube-state-metrics:8080"] + # period: 10s + # add_metadata: true + # metricsets: + # - state_node + # - state_deployment + # - state_replicaset + # - state_pod + # - state_container + # - state_cronjob + # - state_resourcequota + # - state_statefulset + # # Uncomment this to get k8s events: + # #- event + # - module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s processors: - add_cloud_metadata: diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml index 0e11187cac3..f82b204b63d 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -7,6 +7,8 @@ metadata: labels: k8s-app: metricbeat spec: + # Set to 0 if using leader election provider with the Daemonset + replicas: 1 selector: matchLabels: k8s-app: metricbeat diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 411a7c9ae25..0fa34f5df95 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -197,3 +197,44 @@ metricbeat 1 1 1 1 1m ------------------------------------------------ Metrics should start flowing to Elasticsearch. + + +[float] +==== Deploying Metricbeat Daemonset with Leader Election enabled + +In some cases users may want to avoid deploying both a Deployment and a Daemonset +to collect cluser-wide metrics and node-level metrics. +For this case, we provide the option to deploy Metricbeat only as a Daemonset +and leverage the leader election feature which allows to define configurations +that are enabled only by the leader Pod. The leader Pod is automatically chosen +between the Pods of the Daemonset. +Here is an example of a configuration that enables leader election: +[source,yaml] +----- +metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + identifier: leaderelectionmetricbeat + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node +----- +Users can find more info about the `unique` and `identifier` options at <>. + +Users can enable the respective parts the Daemonset ConfigMap and +set the `replicas` of the Deployment to `0` in order to only deploy +the Daemonset on the cluster with the leader election provider enabled +in order to collect cluster-wide metrics: +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-kubernetes.yaml +kubectl create -f metricbeat-kubernetes.yaml +------------------------------------------------