User-facing documentation for Logstash on ECK

config/recipes/logstash/README.asciidoc

@@ -8,7 +8,31 @@ endif::[]
 
 = Using Logstash with ECK
 
-This recipe demonstrates how to run the link:https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html[Logstash log parsing example] on Kubernetes with Elasticsearch, Kibana and Filebeat deployed via ECK.
+These recipes demonstrate how to run Logstash, Elasticsearch, Kibana and Filebeat deployed via ECK, using the link:https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html[Logstash log parsing example] as a starting point.
 
+===== Inline Pipeline usage - `logstash-eck.yaml`
+
+Deploys Logstash with the pipeline defined inline in the CRD.
+
+===== Pipeline as Secret - `logstash-pipeline-as-secret.yaml`
+
+Deploys Logstash with the pipeline defined in a Secret and referred to via `pipelinesRef`.
+
+===== Pipeline as mounted volume - `logstash-pipeline-as-volume.yaml`
+
+Deploys Logstash with the pipeline details defined in the CRD, and the pipeline itself mounted as a volume.
+
+===== Logstash with multiple pipelines and multiple elasticsearchRefs - `logstash-multi.yaml`
+
+Deploys Logstash with multiple pipelines, each of which sends to a separate Elasticsearch cluster.
+
+===== Logstash with Stack Monitoring - `logstash-monitored.yaml`
+
+Deploys Logstash and a dedicated Elasticsearch and Kibana monitoring cluster, and sends Logstash monitoring data to that cluster.
+
+===== Logstash and Elasticsearch with custom role - `logstash-es-role.yaml`
+
+Deploys Logstash and Elasticsearch and a Secret to customize Elasticsearch role `eck_logstash_user_role`. The role is essential for Logstash to have privileges to write document to custom index "my-index".
+
+CAUTION: These recipes use the `node.store.allow_mmap: false` configuration value to avoid configuring memory mapping settings on the underlying host. This could have a significant performance impact on your Elasticsearch cluster and should not be used in production without careful consideration. See https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html for more information.
 
-CAUTION: This recipe uses the `node.store.allow_mmap: false` configuration value to avoid configuring memory mapping settings on the underlying host. This could have a significant performance impact on your Elasticsearch cluster and should not be used in production without careful consideration. See https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html for more information.

config/recipes/logstash/logstash-eck.yaml

@@ -0,0 +1,108 @@
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+spec:
+  version: 8.7.0
+  nodeSets:
+    - name: default
+      count: 3
+      config:
+        # This setting has performance implications. See the README for more details.
+        node.store.allow_mmap: false
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+spec:
+  version: 8.7.0
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+---
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+  name: filebeat
+spec:
+  type: filebeat
+  version: 8.7.0
+  config:
+    filebeat.inputs:
+      - type: log
+        paths:
+          - /data/logstash-tutorial.log
+    output.logstash:
+      hosts: ["logstash-ls-api.default.svc:5044"]
+  deployment:
+    podTemplate:
+      spec:
+        automountServiceAccountToken: true
+        initContainers:
+          - name: download-tutorial
+            image: curlimages/curl
+            command: ["/bin/sh"]
+            args: ["-c", "curl -L https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz | gunzip -c > /data/logstash-tutorial.log"]
+            volumeMounts:
+              - name: data
+                mountPath: /data
+        containers:
+          - name: filebeat
+            volumeMounts:
+              - name: data
+                mountPath: /data
+              - name: beat-data
+                mountPath: /usr/share/filebeat/data
+        volumes:
+          - name: data
+            emptydir: {}
+          - name: beat-data
+            emptydir: {}
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash
+spec:
+  count: 1
+  version: 8.7.0
+  elasticsearchRefs:
+    - clusterName: eck
+      name: elasticsearch
+  pipelines:
+    - pipeline.id: main
+      config.string: |
+        input {
+          beats {
+            port => 5044
+          }
+        }
+        filter {
+          grok {
+            match => { "message" => "%{HTTPD_COMMONLOG}"}
+          }
+          geoip {
+            source => "[source][address]"
+            target => "[source]"
+          }
+        }
+        output {
+          elasticsearch {
+            hosts => [ "${ECK_ES_HOSTS}" ]
+            user => "${ECK_ES_USER}"
+            password => "${ECK_ES_PASSWORD}"
+            cacert => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}"
+          }
+        }
+  services:
+    - name: beats
+      service:
+        spec:
+          type: ClusterIP
+          ports:
+            - port: 5044
+              name: "filebeat"
+              protocol: TCP
+              targetPort: 5044

config/recipes/logstash/logstash-es-role.yaml

@@ -0,0 +1,55 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: my-roles-secret
+stringData:
+  roles.yml: |-
+    eck_logstash_user_role:
+      cluster: [ "monitor", "manage_ilm", "read_ilm", "manage_logstash_pipelines", "manage_index_templates", "cluster:admin/ingest/pipeline/get"]
+      indices:
+      - names: [ "my-index", "logstash", "logstash-*", "ecs-logstash", "ecs-logstash-*", "logs-*", "metrics-*", "synthetics-*", "traces-*" ]
+        privileges: [ "manage", "write", "create_index", "read", "view_index_metadata" ]
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+spec:
+  version: 8.7.0
+  auth:
+    roles:
+      - secretName: my-roles-secret
+  nodeSets:
+    - name: default
+      count: 3
+      config:
+        node.store.allow_mmap: false
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash
+spec:
+  count: 1
+  version: 8.7.0
+  elasticsearchRefs:
+    - name: elasticsearch
+      clusterName: eck
+  pipelines:
+    - pipeline.id: main
+      config.string: |
+        input { exec { command => "uptime" interval => 10 } } 
+        output { 
+          elasticsearch {
+            hosts => [ "${ECK_ES_HOSTS}" ]
+            ssl => true
+            cacert => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}"
+            user => "${ECK_ES_USER}"
+            password => "${ECK_ES_PASSWORD}"
+            index => "my-index"
+            data_stream => false
+            ilm_enabled => false
+            manage_template => false
+          } 
+        }
+---

config/recipes/logstash/logstash-monitored.yaml

@@ -0,0 +1,134 @@
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+spec:
+  version: 8.7.0
+  nodeSets:
+    - name: default
+      count: 3
+      config:
+        # This setting has performance implications. See the README for more details.
+        node.store.allow_mmap: false
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+spec:
+  version: 8.7.0
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+---
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+  name: filebeat
+spec:
+  type: filebeat
+  version: 8.7.0
+  config:
+    filebeat.inputs:
+      - type: log
+        paths:
+          - /data/logstash-tutorial.log
+    output.logstash:
+      hosts: ["logstash-ls-api.default.svc:5044"]
+  deployment:
+    podTemplate:
+      spec:
+        automountServiceAccountToken: true
+        initContainers:
+          - name: download-tutorial
+            image: curlimages/curl
+            command: ["/bin/sh"]
+            args: ["-c", "curl -L https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz | gunzip -c > /data/logstash-tutorial.log"]
+            volumeMounts:
+              - name: data
+                mountPath: /data
+        containers:
+          - name: filebeat
+            volumeMounts:
+              - name: data
+                mountPath: /data
+              - name: beat-data
+                mountPath: /usr/share/filebeat/data
+        volumes:
+          - name: data
+            emptydir: {}
+          - name: beat-data
+            emptydir: {}
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash
+spec:
+  count: 1
+  version: 8.7.0
+  elasticsearchRefs:
+    - clusterName: eck
+      name: elasticsearch
+  monitoring:
+    metrics:
+      elasticsearchRefs:
+        - name: elasticsearch-monitoring
+  pipelines:
+    - pipeline.id: main
+      config.string: |
+        input {
+          beats {
+            port => 5044
+          }
+        }
+        filter {
+          grok {
+            match => { "message" => "%{HTTPD_COMMONLOG}"}
+          }
+          geoip {
+            source => "[source][address]"
+            target => "[source]"
+          }
+        }
+        output {
+          elasticsearch {
+            hosts => [ "${ECK_ES_HOSTS}" ]
+            user => "${ECK_ES_USER}"
+            password => "${ECK_ES_PASSWORD}"
+            cacert => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}"
+          }
+        }
+  services:
+    - name: beats
+      service:
+        spec:
+          type: ClusterIP
+          ports:
+            - port: 5044
+              name: "filebeat"
+              protocol: TCP
+              targetPort: 5044
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch-monitoring
+spec:
+  version: 8.7.0
+  nodeSets:
+    - name: default
+      count: 3
+      config:
+        node.store.allow_mmap: false
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana-monitoring
+spec:
+  version: 8.7.0
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch-monitoring