User facing documentation for Logstash on ECK

NOTICE.txt

@@ -2914,11 +2914,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Module  : go.elastic.co/apm/module/apmelasticsearch/v2
-Version : v2.3.0
-Time    : 2023-03-31T03:30:00Z
+Version : v2.4.1
+Time    : 2023-04-27T13:39:08Z
 Licence : Apache-2.0
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/v2@v2.3.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/v2@v2.4.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -3125,11 +3125,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasti
 
 --------------------------------------------------------------------------------
 Module  : go.elastic.co/apm/module/apmhttp/v2
-Version : v2.3.0
-Time    : 2023-03-31T03:30:00Z
+Version : v2.4.1
+Time    : 2023-04-27T13:39:08Z
 Licence : Apache-2.0
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/v2@v2.3.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/v2@v2.4.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -3336,11 +3336,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/v
 
 --------------------------------------------------------------------------------
 Module  : go.elastic.co/apm/module/apmzap/v2
-Version : v2.3.0
-Time    : 2023-03-31T03:30:00Z
+Version : v2.4.1
+Time    : 2023-04-27T13:39:08Z
 Licence : Apache-2.0
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmzap/v2@v2.3.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmzap/v2@v2.4.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -3547,11 +3547,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmzap/v2
 
 --------------------------------------------------------------------------------
 Module  : go.elastic.co/apm/v2
-Version : v2.3.0
-Time    : 2023-03-31T03:30:00Z
+Version : v2.4.1
+Time    : 2023-04-27T13:39:08Z
 Licence : Apache-2.0
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.3.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.4.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

cmd/manager/main.go

@@ -183,6 +183,11 @@ func Command() *cobra.Command {
 		container.DefaultContainerRegistry,
 		"Container registry to use when downloading Elastic Stack container images",
 	)
+	cmd.Flags().String(
+		operator.ContainerRepositoryFlag,
+		"",
+		"Container repository to use when downloading Elastic Stack container images",
+	)
 	cmd.Flags().String(
 		operator.ContainerSuffixFlag,
 		"",
@@ -474,6 +479,13 @@ func startOperator(ctx context.Context) error {
 	log.Info("Setting default container registry", "container_registry", containerRegistry)
 	container.SetContainerRegistry(containerRegistry)
 
+	// set the default container repository
+	containerRepository := viper.GetString(operator.ContainerRepositoryFlag)
+	if containerRepository != "" {
+		log.Info("Setting default container repository", "container_repository", containerRepository)
+		container.SetContainerRepository(containerRepository)
+	}
+
 	// allow users to specify a container suffix unless --ubi-only mode is active
 	suffix := viper.GetString(operator.ContainerSuffixFlag)
 	if len(suffix) > 0 {

config/crds/v1/all-crds.yaml

@@ -757,9 +757,9 @@ spec:
                 - fleet
                 type: string
               policyID:
-                description: PolicyID optionally determines into which Agent Policy
-                  this Agent will be enrolled. If left empty the default policy will
-                  be used.
+                description: PolicyID determines into which Agent Policy this Agent
+                  will be enrolled. This field will become mandatory in a future release,
+                  default policies are deprecated since 8.1.0.
                 type: string
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain

config/crds/v1/bases/agent.k8s.elastic.co_agents.yaml

@@ -16310,9 +16310,9 @@ spec:
                 - fleet
                 type: string
               policyID:
-                description: PolicyID optionally determines into which Agent Policy
-                  this Agent will be enrolled. If left empty the default policy will
-                  be used.
+                description: PolicyID determines into which Agent Policy this Agent
+                  will be enrolled. This field will become mandatory in a future release,
+                  default policies are deprecated since 8.1.0.
                 type: string
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain

config/recipes/elastic-agent/fleet-apm-integration.yaml

@@ -22,7 +22,6 @@ spec:
     xpack.fleet.agentPolicies:
     - name: Fleet Server on ECK policy
       id: eck-fleet-server
-      is_default_fleet_server: true
       namespace: default
       monitoring_enabled:
       - logs
@@ -40,7 +39,6 @@ spec:
       - logs
       - metrics
       unenroll_timeout: 900
-      is_default: true
       package_policies:
       - name: system-1
         id: system-1
@@ -80,6 +78,7 @@ spec:
   - name: elasticsearch
   mode: fleet
   fleetServerEnabled: true
+  policyID: eck-fleet-server
   deployment:
     replicas: 1
     podTemplate:
@@ -100,6 +99,7 @@ spec:
   fleetServerRef: 
     name: fleet-server
   mode: fleet
+  policyID: eck-agent
   deployment:
     replicas: 1
     podTemplate:

config/recipes/elastic-agent/fleet-custom-logs-integration.yaml

@@ -27,7 +27,6 @@ spec:
       - logs
       - metrics
       unenroll_timeout: 900
-      is_default_fleet_server: true
       package_policies:
       - name: fleet_server-1
         id: fleet_server-1
@@ -39,8 +38,7 @@ spec:
       monitoring_enabled:
       - logs
       - metrics
-      unenroll_timeout: 900  
-      is_default: true
+      unenroll_timeout: 900
       package_policies:
       - name: system-1
         id: system-1
@@ -89,6 +87,7 @@ spec:
   - name: elasticsearch
   mode: fleet
   fleetServerEnabled: true
+  policyID: eck-fleet-server
   deployment:
     replicas: 1
     podTemplate:
@@ -109,6 +108,7 @@ spec:
   fleetServerRef: 
     name: fleet-server
   mode: fleet
+  policyID: eck-agent
   daemonSet:
     podTemplate:
       spec:

config/recipes/elastic-agent/fleet-kubernetes-integration.yaml

@@ -27,7 +27,6 @@ spec:
       - logs
       - metrics
       unenroll_timeout: 900
-      is_default_fleet_server: true
       package_policies:
       - name: fleet_server-1
         id: fleet_server-1
@@ -40,7 +39,6 @@ spec:
       - logs
       - metrics
       unenroll_timeout: 900
-      is_default: true
       package_policies:
       - package:
           name: system
@@ -73,6 +71,7 @@ spec:
   - name: elasticsearch
   mode: fleet
   fleetServerEnabled: true
+  policyID: eck-fleet-server
   deployment:
     replicas: 1
     podTemplate:
@@ -93,6 +92,7 @@ spec:
   fleetServerRef: 
     name: fleet-server
   mode: fleet
+  policyID: eck-agent
   daemonSet:
     podTemplate:
       spec:

config/recipes/logstash/README.asciidoc

@@ -8,7 +8,27 @@ endif::[]
 
 = Using Logstash with ECK
 
-This recipe demonstrates how to run the link:https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html[Logstash log parsing example] on Kubernetes with Elasticsearch, Kibana and Filebeat deployed via ECK.
+These recipes demonstrate how to run Logstash, Elasticsearch, Kibana and Filebeat deployed via ECK, using the link:https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html[Logstash log parsing example] as a starting point.
 
+===== Inline Pipeline usage - `logstash-eck.yaml`
+
+Deploys Logstash with the pipeline defined inline in the CRD.
+
+===== Pipeline as Secret - `logstash-pipeline-as-secret.yaml`
+
+Deploys Logstash with the pipeline defined in a Secret and referred to via `pipelinesRef`.
+
+===== Pipeline as mounted volume - `logstash-pipeline-as-volume.yaml`
+
+Deploys Logstash with the pipeline details defined in the CRD, and the pipeline itself mounted as a volume.
+
+===== Logstash with Stack Monitoring - `logstash-monitored.yaml`
+
+Deploys Logstash and a dedicated Elasticsearch and Kibana monitoring cluster, and sends Logstash monitoring data to that cluster.
+
+===== Logstash and Elasticsearch with custom role - `logstash-es-role.yaml`
+
+Deploys Logstash and Elasticsearch and a Secret to customize Elasticsearch role `eck_logstash_user_role`. The role is essential for Logstash to have privileges to write document to custom index "my-index".
+
+CAUTION: These recipes use the `node.store.allow_mmap: false` configuration value to avoid configuring memory mapping settings on the underlying host. This could have a significant performance impact on your Elasticsearch cluster and should not be used in production without careful consideration. See https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html for more information.
 
-CAUTION: This recipe uses the `node.store.allow_mmap: false` configuration value to avoid configuring memory mapping settings on the underlying host. This could have a significant performance impact on your Elasticsearch cluster and should not be used in production without careful consideration. See https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html for more information.

config/recipes/logstash/logstash-eck.yaml

@@ -0,0 +1,108 @@
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+spec:
+  version: 8.7.0
+  nodeSets:
+    - name: default
+      count: 3
+      config:
+        # This setting has performance implications. See the README for more details.
+        node.store.allow_mmap: false
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+spec:
+  version: 8.7.0
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+---
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+  name: filebeat
+spec:
+  type: filebeat
+  version: 8.7.0
+  config:
+    filebeat.inputs:
+      - type: log
+        paths:
+          - /data/logstash-tutorial.log
+    output.logstash:
+      hosts: ["logstash-ls-api.default.svc:5044"]
+  deployment:
+    podTemplate:
+      spec:
+        automountServiceAccountToken: true
+        initContainers:
+          - name: download-tutorial
+            image: curlimages/curl
+            command: ["/bin/sh"]
+            args: ["-c", "curl -L https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz | gunzip -c > /data/logstash-tutorial.log"]
+            volumeMounts:
+              - name: data
+                mountPath: /data
+        containers:
+          - name: filebeat
+            volumeMounts:
+              - name: data
+                mountPath: /data
+              - name: beat-data
+                mountPath: /usr/share/filebeat/data
+        volumes:
+          - name: data
+            emptydir: {}
+          - name: beat-data
+            emptydir: {}
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash
+spec:
+  count: 1
+  version: 8.7.0
+  elasticsearchRefs:
+    - clusterName: eck
+      name: elasticsearch
+  pipelines:
+    - pipeline.id: main
+      config.string: |
+        input {
+          beats {
+            port => 5044
+          }
+        }
+        filter {
+          grok {
+            match => { "message" => "%{HTTPD_COMMONLOG}"}
+          }
+          geoip {
+            source => "[source][address]"
+            target => "[source]"
+          }
+        }
+        output {
+          elasticsearch {
+            hosts => [ "${ECK_ES_HOSTS}" ]
+            user => "${ECK_ES_USER}"
+            password => "${ECK_ES_PASSWORD}"
+            cacert => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}"
+          }
+        }
+  services:
+    - name: beats
+      service:
+        spec:
+          type: ClusterIP
+          ports:
+            - port: 5044
+              name: "filebeat"
+              protocol: TCP
+              targetPort: 5044