[kots]: configure a log collector for ephemeral containers

.werft/jobs/build/build-and-publish.ts

@@ -150,8 +150,8 @@ function publishKots(werft: Werft, jobConfig: JobConfig) {
         { slice: phases.PUBLISH_KOTS },
     );
 
-    // Generate the logo
-    exec(`make logo -C ${REPLICATED_DIR}`, { slice: phases.PUBLISH_KOTS });
+    // Generate the logo and pull any Helm charts
+    exec(`make logo helm -C ${REPLICATED_DIR}`, { slice: phases.PUBLISH_KOTS });
 
     // Update the additionalImages in the kots-app.yaml
     exec(`/tmp/installer mirror kots --file ${REPLICATED_YAML_DIR}/kots-app.yaml`, { slice: phases.PUBLISH_KOTS });

install/kots/BUILD.yaml

@@ -11,4 +11,4 @@ packages:
       - REPLICATED_APP
     config:
       commands:
-        - ["make", "lint"]
+        - ["make", "helm", "lint"]

install/kots/Makefile

@@ -1,9 +1,10 @@
 CHANNEL_STABLE = Stable
 CHANNEL_BETA = Beta
 CHANNEL_UNSTABLE = Unstable
+CHARTS_DIR = charts
 YAML_DIR = manifests
 
-all: logo lint create_dev_release
+all: logo helm lint create_dev_release
 
 create_dev_release:
 	@if [ "${REPLICATED_DEV_CHANNEL}" = "" ]; then \
@@ -23,6 +24,12 @@ create_unstable_release:
 	replicated release create --lint --ensure-channel --yaml-dir ${YAML_DIR} --promote ${CHANNEL_UNSTABLE}
 .PHONY: create_unstable_release
 
+helm:
+	@echo "Installing Helm dependencies"
+	@rm -f ${YAML_DIR}/*.tgz
+	@for f in $(shell ls -d ${CHARTS_DIR}/*/); do cd $${f} && helm dep up && helm package . --destination ../../${YAML_DIR} && cd -; done
+.PHONY: helm
+
 logo:
 	@echo "Generating Base64 logo and saving to manifests/kots-app.yaml"
 
@@ -32,4 +39,3 @@ logo:
 lint:
 	replicated release lint --yaml-dir ${YAML_DIR}
 .PHONY: lint
-

install/kots/charts/fluent-bit/Chart.yaml

@@ -0,0 +1,11 @@
+# Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+# Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+
+apiVersion: v2
+description: Gitpod fluent-bit
+name: fluent-bit
+version: 0.20.2
+dependencies:
+  - name: fluent-bit
+    version: 0.20.2
+    repository: https://fluent.github.io/helm-charts

install/kots/manifests/gitpod-log-collector.yaml

@@ -0,0 +1,84 @@
+# Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+# Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+
+apiVersion: kots.io/v1beta1
+kind: HelmChart
+metadata:
+  name: fluent-bit
+spec:
+  chart:
+    name: fluent-bit
+    chartVersion: 0.20.2
+  helmVersion: v3
+  useHelmInstall: true
+  weight: 10
+  values:
+    fluent-bit:
+      extraVolumes:
+        - name: collector
+          hostPath:
+            path: /gitpod
+            type: DirectoryOrCreate
+
+      extraVolumeMounts:
+        - name: collector
+          mountPath: /gitpod
+
+      extraContainers:
+        - name: kots
+          image: busybox:1
+          command:
+            - sh
+            - -c
+          args:
+            - while true; do echo "waiting" && sleep 60; done
+          volumeMounts:
+            - name: collector
+              mountPath: /gitpod
+              readOnly: true
+
+      labels:
+        app: gitpod
+        component: gitpod-log-collector
+
+      podLabels:
+        app: gitpod
+        component: gitpod-log-collector
+
+      # Config requires four space indentations
+      config:
+        inputs: |
+          [INPUT]
+              Name tail
+              Path /var/log/containers/*.log
+              Parser docker
+              Tag <pod_name>.<container_name>
+              Tag_Regex (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-
+              Skip_Long_Lines On
+
+        filters: |
+          [FILTER]
+              Name stdout
+
+        outputs: |
+          [OUTPUT]
+              Name file
+              Match imagebuild*
+              Mkdir true
+              Path /gitpod/log-collector
+              Format plain
+
+          [OUTPUT]
+              Name file
+              Match installer*
+              Mkdir true
+              Path /gitpod/log-collector
+              Format plain
+
+          # Workspaces
+          [OUTPUT]
+              Name file
+              Match ws*.workspace
+              Mkdir true
+              Path /gitpod/log-collector
+              Format plain

install/kots/manifests/kots-app.yaml

@@ -24,3 +24,6 @@ spec:
     - deployment/ws-manager-bridge
     - deployment/ws-proxy
     - service/proxy
+  additionalImages:
+    - cr.fluentbit.io/fluent/fluent-bit:1.9.4 # Defined by Fluent Bit Helm chart
+    - busybox:1 # Used to pull log files to pull from Fluent Bit, which doesn't container tar binary

install/kots/manifests/kots-support-bundle.yaml

@@ -38,12 +38,20 @@ spec:
     - clusterInfo: {}
     - clusterResources: {}
     - logs:
+        name: logs
         selector:
           - app=gitpod
         namespace: '{{repl Namespace }}'
         limits:
           maxAge: 720h
           maxLines: 10000
+    - copy:
+        name: log-collector
+        selector:
+          - component=gitpod-log-collector
+        namespace: '{{repl Namespace }}'
+        containerPath: /gitpod/log-collector
+        containerName: kots
     - configMap:
         selector:
           - app=gitpod