From 609ea78815ddb03dc350b81e83eb3e12959943f6 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Fri, 29 Oct 2021 13:12:05 +0200
Subject: [PATCH 01/18] incresaed cleanup operator time

---
 chart-infra/templates/kube-cleanup-operator-deployment.yaml | 3 ++-
 pipeline-runner/R/gem2s-7-upload_to_aws.R                   | 1 +
 pipeline-runner/init.R                                      | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/chart-infra/templates/kube-cleanup-operator-deployment.yaml b/chart-infra/templates/kube-cleanup-operator-deployment.yaml
index 772273f1..8e5415c6 100644
--- a/chart-infra/templates/kube-cleanup-operator-deployment.yaml
+++ b/chart-infra/templates/kube-cleanup-operator-deployment.yaml
@@ -30,7 +30,8 @@ spec:
         # delete orphaned pods after 1s, so successfully run
         # workers wont linger around
         - --legacy-mode=false
-        - --delete-orphaned-pods-after=1s
+        # wait up to 6 minutes to avoid removing pods which are in containerCreating but are already assigned
+        - --delete-orphaned-pods-after=3600s
         image: quay.io/lwolf/kube-cleanup-operator
         imagePullPolicy: Always
         name: cleanup-operator
diff --git a/pipeline-runner/R/gem2s-7-upload_to_aws.R b/pipeline-runner/R/gem2s-7-upload_to_aws.R
index f718ff32..d08dfc60 100644
--- a/pipeline-runner/R/gem2s-7-upload_to_aws.R
+++ b/pipeline-runner/R/gem2s-7-upload_to_aws.R
@@ -29,6 +29,7 @@ upload_to_aws <- function(input, pipeline_config, prev_out) {
   fpath <- file.path(tempdir(), 'experiment.rds')
   saveRDS(scdata, fpath, compress = FALSE)
 
+  # can only upload up to 50Gb because part numbers can be any number from 1 to 10,000, inclusive.
   put_object_in_s3_multipart(pipeline_config,
                              bucket = pipeline_config$source_bucket,
                              object = fpath,
diff --git a/pipeline-runner/init.R b/pipeline-runner/init.R
index 80d02e7b..afc14a32 100644
--- a/pipeline-runner/init.R
+++ b/pipeline-runner/init.R
@@ -346,7 +346,7 @@ init <- function() {
                     cause = error_txt
                 )
 
-                send_pipeline_fail_update(pipeline_config, input_parse$experimentId, input_parse$processName, "Error message placeholder")
+                send_pipeline_fail_update(pipeline_config, input_parse$experimentId, input_parse$processName, error_txt)
 
                 message("Sent task failure to state machine task: ", taskToken)
                 message("recovered from error:", e$message)

From c3581c4781db3bc487fc906b965aca7f01b93128 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Fri, 29 Oct 2021 20:47:16 +0200
Subject: [PATCH 02/18] exiting 0 when activity error happens

---
 pipeline-runner/init.R | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pipeline-runner/init.R b/pipeline-runner/init.R
index afc14a32..2ad5b2d0 100644
--- a/pipeline-runner/init.R
+++ b/pipeline-runner/init.R
@@ -309,10 +309,15 @@ init <- function() {
     message("Waiting for tasks")
 
     repeat {
-        c(taskToken, input) %<-% states$get_activity_task(
-            activityArn = pipeline_config$activity_arn,
-            workerName = pipeline_config$pod_name
-        )
+        tryCatch( {
+            c(taskToken, input) %<-% states$get_activity_task(
+                activityArn = pipeline_config$activity_arn,
+                workerName = pipeline_config$pod_name
+            )
+        }, error = function(e) {
+            message("Error trying to get activity, exiting peacefully")
+            quit('no', 0)
+        })
 
         if(is.null(taskToken) || !length(taskToken) || taskToken == "") {
             message('No input received during last poll, shutting down...')

From d04b596c34c8e796ab677d72ee1411dc493514cf Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 12:07:55 +0200
Subject: [PATCH 03/18] removed try catch changes

---
 pipeline-runner/init.R | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/pipeline-runner/init.R b/pipeline-runner/init.R
index 2ad5b2d0..afc14a32 100644
--- a/pipeline-runner/init.R
+++ b/pipeline-runner/init.R
@@ -309,15 +309,10 @@ init <- function() {
     message("Waiting for tasks")
 
     repeat {
-        tryCatch( {
-            c(taskToken, input) %<-% states$get_activity_task(
-                activityArn = pipeline_config$activity_arn,
-                workerName = pipeline_config$pod_name
-            )
-        }, error = function(e) {
-            message("Error trying to get activity, exiting peacefully")
-            quit('no', 0)
-        })
+        c(taskToken, input) %<-% states$get_activity_task(
+            activityArn = pipeline_config$activity_arn,
+            workerName = pipeline_config$pod_name
+        )
 
         if(is.null(taskToken) || !length(taskToken) || taskToken == "") {
             message('No input received during last poll, shutting down...')

From b7474eabb617af837d045595e08b3f2bb1616213 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 12:33:46 +0200
Subject: [PATCH 04/18] added taskName to qc message outer structure

---
 pipeline-runner/R/handle_data.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pipeline-runner/R/handle_data.R b/pipeline-runner/R/handle_data.R
index 28bf7732..6ce3da21 100644
--- a/pipeline-runner/R/handle_data.R
+++ b/pipeline-runner/R/handle_data.R
@@ -37,6 +37,7 @@ send_output_to_api <- function(pipeline_config, input, plot_data_keys, output) {
 
   msg <- list(
     experimentId = input$experimentId,
+    taskName = input$taskName,
     input = input,
     output = list(
       bucket = pipeline_config$results_bucket,

From c3c04c5fa1dd415ef698454adf53391f29d83bdc Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 13:49:37 +0200
Subject: [PATCH 05/18] added event exporter deployment

---
 .../kubernetes-event-exporter-config.yaml     | 20 ++++++++++++
 .../kubernetes-event-exporter-deployment.yaml | 31 +++++++++++++++++++
 .../kubernetes-event-exporter-roles.yaml      | 23 ++++++++++++++
 3 files changed, 74 insertions(+)
 create mode 100644 chart-infra/templates/kubernetes-event-exporter-config.yaml
 create mode 100644 chart-infra/templates/kubernetes-event-exporter-deployment.yaml
 create mode 100644 chart-infra/templates/kubernetes-event-exporter-roles.yaml

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
new file mode 100644
index 00000000..1f335174
--- /dev/null
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: event-exporter-cfg
+  namespace: "{{ .Values.namespace }}"
+data:
+  config.yaml: |
+    logLevel: error
+    logFormat: json
+    route:
+      routes:
+        - match:
+            - receiver: "cleanup"
+    receivers:
+      - name: "cleanup"
+        webhook:
+          endpoint: "https://api-{{ .Values.sandboxId }}.scp-staging.biomage.net"
+          headers:
+            User-Agent: kube-event-exporter 1.0
+          layout: # Optional
\ No newline at end of file
diff --git a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
new file mode 100644
index 00000000..d855677f
--- /dev/null
+++ b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: event-exporter
+  namespace: "{{ .Values.namespace }}"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: event-exporter
+        version: v1
+    spec:
+      serviceAccountName: event-exporter
+      containers:
+        - name: event-exporter
+          image: ghcr.io/opsgenie/kubernetes-event-exporter:v0.10
+          imagePullPolicy: IfNotPresent
+          args:
+            - -conf=/data/config.yaml
+          volumeMounts:
+            - mountPath: /data
+              name: cfg
+      volumes:
+        - name: cfg
+          configMap:
+            name: event-exporter-cfg
+  selector:
+    matchLabels:
+      app: event-exporter
+      version: v1
\ No newline at end of file
diff --git a/chart-infra/templates/kubernetes-event-exporter-roles.yaml b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
new file mode 100644
index 00000000..d0c331d9
--- /dev/null
+++ b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "{{ .Values.namespace }}"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: "{{ .Values.namespace }}"
+  name: event-exporter
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: event-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
+subjects:
+  - kind: ServiceAccount
+    namespace: "{{ .Values.namespace }}"
+    name: event-exporter
\ No newline at end of file

From 61ceec16dc82c0adc9f07b20fe36f48bea59400d Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 14:23:05 +0200
Subject: [PATCH 06/18] added correct endpoint and temporary layout

---
 .../kubernetes-event-exporter-config.yaml       | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 1f335174..96356670 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -14,7 +14,20 @@ data:
     receivers:
       - name: "cleanup"
         webhook:
-          endpoint: "https://api-{{ .Values.sandboxId }}.scp-staging.biomage.net"
+          endpoint: "https://api-{{ .Values.sandboxId }}.scp-staging.biomage.net/v1/kubernetesEvents"
           headers:
             User-Agent: kube-event-exporter 1.0
-          layout: # Optional
\ No newline at end of file
+          layout:
+            eventType: "kubernetes-event"
+            createdAt: "{{ .GetTimestampMs }}"
+            details:
+              message: "{{ .Message }}"
+              reason: "{{ .Reason }}"
+              type: "{{ .Type }}"
+              count: "{{ .Count }}"
+              kind: "{{ .InvolvedObject.Kind }}"
+              name: "{{ .InvolvedObject.Name }}"
+              namespace: "{{ .Namespace }}"
+              component: "{{ .Source.Component }}"
+              host: "{{ .Source.Host }}"
+              labels: "{{ toJson .InvolvedObject.Labels}}"
\ No newline at end of file

From 75da79d805e1d15e30e6659c617ea50c647dbf73 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 14:45:50 +0200
Subject: [PATCH 07/18] remove layout from kubernetes events

---
 .../kubernetes-event-exporter-config.yaml         | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 96356670..96ba9602 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -17,17 +17,4 @@ data:
           endpoint: "https://api-{{ .Values.sandboxId }}.scp-staging.biomage.net/v1/kubernetesEvents"
           headers:
             User-Agent: kube-event-exporter 1.0
-          layout:
-            eventType: "kubernetes-event"
-            createdAt: "{{ .GetTimestampMs }}"
-            details:
-              message: "{{ .Message }}"
-              reason: "{{ .Reason }}"
-              type: "{{ .Type }}"
-              count: "{{ .Count }}"
-              kind: "{{ .InvolvedObject.Kind }}"
-              name: "{{ .InvolvedObject.Name }}"
-              namespace: "{{ .Namespace }}"
-              component: "{{ .Source.Component }}"
-              host: "{{ .Source.Host }}"
-              labels: "{{ toJson .InvolvedObject.Labels}}"
\ No newline at end of file
+

From ba7e4870b6ef5a9fb8b6d1f706409793b6a2f6ec Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 15:11:38 +0200
Subject: [PATCH 08/18] changed var subs in yamls

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 2 +-
 .../templates/kubernetes-event-exporter-deployment.yaml     | 2 +-
 chart-infra/templates/kubernetes-event-exporter-roles.yaml  | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 96ba9602..b6ac99b8 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: event-exporter-cfg
-  namespace: "{{ .Values.namespace }}"
+  namespace: {{.Release.Namespace}}
 data:
   config.yaml: |
     logLevel: error
diff --git a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
index d855677f..2ad26cd3 100644
--- a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: event-exporter
-  namespace: "{{ .Values.namespace }}"
+  namespace: {{.Release.Namespace}}
 spec:
   replicas: 1
   template:
diff --git a/chart-infra/templates/kubernetes-event-exporter-roles.yaml b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
index d0c331d9..7a5886c9 100644
--- a/chart-infra/templates/kubernetes-event-exporter-roles.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
@@ -1,12 +1,12 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: "{{ .Values.namespace }}"
+  name: {{.Release.Namespace}}
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  namespace: "{{ .Values.namespace }}"
+  namespace: {{.Release.Namespace}}
   name: event-exporter
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -19,5 +19,5 @@ roleRef:
   name: view
 subjects:
   - kind: ServiceAccount
-    namespace: "{{ .Values.namespace }}"
+    namespace: {{.Release.Namespace}}
     name: event-exporter
\ No newline at end of file

From e1aa2a458ce96ec016a08c0d5b6a55dbdf886d98 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 15:22:35 +0200
Subject: [PATCH 09/18] removed namespace creation from kubernetes event
 exporter

---
 chart-infra/templates/kubernetes-event-exporter-roles.yaml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-roles.yaml b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
index 7a5886c9..2f8a30bb 100644
--- a/chart-infra/templates/kubernetes-event-exporter-roles.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{.Release.Namespace}}
----
-apiVersion: v1
 kind: ServiceAccount
 metadata:
   namespace: {{.Release.Namespace}}

From b313927776eda53b3d51e34447abadc26df8e38e Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 15:38:14 +0200
Subject: [PATCH 10/18] logLevel info

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index b6ac99b8..a8721e11 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: {{.Release.Namespace}}
 data:
   config.yaml: |
-    logLevel: error
+    logLevel: info
     logFormat: json
     route:
       routes:

From 5a4f23463607946903fd8195e12769c1763e13b2 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 17:15:16 +0200
Subject: [PATCH 11/18] only chek namespace

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index a8721e11..03faedaa 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -10,7 +10,8 @@ data:
     route:
       routes:
         - match:
-            - receiver: "cleanup"
+          - namespace: "{{ .Release.Namespace }}
+            receiver: "cleanup"
     receivers:
       - name: "cleanup"
         webhook:

From 2ab6da6aec53b526f557aa3f379bc84c9f62e2dc Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 17:39:14 +0200
Subject: [PATCH 12/18] fine-grained matching

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml  | 5 ++++-
 .../templates/kubernetes-event-exporter-deployment.yaml      | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 03faedaa..5767a864 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -10,7 +10,10 @@ data:
     route:
       routes:
         - match:
-          - namespace: "{{ .Release.Namespace }}
+          - namespace: "{{ .Release.Namespace }}"
+            kind: "Pod"
+            labels:
+              type: "pipeline"
             receiver: "cleanup"
     receivers:
       - name: "cleanup"
diff --git a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
index 2ad26cd3..dbb17903 100644
--- a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: event-exporter
-  namespace: {{.Release.Namespace}}
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: 1
   template:

From f18100e6c580435f410e9a2c9ee1a9db12cae630 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 18:11:38 +0200
Subject: [PATCH 13/18] added more matchers

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 5767a864..637fdc65 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -11,10 +11,11 @@ data:
       routes:
         - match:
           - namespace: "{{ .Release.Namespace }}"
-            kind: "Pod"
+          - reason: "BackOff"
+          - kind: "Pod"
             labels:
               type: "pipeline"
-            receiver: "cleanup"
+          - receiver: "cleanup"
     receivers:
       - name: "cleanup"
         webhook:

From 51b8720d027c66fd906d382e95ca9b66c9f7f2a3 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sat, 30 Oct 2021 19:47:21 +0200
Subject: [PATCH 14/18] removed cleanup operator

---
 .../templates/cleanup-operator-roles.yaml     | 45 +++++++++
 .../kube-cleanup-operator-deployment.yaml     | 93 -------------------
 2 files changed, 45 insertions(+), 93 deletions(-)
 create mode 100644 chart-infra/templates/cleanup-operator-roles.yaml
 delete mode 100644 chart-infra/templates/kube-cleanup-operator-deployment.yaml

diff --git a/chart-infra/templates/cleanup-operator-roles.yaml b/chart-infra/templates/cleanup-operator-roles.yaml
new file mode 100644
index 00000000..83735601
--- /dev/null
+++ b/chart-infra/templates/cleanup-operator-roles.yaml
@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cleanup-operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cleanup-operator
+rules:
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups: ["batch", "extensions"]
+  resources:
+  - jobs
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups: ["apps"]
+  resources:
+  - deployments
+  - deployments/scale
+  verbs:
+  - get
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cleanup-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cleanup-operator
+subjects:
+- kind: ServiceAccount
+  name: cleanup-operator
\ No newline at end of file
diff --git a/chart-infra/templates/kube-cleanup-operator-deployment.yaml b/chart-infra/templates/kube-cleanup-operator-deployment.yaml
deleted file mode 100644
index 8e5415c6..00000000
--- a/chart-infra/templates/kube-cleanup-operator-deployment.yaml
+++ /dev/null
@@ -1,93 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    run: cleanup-operator
-    sandboxId: "{{ .Values.sandboxId }}"
-  name: cleanup-operator
-  namespace: {{.Release.Namespace}}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      run: cleanup-operator
-      sandboxId: "{{ .Values.sandboxId }}"
-  strategy:
-    rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 1
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        run: cleanup-operator
-        sandboxId: "{{ .Values.sandboxId }}"
-    spec:
-      serviceAccountName: cleanup-operator
-      containers:
-      - args:
-        - --namespace={{.Release.Namespace}}
-        # delete orphaned pods after 1s, so successfully run
-        # workers wont linger around
-        - --legacy-mode=false
-        # wait up to 6 minutes to avoid removing pods which are in containerCreating but are already assigned
-        - --delete-orphaned-pods-after=3600s
-        image: quay.io/lwolf/kube-cleanup-operator
-        imagePullPolicy: Always
-        name: cleanup-operator
-        resources:
-          requests:
-            cpu: 50m
-            memory: 50Mi
-          limits:
-            cpu: 50m
-            memory: 50Mi
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      terminationGracePeriodSeconds: 30
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cleanup-operator
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: cleanup-operator
-rules:
-- apiGroups: [""]
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - delete
-- apiGroups: ["batch", "extensions"]
-  resources:
-  - jobs
-  verbs:
-  - get
-  - list
-  - watch
-  - delete
-- apiGroups: ["apps"]
-  resources:
-  - deployments
-  - deployments/scale
-  verbs:
-  - get
-  - patch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: cleanup-operator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: cleanup-operator
-subjects:
-- kind: ServiceAccount
-  name: cleanup-operator
\ No newline at end of file

From c34eb6aec829c0dced4967c9e8629033e92c14eb Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sun, 31 Oct 2021 00:49:51 +0200
Subject: [PATCH 15/18] changed config formatting

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 637fdc65..41a96e3a 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -11,11 +11,11 @@ data:
       routes:
         - match:
           - namespace: "{{ .Release.Namespace }}"
-          - reason: "BackOff"
-          - kind: "Pod"
+            reason: "BackOff"
+            kind: "Pod"
             labels:
               type: "pipeline"
-          - receiver: "cleanup"
+            receiver: "cleanup"
     receivers:
       - name: "cleanup"
         webhook:

From 4600c578bf08319f686af846eb13231353f5ad88 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sun, 31 Oct 2021 01:44:41 +0200
Subject: [PATCH 16/18] changed service account

---
 .../kubernetes-event-exporter-config.yaml     |  6 ++--
 .../kubernetes-event-exporter-deployment.yaml |  2 +-
 .../kubernetes-event-exporter-roles.yaml      | 36 +++++++++----------
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 41a96e3a..637fdc65 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -11,11 +11,11 @@ data:
       routes:
         - match:
           - namespace: "{{ .Release.Namespace }}"
-            reason: "BackOff"
-            kind: "Pod"
+          - reason: "BackOff"
+          - kind: "Pod"
             labels:
               type: "pipeline"
-            receiver: "cleanup"
+          - receiver: "cleanup"
     receivers:
       - name: "cleanup"
         webhook:
diff --git a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
index dbb17903..aacf917a 100644
--- a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
@@ -11,7 +11,7 @@ spec:
         app: event-exporter
         version: v1
     spec:
-      serviceAccountName: event-exporter
+      serviceAccountName: cleanup-operator
       containers:
         - name: event-exporter
           image: ghcr.io/opsgenie/kubernetes-event-exporter:v0.10
diff --git a/chart-infra/templates/kubernetes-event-exporter-roles.yaml b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
index 2f8a30bb..bfd405a1 100644
--- a/chart-infra/templates/kubernetes-event-exporter-roles.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
@@ -1,18 +1,18 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: {{.Release.Namespace}}
-  name: event-exporter
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: event-exporter
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: view
-subjects:
-  - kind: ServiceAccount
-    namespace: {{.Release.Namespace}}
-    name: event-exporter
\ No newline at end of file
+# apiVersion: v1
+# kind: ServiceAccount
+# metadata:
+#   namespace: {{.Release.Namespace}}
+#   name: event-exporter
+# ---
+# apiVersion: rbac.authorization.k8s.io/v1
+# kind: ClusterRoleBinding
+# metadata:
+#   name: event-exporter
+# roleRef:
+#   apiGroup: rbac.authorization.k8s.io
+#   kind: ClusterRole
+#   name: view
+# subjects:
+#   - kind: ServiceAccount
+#     namespace: {{.Release.Namespace}}
+#     name: event-exporter
\ No newline at end of file

From 4c34d484ceac009fa3b4f7feb02c201dfe112c92 Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sun, 31 Oct 2021 09:58:47 +0100
Subject: [PATCH 17/18] reverted back to event-exporter service account

---
 .../kubernetes-event-exporter-config.yaml     |  4 +--
 .../kubernetes-event-exporter-deployment.yaml |  2 +-
 .../kubernetes-event-exporter-roles.yaml      | 36 +++++++++----------
 3 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index 637fdc65..b6f778be 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -10,12 +10,10 @@ data:
     route:
       routes:
         - match:
-          - namespace: "{{ .Release.Namespace }}"
-          - reason: "BackOff"
           - kind: "Pod"
             labels:
               type: "pipeline"
-          - receiver: "cleanup"
+            receiver: "cleanup"
     receivers:
       - name: "cleanup"
         webhook:
diff --git a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
index aacf917a..dbb17903 100644
--- a/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-deployment.yaml
@@ -11,7 +11,7 @@ spec:
         app: event-exporter
         version: v1
     spec:
-      serviceAccountName: cleanup-operator
+      serviceAccountName: event-exporter
       containers:
         - name: event-exporter
           image: ghcr.io/opsgenie/kubernetes-event-exporter:v0.10
diff --git a/chart-infra/templates/kubernetes-event-exporter-roles.yaml b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
index bfd405a1..2f8a30bb 100644
--- a/chart-infra/templates/kubernetes-event-exporter-roles.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-roles.yaml
@@ -1,18 +1,18 @@
-# apiVersion: v1
-# kind: ServiceAccount
-# metadata:
-#   namespace: {{.Release.Namespace}}
-#   name: event-exporter
-# ---
-# apiVersion: rbac.authorization.k8s.io/v1
-# kind: ClusterRoleBinding
-# metadata:
-#   name: event-exporter
-# roleRef:
-#   apiGroup: rbac.authorization.k8s.io
-#   kind: ClusterRole
-#   name: view
-# subjects:
-#   - kind: ServiceAccount
-#     namespace: {{.Release.Namespace}}
-#     name: event-exporter
\ No newline at end of file
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: {{.Release.Namespace}}
+  name: event-exporter
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: event-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
+subjects:
+  - kind: ServiceAccount
+    namespace: {{.Release.Namespace}}
+    name: event-exporter
\ No newline at end of file

From 20d993393549180294f6ca5fe13dc7b8dd838f5c Mon Sep 17 00:00:00 2001
From: Pol Alvarez <pol.avms@gmail.com>
Date: Sun, 31 Oct 2021 12:26:32 +0100
Subject: [PATCH 18/18] filtering only by pod events

---
 chart-infra/templates/kubernetes-event-exporter-config.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/chart-infra/templates/kubernetes-event-exporter-config.yaml b/chart-infra/templates/kubernetes-event-exporter-config.yaml
index b6f778be..4ebe30d1 100644
--- a/chart-infra/templates/kubernetes-event-exporter-config.yaml
+++ b/chart-infra/templates/kubernetes-event-exporter-config.yaml
@@ -11,8 +11,6 @@ data:
       routes:
         - match:
           - kind: "Pod"
-            labels:
-              type: "pipeline"
             receiver: "cleanup"
     receivers:
       - name: "cleanup"