ROX-16643 Move infra-workloads to dedicated node pools

.secrets.baseline

@@ -322,7 +322,7 @@
         "filename": "dp-terraform/helm/rhacs-terraform/charts/observability/templates/01-operator-06-cr.yaml",
         "hashed_secret": "3e513f12b341ed3327bea645a728401b5d0f9ddb",
         "is_verified": false,
-        "line_number": 15
+        "line_number": 21
       }
     ],
     "dp-terraform/helm/rhacs-terraform/charts/secured-cluster/init-bundle.yaml": [
@@ -564,5 +564,5 @@
       }
     ]
   },
-  "generated_at": "2023-11-06T14:09:00Z"
+  "generated_at": "2023-11-13T13:31:40Z"
 }

dp-terraform/helm/rhacs-terraform/charts/audit-logs/templates/06-statefulset.yaml

@@ -46,6 +46,12 @@ spec:
                     values:
                       - {{ include "aggregator.fullname" . }}
               topologyKey: topology.kubernetes.io/zone
+      {{- if .Values.tolerations }}
+      tolerations: {{- toYaml .Values.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- end }}
       terminationGracePeriodSeconds: 60
       securityContext: {}
       containers:

dp-terraform/helm/rhacs-terraform/charts/audit-logs/values.yaml

@@ -60,3 +60,7 @@ customConfig:
 secrets:
   aws_region: "us-east-1"
   aws_role_arn: ""
+
+nodeSelector: {}
+
+tolerations: []

dp-terraform/helm/rhacs-terraform/charts/cloudwatch/templates/01-operator-04-deployment.yaml

@@ -18,6 +18,15 @@ spec:
       labels:
         app: cloudwatch-exporter
     spec:
+      {{- if .Values.affinity }}
+      affinity: {{ toYaml .Values.affinity | indent 8 }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{ toYaml .Values.tolerations | indent 8 }}
+      {{- end }}
       containers:
         - name: cloudwatch-exporter
           image: {{ .Values.image | quote }}

dp-terraform/helm/rhacs-terraform/charts/cloudwatch/values.yaml

@@ -6,3 +6,9 @@ aws:
 clusterName: ""
 environment: ""
 image: "ghcr.io/nerdswords/yet-another-cloudwatch-exporter:v0.55.0"
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

dp-terraform/helm/rhacs-terraform/charts/logging/templates/01-logging-03-cluster-logging.yaml

@@ -11,5 +11,14 @@ spec:
   managementState: "Managed"
   collection:
     logs:
+      {{- if .Values.tolerations }}
+      tolerations: {{ toYaml .Values.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.resources }}
+      resources: {{ toYaml .Values.resources | nindent 8 }}
+      {{- end }}
       type: "fluentd"
       fluentd: {}

dp-terraform/helm/rhacs-terraform/charts/logging/values.yaml

@@ -7,3 +7,9 @@ aws:
   region: "us-east-1"
   accessKeyId: ""
   secretAccessKey: ""
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

dp-terraform/helm/rhacs-terraform/charts/observability/templates/01-operator-06-cr.yaml

@@ -4,6 +4,12 @@ metadata:
   name: observability-stack
   namespace: {{ include "observability.namespace" . }}
 spec:
+  {{- if .Values.affinity }}
+  affinity: {{ .Values.affinity | toYaml | nindent 4 }}
+  {{- end }}
+  {{- if .Values.tolerations }}
+  tolerations: {{ .Values.tolerations | toYaml | nindent 4 }}
+  {{- end }}
   # The cluster ID is added as a label to all metrics when interacting with external services.
   clusterId: {{ .Values.clusterName | quote }}
   configurationSelector:

dp-terraform/helm/rhacs-terraform/charts/observability/values.yaml

@@ -84,3 +84,9 @@ alertManager:
       memory: 256Mi
     limits:
       memory: 256Mi
+
+affinity: {}
+
+tolerations: []
+
+# observability operator doesn't expose nodeSelector

dp-terraform/helm/rhacs-terraform/charts/secured-cluster/templates/secured-cluster-cr.yaml

@@ -16,23 +16,52 @@ spec:
   clusterName: {{ required "clusterName is required when secured-cluster is enabled" .Values.clusterName }}
   centralEndpoint: {{ required "centralEndpoint is required when secured-cluster is enabled" .Values.centralEndpoint }}
   admissionControl:
-    resources:
-      requests:
-        memory: 100Mi
-        cpu: 100m
+    {{- if .Values.admissionControl.resources }}
+    resources: {{ toYaml .Values.admissionControl.resources | nindent 6 }}
+    {{- end }}
+    {{- if .Values.admissionControl.tolerations }}
+    tolerations: {{ toYaml .Values.admissionControl.tolerations | nindent 6 }}
+    {{- end }}
+    {{- if .Values.admissionControl.nodeSelector }}
+    nodeSelector: {{ toYaml .Values.admissionControl.nodeSelector | nindent 6 }}
+    {{- end }}
   sensor:
-    resources:
-      requests:
-        memory: 100Mi
-        cpu: 100m
+    {{- if .Values.sensor.resources }}
+    resources: {{ toYaml .Values.sensor.resources | nindent 6 }}
+    {{- end }}
+    {{- if .Values.sensor.tolerations }}
+    tolerations: {{ toYaml .Values.sensor.tolerations | nindent 6 }}
+    {{- end }}
+    {{- if .Values.sensor.nodeSelector }}
+    nodeSelector: {{ toYaml .Values.sensor.nodeSelector | nindent 6 }}
+    {{- end }}
   perNode:
     collector:
-      resources:
-        requests:
-          memory: 100Mi
-          cpu: 100m
+      {{- if .Values.collector.resources }}
+      resources: {{ toYaml .Values.collector.resources | nindent 8 }}
+      {{- end }}
     compliance:
-      resources:
-        requests:
-          memory: 100Mi
-          cpu: 100m
+      {{- if .Values.compliance.resources }}
+      resources: {{ toYaml .Values.compliance.resources | nindent 8 }}
+      {{- end }}
+  scanner:
+    analyzer:
+      {{- if .Values.scanner.analyzer.resources }}
+      resources: {{ toYaml .Values.scanner.analyzer.resources | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scanner.analyzer.tolerations }}
+      tolerations: {{ toYaml .Values.scanner.analyzer.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scanner.analyzer.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.scanner.analyzer.nodeSelector | nindent 8 }}
+      {{- end }}
+      db:
+        {{- if .Values.scanner.analyzer.db.resources }}
+        resources: {{ toYaml .Values.scanner.analyzer.db.resources | nindent 10 }}
+        {{- end }}
+        {{- if .Values.scanner.analyzer.db.tolerations }}
+        tolerations: {{ toYaml .Values.scanner.analyzer.db.tolerations | nindent 10 }}
+        {{- end }}
+        {{- if .Values.scanner.analyzer.db.nodeSelector }}
+        nodeSelector: {{ toYaml .Values.scanner.analyzer.db.nodeSelector | nindent 10 }}
+        {{- end }}

dp-terraform/helm/rhacs-terraform/charts/secured-cluster/values.yaml

@@ -8,13 +8,49 @@ admissionControl:
   serviceTLS:
     cert: ""
     key: ""
+  resources:
+    requests:
+      memory: 100Mi
+      cpu: 100m
+  tolerations: []
+  nodeSelector: {}
 ca:
   cert: ""
 collector:
   serviceTLS:
     cert: ""
     key: ""
+  resources:
+    requests:
+      memory: 50Mi
+      cpu: 10m
+compliance:
+  resources:
+    requests:
+      memory: 100Mi
+      cpu: 100m
+scanner:
+  analyzer:
+    tolerations: []
+    nodeSelector: {}
+    resources:
+      requests:
+        memory: 100Mi
+        cpu: 100m
+    db:
+      tolerations: []
+      nodeSelector: {}
+      resources:
+        requests:
+          memory: 100Mi
+          cpu: 100m
 sensor:
   serviceTLS:
     cert: ""
     key: ""
+  resources:
+    requests:
+      memory: 100Mi
+      cpu: 10m
+  tolerations: []
+  nodeSelector: {}

dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml

@@ -20,6 +20,15 @@ spec:
       labels:
         app: fleetshard-sync
     spec:
+      {{- if .Values.fleetshardSync.affinity }}
+      affinity: {{ toYaml .Values.fleetshardSync.affinity | nindent 8 }}
+      {{- end }}
+      {{- if .Values.fleetshardSync.tolerations }}
+      tolerations: {{ toYaml .Values.fleetshardSync.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.fleetshardSync.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.fleetshardSync.nodeSelector | nindent 8 }}
+      {{- end }}
       serviceAccountName: fleetshard-sync
       containers:
       - name: fleetshard-sync

dp-terraform/helm/rhacs-terraform/values.yaml

@@ -53,6 +53,13 @@ fleetshardSync:
     enabled: false
   targetedOperatorUpgrades:
     enabled: false
+  nodeSelector:
+    node-role.kubernetes.io/acscs-infra: ""
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
+
 acsOperator:
   enabled: false
   channel: latest
@@ -72,10 +79,27 @@ cloudwatch:
     secretAccessKey: ""
   clusterName: ""
   environment: ""
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
+  nodeSelector:
+    node-role.kubernetes.io/acscs-infra: ""
 
 # See available parameters in charts/observability/values.yaml
 # - enabled flag is used to completely enable/disable observability sub-chart
 observability:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/acscs-infra
+                operator: Exists
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
   enabled: true
   clusterName: ""
   github:
@@ -98,6 +122,12 @@ logging:
   aws:
     accessKeyId: ""
     secretAccessKey: ""
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
+  nodeSelector:
+    node-role.kubernetes.io/acscs-infra: ""
 
 # See available parameters in charts/audit-logs/values.yaml
 # - enabled flag is used to completely enable/disable logging sub-chart
@@ -106,6 +136,12 @@ audit-logs:
   image: 'registry.redhat.io/openshift-logging/vector-rhel8:v0.21'
   annotations: {}
   replicas: 3
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
+  nodeSelector:
+    node-role.kubernetes.io/acscs-infra: ""
   persistence:
     enabled: true
     storageClassName: ""
@@ -157,6 +193,12 @@ secured-cluster:
     serviceTLS:
       cert: ""
       key: ""
+    tolerations:
+      - key: node-role.kubernetes.io/acscs-infra
+        operator: Exists
+        effect: NoSchedule
+    nodeSelector:
+      node-role.kubernetes.io/acscs-infra: ""
   ca:
     cert: ""
   collector:
@@ -167,6 +209,27 @@ secured-cluster:
     serviceTLS:
       cert: ""
       key: ""
+    tolerations:
+      - key: node-role.kubernetes.io/acscs-infra
+        operator: Exists
+        effect: NoSchedule
+    nodeSelector:
+      node-role.kubernetes.io/acscs-infra: ""
+  scanner:
+    analyzer:
+      tolerations:
+        - key: node-role.kubernetes.io/acscs-infra
+          operator: Exists
+          effect: NoSchedule
+      nodeSelector:
+        node-role.kubernetes.io/acscs-infra: ""
+      db:
+        tolerations:
+          - key: node-role.kubernetes.io/acscs-infra
+            operator: Exists
+            effect: NoSchedule
+        nodeSelector:
+          node-role.kubernetes.io/acscs-infra: ""
 
 external-secrets:
   fullnameOverride: rhacs-external-secrets
@@ -188,6 +251,12 @@ external-secrets:
     image:
       repository: quay.io/app-sre/external-secrets
       tag: v0.9.5
+  tolerations:
+    - key: node-role.kubernetes.io/acscs-infra
+      operator: Exists
+      effect: NoSchedule
+  nodeSelector:
+    node-role.kubernetes.io/acscs-infra: ""
 
 secretStore:
   aws:

fleetshard/pkg/central/charts/data/rhacs-operator/templates/rhacs-operator-deployment.yaml

@@ -27,6 +27,15 @@ spec:
         app: rhacs-operator
         control-plane: controller-manager
     spec:
+      {{- if .affinity }}
+      affinity: {{ .affinity | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .nodeSelector }}
+      nodeSelector: {{ .nodeSelector | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .tolerations }}
+      tolerations: {{ .tolerations | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - args:
             - --secure-listen-address=0.0.0.0:8443