feat(cik8s) add a new 3x node pool dedicated to the bom builds

eks-cluster.tf

@@ -121,8 +121,58 @@ module "eks" {
         "ci.jenkins.io/agents-density"                    = 3,
       }
       attach_cluster_primary_security_group = true
+      labels = {
+        "ci.jenkins.io/agents-density" = 3,
+      }
+    },
+    # This list of worker pool is aimed at mixed spot instances type, to ensure that we always get the most available (e.g. the cheaper) spot size
+    # as per https://aws.amazon.com/blogs/compute/cost-optimization-and-resilience-eks-with-spot-instances/
+    # Pricing table for 2023: https://docs.google.com/spreadsheets/d/1_C0I0jE-X0e0vDcdKOFIWcnwpOqWC8RQ4YOCgXNnplY/edit?usp=sharing
+    spot_linux_4xlarge_bom = {
+      # 4xlarge: Instances supporting 3 pods (limited to 4 vCPUs/8 Gb) each with 1 vCPU/1Gb margin
+      name          = "spot-linux-4xlarge"
+      capacity_type = "SPOT"
+      # Less than 5% eviction rate, cost below $0.08 per pod per hour
+      instance_types = [
+        "c5.4xlarge",
+        "c5a.4xlarge"
+      ]
+      block_device_mappings = {
+        xvda = {
+          device_name = "/dev/xvda"
+          ebs = {
+            volume_size           = 90 # With 3 pods / machine, that can use ~30 Gb each at the same time (`emptyDir`)
+            volume_type           = "gp3"
+            iops                  = 3000 # Max included with gp3 without additional cost
+            throughput            = 125  # Max included with gp3 without additional cost
+            encrypted             = false
+            delete_on_termination = true
+          }
+        }
+      }
+      spot_instance_pools = 3 # Amount of different instance that we can use
+      min_size            = 0
+      max_size            = 50
+      desired_size        = 0
+      kubelet_extra_args  = "--node-labels=node.kubernetes.io/lifecycle=spot"
+      tags = {
+        "k8s.io/cluster-autoscaler/enabled"               = true,
+        "k8s.io/cluster-autoscaler/${local.cluster_name}" = "owned",
+        "ci.jenkins.io/agents-density"                    = 3,
+      }
+      attach_cluster_primary_security_group = true
+      labels = {
+        "ci.jenkins.io/agents-density" = 3,
+      }
+      taints = [
+        {
+          key    = "ci.jenkins.io/bom"
+          value  = "true"
+          effect = "NO_SCHEDULE"
+        }
+      ]
     },
-    spot_linux_24xlarge = {
+    spot_linux_24xlarge_bom = {
       # 24xlarge: Instances supporting 23 pods (limited to 4 vCPUs/8 Gb) each with 1 vCPU/1Gb margin
       name          = "spot-linux-24xlarge"
       capacity_type = "SPOT"
@@ -164,7 +214,6 @@ module "eks" {
           effect = "NO_SCHEDULE"
         }
       ]
-      // TODO: taints + label (nodeselector): https://github.com/jenkins-infra/release/blob/41877fe2881e5b211550536ecb3d5b0123a08534/PodTemplates.d/package-windows.yaml#L39-L50
     },
   }