feat(flux): big brain operator

.taskfiles/bootstrap/Taskfile.yaml

@@ -1,20 +1,15 @@
 ---
 version: '3'
 
-vars:
-  BOOTSTRAP_RESOURCES_DIR: '{{.ROOT_DIR}}/.taskfiles/bootstrap/resources'
-
 tasks:
 
   kubernetes:
-    desc: Bootstrap a Talos Kubernetes cluster backed by flux, sops, and rook
-    prompt: Bootstrap a Talos Kubernetes cluster ... continue?
+    desc: Bootstrap Talos
+    prompt: Bootstrap Talos Cluster?
     cmds:
       - task: etcd
       - task: kubeconfig
       - task: apps
-      - task: rook
-      - task: flux
     preconditions:
       - talosctl config info >/dev/null 2>&1
       - test -f {{.TALOS_DIR}}/clusterconfig/talosconfig
@@ -28,43 +23,19 @@ tasks:
     cmd: talosctl kubeconfig --nodes {{.TALOS_CONTROLLER}} --force {{.KUBERNETES_DIR}}
 
   apps:
-    internal: true
+    desc: Bootstrap Apps
+    prompt: Bootstrap apps into the Talos cluster?
     cmds:
-      - until kubectl wait --for=condition=Ready=False nodes --all --timeout=10m; do sleep 10; done
+      - until kubectl wait nodes --for=condition=Ready=False --all --timeout=10m; do sleep 5; done
+      - op run --env-file {{.KUBERNETES_DIR}}/bootstrap/bootstrap.env --no-masking -- minijinja-cli {{.KUBERNETES_DIR}}/bootstrap/apps/templates/resources.yaml.j2 | kubectl apply --server-side --filename -
       - helmfile --quiet --file {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml apply --skip-diff-on-install --suppress-diff
-      - until kubectl wait --for=condition=Ready nodes --all --timeout=10m; do sleep 10; done
-    env:
-      KUBERNETES_DIR: '{{.KUBERNETES_DIR}}'
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml
-
-  rook:
-    internal: true
-    cmds:
-      - minijinja-cli {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2 | kubectl apply --server-side --filename -
-      - until kubectl --namespace default get job/wipe-rook &>/dev/null; do sleep 5; done
-      - kubectl --namespace default wait job/wipe-rook --for=condition=complete --timeout=5m
-      - stern --namespace default job/wipe-rook --no-follow
-      - kubectl --namespace default delete job wipe-rook
+      - helmfile --quiet --file {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml destroy --selector release=wipe-rook
     env:
+      FLUX_GITHUB_PUBLIC_KEYS:
+        sh: curl -fsSL https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]'
       MODEL: SAMSUNG_MZQL23T8HCLS-00A07
       NODE_COUNT:
         sh: talosctl config info --output json | jq --raw-output '.nodes | length'
-    preconditions:
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2
-
-  flux:
-    internal: true
-    cmds:
-      - for: { var: TEMPLATES }
-        cmd: op run --env-file {{.KUBERNETES_DIR}}/bootstrap/bootstrap.env --no-masking -- minijinja-cli {{.ITEM}} | kubectl apply --server-side --filename -
-      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
-    vars:
-      TEMPLATES:
-        sh: ls {{.KUBERNETES_DIR}}/bootstrap/apps/*.j2
-    env:
       VAULT: K8s
-      FLUX_GITHUB_PUBLIC_KEYS:
-        sh: curl -fsSL https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]'
     preconditions:
       - op user get --me

kubernetes/apps/flux-system/flux/app/helmrelease.yaml → kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml

@@ -2,16 +2,16 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: flux
+  name: flux-operator
 spec:
   interval: 30m
   chart:
     spec:
-      chart: flux2
-      version: 2.14.1
+      chart: flux-operator
+      version: 0.12.0
       sourceRef:
         kind: HelmRepository
-        name: fluxcd-community
+        name: controlplaneio
         namespace: flux-system
   install:
     remediation:
@@ -23,4 +23,4 @@ spec:
       retries: 3
   valuesFrom:
     - kind: ConfigMap
-      name: flux-helm-values
+      name: flux-operator-helm-values

kubernetes/apps/flux-system/flux/app/kustomization.yaml → kubernetes/apps/flux-system/flux-operator/app/kustomization.yaml

@@ -4,8 +4,8 @@ kind: Kustomization
 resources:
   - ./helmrelease.yaml
 configMapGenerator:
-  - name: flux-helm-values
+  - name: flux-operator-helm-values
     files:
-      - ./resources/values.yaml
+      - ./values.yaml
 generatorOptions:
   disableNameSuffixHash: true

kubernetes/apps/flux-system/flux-operator/app/resources/values.yaml

@@ -0,0 +1,3 @@
+---
+serviceMonitor:
+  create: true

kubernetes/apps/flux-system/flux-operator/instance/helmrelease.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: flux-instance
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: flux-instance
+      version: 0.12.0
+      sourceRef:
+        kind: HelmRepository
+        name: controlplaneio
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: flux-instance-helm-values

kubernetes/apps/flux-system/flux-operator/instance/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  - ./prometheusrule.yaml
+configMapGenerator:
+  - name: flux-instance-helm-values
+    files:
+      - ./values.yaml
+generatorOptions:
+  disableNameSuffixHash: true

kubernetes/apps/flux-system/flux-operator/instance/resources/values.yaml

@@ -0,0 +1,91 @@
+---
+instance:
+  cluster:
+    networkPolicy: false
+  components:
+    - source-controller
+    - kustomize-controller
+    - helm-controller
+    - notification-controller
+  sync:
+    kind: GitRepository
+    name: k8s-gitops
+    url: ssh://[email protected]/buroa/k8s-gitops
+    ref: refs/heads/master
+    path: kubernetes/flux
+    pullSecret: github-deploy-key
+  kustomize:
+    patches:
+      # Increase the number of workers and limits
+      # Ref: https://fluxcd.io/flux/installation/configuration/vertical-scaling/#increase-the-number-of-workers-and-limits
+      - patch: |
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --concurrent=10
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --requeue-dependency=5s
+        target:
+          kind: Deployment
+          name: (kustomize-controller|helm-controller|source-controller)
+      - patch: |
+          apiVersion: apps/v1
+          kind: Deployment
+          metadata:
+            name: all
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: manager
+                    resources:
+                      limits:
+                        memory: 2Gi
+        target:
+          kind: Deployment
+          name: (kustomize-controller|helm-controller|source-controller)
+      # Enable in-memory kustomize builds
+      # Ref: https://fluxcd.io/flux/installation/configuration/vertical-scaling/#enable-in-memory-kustomize-builds
+      - patch: |
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --concurrent=20
+          - op: replace
+            path: /spec/template/spec/volumes/0
+            value:
+              name: temp
+              emptyDir:
+                medium: Memory
+        target:
+          kind: Deployment
+          name: kustomize-controller
+      # Enable Helm repositories caching
+      # Ref: https://fluxcd.io/flux/installation/configuration/vertical-scaling/#enable-helm-repositories-caching
+      - patch: |
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --helm-cache-max-size=10
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --helm-cache-ttl=60m
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --helm-cache-purge-interval=5m
+        target:
+          kind: Deployment
+          name: source-controller
+      # Flux near OOM detection for Helm
+      # Ref: https://fluxcd.io/flux/installation/configuration/helm-oom-detection/
+      - patch: |
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --feature-gates=OOMWatch=true
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --oom-watch-memory-threshold=95
+          - op: add
+            path: /spec/template/spec/containers/0/args/-
+            value: --oom-watch-interval=500ms
+        target:
+          kind: Deployment
+          name: helm-controller

kubernetes/apps/flux-system/flux/ks.yaml → kubernetes/apps/flux-system/flux-operator/ks.yaml

@@ -2,14 +2,14 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app flux
+  name: &app flux-operator
   namespace: flux-system
 spec:
   targetNamespace: flux-system
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/flux-system/flux/app
+  path: ./kubernetes/apps/flux-system/flux-operator/app
   prune: true
   sourceRef:
     kind: GitRepository
@@ -22,16 +22,16 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app flux-config
+  name: &app flux-instance
   namespace: flux-system
 spec:
   targetNamespace: flux-system
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
   dependsOn:
-    - name: flux
-  path: ./kubernetes/apps/flux-system/flux/config
+    - name: flux-operator
+  path: ./kubernetes/apps/flux-system/flux-operator/instance
   prune: true
   sourceRef:
     kind: GitRepository