feat(flux): big brain operator

.github/workflows/flux-diff.yaml

@@ -52,7 +52,7 @@ jobs:
             --strip-attrs "helm.sh/chart,checksum/config,app.kubernetes.io/version,chart"
             --limit-bytes 10000
             --all-namespaces
-            --sources "k8s-gitops"
+            --sources "flux-system"
             --output-file diff.patch
 
       - name: Generate Diff

.taskfiles/bootstrap/Taskfile.yaml

@@ -1,20 +1,15 @@
 ---
 version: '3'
 
-vars:
-  BOOTSTRAP_RESOURCES_DIR: '{{.ROOT_DIR}}/.taskfiles/bootstrap/resources'
-
 tasks:
 
   kubernetes:
-    desc: Bootstrap a Talos Kubernetes cluster backed by flux, sops, and rook
-    prompt: Bootstrap a Talos Kubernetes cluster ... continue?
+    desc: Bootstrap Talos
+    prompt: Bootstrap Talos Cluster?
     cmds:
       - task: etcd
       - task: kubeconfig
       - task: apps
-      - task: rook
-      - task: flux
     preconditions:
       - talosctl config info >/dev/null 2>&1
       - test -f {{.TALOS_DIR}}/clusterconfig/talosconfig
@@ -28,43 +23,19 @@ tasks:
     cmd: talosctl kubeconfig --nodes {{.TALOS_CONTROLLER}} --force {{.KUBERNETES_DIR}}
 
   apps:
-    internal: true
+    desc: Bootstrap Apps
+    prompt: Bootstrap apps into the Talos cluster?
     cmds:
-      - until kubectl wait --for=condition=Ready=False nodes --all --timeout=10m; do sleep 10; done
+      - until kubectl wait nodes --for=condition=Ready=False --all --timeout=10m; do sleep 5; done
+      - op run --env-file {{.KUBERNETES_DIR}}/bootstrap/bootstrap.env --no-masking -- minijinja-cli {{.KUBERNETES_DIR}}/bootstrap/apps/templates/resources.yaml.j2 | kubectl apply --server-side --filename -
       - helmfile --quiet --file {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml apply --skip-diff-on-install --suppress-diff
-      - until kubectl wait --for=condition=Ready nodes --all --timeout=10m; do sleep 10; done
-    env:
-      KUBERNETES_DIR: '{{.KUBERNETES_DIR}}'
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml
-
-  rook:
-    internal: true
-    cmds:
-      - minijinja-cli {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2 | kubectl apply --server-side --filename -
-      - until kubectl --namespace default get job/wipe-rook &>/dev/null; do sleep 5; done
-      - kubectl --namespace default wait job/wipe-rook --for=condition=complete --timeout=5m
-      - stern --namespace default job/wipe-rook --no-follow
-      - kubectl --namespace default delete job wipe-rook
+      - helmfile --quiet --file {{.KUBERNETES_DIR}}/bootstrap/apps/helmfile.yaml destroy --selector release=wipe-rook
     env:
+      FLUX_GITHUB_PUBLIC_KEYS:
+        sh: curl -fsSL https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]'
       MODEL: SAMSUNG_MZQL23T8HCLS-00A07
       NODE_COUNT:
         sh: talosctl config info --output json | jq --raw-output '.nodes | length'
-    preconditions:
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2
-
-  flux:
-    internal: true
-    cmds:
-      - for: { var: TEMPLATES }
-        cmd: op run --env-file {{.KUBERNETES_DIR}}/bootstrap/bootstrap.env --no-masking -- minijinja-cli {{.ITEM}} | kubectl apply --server-side --filename -
-      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
-    vars:
-      TEMPLATES:
-        sh: ls {{.KUBERNETES_DIR}}/bootstrap/apps/*.j2
-    env:
       VAULT: K8s
-      FLUX_GITHUB_PUBLIC_KEYS:
-        sh: curl -fsSL https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]'
     preconditions:
       - op user get --me

kubernetes/apps/actions-runner-system/gha-runner-scale-set-controller/ks.yaml

@@ -15,7 +15,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/actions-runner-system/gha-runner-scale-set/ks.yaml

@@ -16,7 +16,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/cert-manager/cert-manager/ks.yaml

@@ -13,7 +13,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m
@@ -36,7 +36,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/databases/cloudnative-pg/ks.yaml

@@ -15,7 +15,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m
@@ -38,7 +38,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/databases/emqx/ks.yaml

@@ -16,7 +16,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m
@@ -38,7 +38,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/external-secrets/external-secrets/ks.yaml

@@ -13,7 +13,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m
@@ -35,7 +35,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/external-secrets/onepassword-connect/ks.yaml

@@ -15,7 +15,7 @@ spec:
   prune: false # revert to true
   sourceRef:
     kind: GitRepository
-    name: k8s-gitops
+    name: flux-system
   wait: true
   interval: 30m
   retryInterval: 1m

kubernetes/apps/flux-system/flux/app/helmrelease.yaml → kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml

@@ -2,16 +2,16 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: flux
+  name: flux-operator
 spec:
   interval: 30m
   chart:
     spec:
-      chart: flux2
-      version: 2.14.1
+      chart: flux-operator
+      version: 0.12.0
       sourceRef:
         kind: HelmRepository
-        name: fluxcd-community
+        name: controlplaneio
         namespace: flux-system
   install:
     remediation:
@@ -23,4 +23,4 @@ spec:
       retries: 3
   valuesFrom:
     - kind: ConfigMap
-      name: flux-helm-values
+      name: flux-operator-helm-values

kubernetes/apps/flux-system/flux/app/kustomization.yaml → kubernetes/apps/flux-system/flux-operator/app/kustomization.yaml

@@ -4,7 +4,7 @@ kind: Kustomization
 resources:
   - ./helmrelease.yaml
 configMapGenerator:
-  - name: flux-helm-values
+  - name: flux-operator-helm-values
     files:
       - ./resources/values.yaml
 generatorOptions:

kubernetes/apps/flux-system/flux-operator/app/resources/values.yaml

@@ -0,0 +1,3 @@
+---
+serviceMonitor:
+  create: true

kubernetes/apps/flux-system/flux-operator/instance/helmrelease.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: flux-instance
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: flux-instance
+      version: 0.12.0
+      sourceRef:
+        kind: HelmRepository
+        name: controlplaneio
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: flux-instance-helm-values

kubernetes/apps/flux-system/flux-operator/instance/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  - ./prometheusrule.yaml
+configMapGenerator:
+  - name: flux-instance-helm-values
+    files:
+      - ./resources/values.yaml
+generatorOptions:
+  disableNameSuffixHash: true

kubernetes/apps/flux-system/flux-operator/instance/prometheusrule.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: flux-instance-rules
+spec:
+  groups:
+    - name: flux-instance.rules
+      rules:
+        - alert: FluxInstanceAbsent
+          expr: absent(flux_instance_info{exported_namespace="flux-system", name="flux"})
+          for: 15m
+          annotations:
+            summary: Flux instance metric is missing
+            description: |
+              The flux_instance_info metric for the Flux instance in namespace {{ $labels.exported_namespace }} is not available.
+          labels:
+            severity: critical
+        - alert: FluxInstanceNotReady
+          expr: flux_instance_info{exported_namespace="flux-system", name="flux", ready!="True"}
+          for: 15m
+          annotations:
+            summary: Flux instance {{ $labels.name }} is not ready
+            description: |
+              The Flux instance in namespace {{ $labels.exported_namespace }} is not ready.
+              Reason: {{ $labels.reason }}
+          labels:
+            severity: critical