template: add Flatcar template

docs/book/src/SUMMARY.md

@@ -6,6 +6,7 @@
 - [Self-managed clusters](./self-managed/index.md)
     - [Provisioning a Cluster](./self-managed/provision.md)
     - [CNI](./self-managed/cni.md)
+    - [Flatcar provisioned cluster](./self-managed/flatcar.md)
 - [Managed clusters - GKE](./managed/index.md)
     - [Provisioning a Cluster](./managed/provision.md)
     - [Cluster Upgrades](./managed/upgrades.md)

docs/book/src/self-managed/flatcar.md

@@ -0,0 +1,27 @@
+# Use Flatcar images
+
+[Flatcar](https://flatcar.org) is a Linux based OS designed to run containers.
+
+## How do I use Flatcar ?
+
+Flatcar uses [Ignition](https://coreos.github.io/ignition/) for initial provisioning instead of cloud-init. It is first required to enable this feature gate before initializing the management cluster:
+```bash
+export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true
+```
+
+Once done, proceed as documented to setup GCP variables. To set the `IMAGE_ID`, use this snippet to get the latest stable Flatcar image:
+```
+VERSION=$(curl -fsSL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt | grep --max-count=1 FLATCAR_VERSION | cut -d = -f 2- | tr '.' '-')
+export IMAGE_ID="projects/kinvolk-public/global/images/flatcar-stable-${VERSION}"
+```
+
+## Generate the workload cluster configuration
+
+Proceed as usual except for the flavor:
+```
+clusterctl generate cluster capi-gcp-quickstart --flavor flatcar > capi-gcp-quickstart.yaml
+```
+
+## Updates configuration
+
+Flatcar auto-update and Kubernetes patch updates are disabled by default. Set `export FLATCAR_DISABLE_AUTO_UPDATE=false` to enable it. This will pull latest Flatcar update and latest Kubernetes patch release. Note that this will reboot your nodes: [`kured`](https://kured.dev/) is recommended to coordinate the nodes reboot.

templates/cluster-template-flatcar.yaml

@@ -0,0 +1,245 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: "${CLUSTER_NAME}"
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks: ["192.168.0.0/16"]
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: GCPCluster
+    name: "${CLUSTER_NAME}"
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: "${CLUSTER_NAME}-control-plane"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPCluster
+metadata:
+  name: "${CLUSTER_NAME}"
+spec:
+  project: "${GCP_PROJECT}"
+  region: "${GCP_REGION}"
+  network:
+    name: "${GCP_NETWORK_NAME}"
+---
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  machineTemplate:
+    infrastructureRef:
+      kind: GCPMachineTemplate
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      name: "${CLUSTER_NAME}-control-plane"
+  kubeadmConfigSpec:
+    files: []
+    format: ignition
+    ignition:
+      containerLinuxConfig:
+        additionalConfig: |
+          storage:
+            links:
+              - path: /etc/extensions/kubernetes.raw
+                hard: false
+                target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+            files:
+              - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
+                mode: 0644
+                contents:
+                  remote:
+                    url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
+              - path: /etc/sysupdate.d/noop.conf
+                mode: 0644
+                contents:
+                  remote:
+                    url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
+              - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+                contents:
+                  remote:
+                    url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+          systemd:
+            units:
+              - name: systemd-sysupdate.service
+                dropins:
+                  - name: kubernetes.conf
+                    contents: |
+                      [Service]
+                      ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
+                      ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
+                      ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
+                      ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
+              - name: update-engine.service
+                # Set this to 'false' if you want to enable Flatcar auto-update
+                mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
+              - name: locksmithd.service
+                # NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
+                mask: true
+              - name: systemd-sysupdate.timer
+                # Set this to 'true' if you want to enable the Kubernetes auto-update.
+                # NOTE: Only patches version will be pulled.
+                enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
+              - name: [email protected]
+                enabled: true
+              - name: kubeadm.service
+                enabled: true
+                dropins:
+                  - name: 10-flatcar.conf
+                    contents: |
+                      [Unit]
+                      Requires=containerd.service coreos-metadata.service
+                      After=containerd.service coreos-metadata.service
+                      [Service]
+                      EnvironmentFile=/run/metadata/flatcar
+    initConfiguration:
+      nodeRegistration:
+        name: $${COREOS_GCE_HOSTNAME}
+        kubeletExtraArgs:
+          cloud-provider: gce
+          feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
+    clusterConfiguration:
+      apiServer:
+        timeoutForControlPlane: 20m
+        extraArgs:
+          cloud-provider: gce
+          feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
+      controllerManager:
+        extraArgs:
+          cloud-provider: gce
+          feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
+          allocate-node-cidrs: "false"
+    joinConfiguration:
+      nodeRegistration:
+        name: $${COREOS_GCE_HOSTNAME}
+        kubeletExtraArgs:
+          cloud-provider: gce
+          feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
+    preKubeadmCommands:
+    - export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
+    - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+    - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+  version: "${KUBERNETES_VERSION}"
+---
+kind: GCPMachineTemplate
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec:
+      instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
+      image: "${IMAGE_ID}"
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: GCPMachineTemplate
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      instanceType: "${GCP_NODE_MACHINE_TYPE}"
+      image: "${IMAGE_ID}"
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      files: []
+      format: ignition
+      ignition:
+        containerLinuxConfig:
+          additionalConfig: |
+            storage:
+              links:
+                - path: /etc/extensions/kubernetes.raw
+                  hard: false
+                  target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+              files:
+                - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
+                  mode: 0644
+                  contents:
+                    remote:
+                      url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
+                - path: /etc/sysupdate.d/noop.conf
+                  mode: 0644
+                  contents:
+                    remote:
+                      url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
+                - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+                  contents:
+                    remote:
+                      url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
+            systemd:
+              units:
+                - name: systemd-sysupdate.service
+                  dropins:
+                    - name: kubernetes.conf
+                      contents: |
+                        [Service]
+                        ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
+                        ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
+                        ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
+                        ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
+                - name: update-engine.service
+                  # Set this to 'false' if you want to enable Flatcar auto-update
+                  mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
+                - name: locksmithd.service
+                  # NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
+                  mask: true
+                - name: systemd-sysupdate.timer
+                  # Set this to 'true' if you want to enable the Kubernetes auto-update.
+                  # NOTE: Only patches version will be pulled.
+                  enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
+                - name: [email protected]
+                  enabled: true
+                - name: kubeadm.service
+                  enabled: true
+                  dropins:
+                    - name: 10-flatcar.conf
+                      contents: |
+                        [Unit]
+                        Requires=containerd.service coreos-metadata.service
+                        After=containerd.service coreos-metadata.service
+                        [Service]
+                        EnvironmentFile=/run/metadata/flatcar
+      joinConfiguration:
+        nodeRegistration:
+          name: $${COREOS_GCE_HOSTNAME}
+          kubeletExtraArgs:
+            cloud-provider: gce
+            feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
+      preKubeadmCommands:
+      - export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
+      - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+      - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml

test/e2e/config/gcp-ci.yaml

@@ -73,6 +73,7 @@ providers:
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml"
+      - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-flatcar.yaml"
 
 variables:
   KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.30.5}"
@@ -86,6 +87,10 @@ variables:
   KUBERNETES_VERSION_UPGRADE_TO: "${KUBERNETES_VERSION_UPGRADE_TO:-v1.30.5}"
   KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.29.5}"
   EXP_CLUSTER_RESOURCE_SET: "true"
+  # Required for Ignition based images (e.g Flatcar)
+  EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"
+  # TODO: Set this based on latest available image
+  FLATCAR_IMAGE_ID: "projects/kinvolk-public/global/images/flatcar-stable-4081-2-0"
   CLUSTER_TOPOLOGY: "true"
   # Cluster Addons
   CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml"