From 466efb8fa7320cbd7e3d19e003ecda8e4d2bacf5 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 29 Oct 2024 16:56:37 +0100 Subject: [PATCH] CAPZ-demo: add automation This change adds automation to prepare and to run the CAPZ demo. Automation currently works around limitations caused by the CAPZ upstream PR not being merged. There are TODOs in capz-demo.env for when the PR is merged upstream. Signed-off-by: Thilo Fromm --- CAPZ-sysext/.gitignore | 3 + CAPZ-sysext/azure.env.template | 27 ++ CAPZ-sysext/capz-demo.env | 231 ++++++++++++ .../cluster-template-flatcar-sysext.yaml | 337 ++++++++++++++++++ 4 files changed, 598 insertions(+) create mode 100644 CAPZ-sysext/.gitignore create mode 100644 CAPZ-sysext/azure.env.template create mode 100644 CAPZ-sysext/capz-demo.env create mode 100644 CAPZ-sysext/cluster-template-flatcar-sysext.yaml diff --git a/CAPZ-sysext/.gitignore b/CAPZ-sysext/.gitignore new file mode 100644 index 0000000..4b32c8e --- /dev/null +++ b/CAPZ-sysext/.gitignore @@ -0,0 +1,3 @@ +*.swp +demo +azure.env diff --git a/CAPZ-sysext/azure.env.template b/CAPZ-sysext/azure.env.template new file mode 100644 index 0000000..a1e950b --- /dev/null +++ b/CAPZ-sysext/azure.env.template @@ -0,0 +1,27 @@ +# Template for Azure CAPZ settings + +# The subscrition ID to use for the workload cluster. +export AZURE_SUBSCRIPTION_ID="TODO add subscription ID" + +# Uncomment and set this to the base64 encoded public component +# of the SSH key you want to use to log into the control plane node, e.g.: +# base64 -w0 ~/.ssh/id_rsa.pub +# Leave commented out if you don't need SSH access. +#export AZURE_SSH_PUBLIC_KEY_B64="" + +# From https://capz.sigs.k8s.io/getting-started: +# az ad sp create-for-rbac --role contributor --scopes="/subscriptions/${AZURE_SUBSCRIPTION_ID}" +export AZURE_TENANT_ID="TODO add 'tenant' from output of az command" +export AZURE_CLIENT_ID="TODO add 'appId from output of az command'" +export AZURE_CLIENT_SECRET="TODO add 'password' from output of az command" + +# +# These usually do not need to be touched +# + +export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="${AZURE_CLIENT_ID}" # for compatibility with CAPZ v1.16 templates + +# AZURE_RESOURCE_GROUP is set in capz-demo.env +export AZURE_CLUSTER_IDENTITY_SECRET_NAME="${AZURE_RESOURCE_GROUP}-cluster-identity-secret" +export CLUSTER_IDENTITY_NAME="${AZURE_RESOURCE_GROUP}-cluster-identity" +export AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE="default" diff --git a/CAPZ-sysext/capz-demo.env b/CAPZ-sysext/capz-demo.env new file mode 100644 index 0000000..94bd9ab --- /dev/null +++ b/CAPZ-sysext/capz-demo.env @@ -0,0 +1,231 @@ +#!/bin/bash + +KUBERNETES_VERSION="v1.30.0" + +AZURE_RESOURCE_GROUP="flatcar-capi-demo-azure" +export AZURE_RESOURCE_GROUP + +WORKER_CONTROLPLANE_NODES=1 +WORKER_NODES=2 + +p() { + echo + echo "#####################################" + echo "${@}" + echo "-------------------------------------" +} +# -- + +check_command() { + local cmd="$1" + + if ! command -v "$cmd" &> /dev/null ; then + echo "'$cmd' could not be found. Please install your distro's '$cmd'." + return 1 + fi + echo " - '$cmd'" +} +# -- + +check_file() { + local f="$1" + + if [ ! -f "./$f" ] ; then + echo "prerequisite '$f' could not be found." + return 1 + fi + + echo " - '$f'" +} +# -- + +get_prerequisites() { + p "Prerequisites: Checking for required host commands." + check_command "kubectl" || return + check_command "yq" || return + check_command "wget" || return + + p "Prerequisites: Checking for prerequisite files." + check_file "azure.env" || return + check_file "cluster-template-flatcar-sysext.yaml" || return + + if [ ! -f ./clusterctl ] ; then + p "Prerequisites: fetching clusterctl" + wget https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/clusterctl-linux-amd64 + mv clusterctl-linux-amd64 clusterctl + chmod 755 clusterctl + fi + + if [ ! -f ./kind ] ; then + p "Prerequisites: fetching kind" + wget https://github.com/kubernetes-sigs/kind/releases/latest/download/kind-linux-amd64 + mv kind-linux-amd64 kind + chmod 755 kind + fi + + if [ ! -f ./helm ] ; then + p "Prerequisites: fetching helm" + curl https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz \ + | tar xz linux-amd64/helm -O >helm + chmod 755 helm + mkdir -p helm-cache + fi +} +# -- + +setup_kind_cluster() { + p "Bootsstrapping cluster" + ./kind create cluster --kubeconfig=./kind-mgmt.kubeconfig + export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true +} +# -- + +kc_mgmt() { + kubectl --kubeconfig=./kind-mgmt.kubeconfig "${@}" + +} +# -- + +kc_worker() { + kubectl --kubeconfig=./${AZURE_RESOURCE_GROUP}.kubeconfig "${@}" +} +# -- + +generate_capz_yaml() { + source ./azure.env + + p "Initialising ClusterAPI Azure provider." + ./clusterctl init --infrastructure azure --kubeconfig=./kind-mgmt.kubeconfig + + # FIXME: add + # --infrastructure azure \ + # --flavor flatcar-sysext \ + # and remove + # --from cluster-template-flatcar-sysext.yaml \ + # when https://github.com/kubernetes-sigs/cluster-api-provider-azure/pull/4575 is merged. + + # TODO: remove when PR is merged + export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3" + export AZURE_LOCATION="northeurope" + export AZURE_NODE_MACHINE_TYPE="Standard_D2s_v3" + + export CI_RG="${AZURE_RESOURCE_GROUP}" + export FLATCAR_VERSION="latest" + + p "Generating ${AZURE_RESOURCE_GROUP}.yaml." + ./clusterctl generate cluster ${AZURE_RESOURCE_GROUP} \ + --from cluster-template-flatcar-sysext.yaml \ + --kubeconfig=./kind-mgmt.kubeconfig \ + --kubernetes-version "${KUBERNETES_VERSION}" \ + --control-plane-machine-count=${WORKER_CONTROLPLANE_NODES} \ + --worker-machine-count=${WORKER_NODES} \ + > ${AZURE_RESOURCE_GROUP}.yaml + + yq -i "with(. | select(.kind == \"AzureClusterIdentity\"); .spec.type |= \"ServicePrincipal\" | .spec.clientSecret.name |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}\" | .spec.clientSecret.namespace |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}\")" "${AZURE_RESOURCE_GROUP}.yaml" +} +# -- + +deploy_capz_cluster() { + + p "Creating client secrets and workload cluster" + + kc_mgmt create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" \ + --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" \ + --namespace "${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}" + + sleep 1 + + kc_mgmt apply -f ./${AZURE_RESOURCE_GROUP}.yaml + + p "Waiting for cluster to be provisioned." + while ! kc_mgmt get cluster | grep "${AZURE_RESOURCE_GROUP}" | grep -i provisioned ; do + sleep 1 + done + + # Hack alert: sometimes ended up with an empty kubeconfig from the command below, so + # we add a defensive sleep. + sleep 2 + + ./clusterctl get kubeconfig ${AZURE_RESOURCE_GROUP} \ + --kubeconfig=./kind-mgmt.kubeconfig \ + --namespace default \ + > ./${AZURE_RESOURCE_GROUP}.kubeconfig + + p "Waiting for all nodes to come up." + local count=0 + local target_count=$((WORKER_CONTROLPLANE_NODES + WORKER_NODES)) + while [ "$count" -lt "$target_count" ] ; do + count="$(kc_worker --request-timeout=5s get nodes \ + | grep "${AZURE_RESOURCE_GROUP}" \ + | wc -l)" + echo "$count of $target_count nodes are up." + done + + local worker_cidr="$(kc_mgmt get cluster "${AZURE_RESOURCE_GROUP}" \ + -o=jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[0]}')" + + p "Deploying Calico to cluster CIDR '$worker_cidr' so worker nodes can talk to each other" + + helm_wrapper() { + ./helm --kubeconfig=./${AZURE_RESOURCE_GROUP}.kubeconfig \ + --registry-config=./${AZURE_RESOURCE_GROUP}.helm-registry.json \ + --repository-cache=./helm-cache \ + --repository-config=./${AZURE_RESOURCE_GROUP}.helm-registry.yaml \ + ${@} + } + + helm_wrapper \ + repo add projectcalico https://docs.tigera.io/calico/charts + + helm_wrapper \ + install calico projectcalico/tigera-operator \ + --version v3.26.1 \ + -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/addons/calico/values.yaml \ + --set-string "installation.calicoNetwork.ipPools[0].cidr=${worker_cidr}" \ + --namespace tigera-operator \ + --create-namespace + + p "Installing Azure cluster controller" + helm_wrapper \ + install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo \ + cloud-provider-azure \ + --generate-name --set infra.clusterName=${AZURE_RESOURCE_GROUP} \ + --set "cloudControllerManager.clusterCIDR=${worker_cidr}" \ + --set-string "cloudControllerManager.caCertDir=/usr/share/ca-certificates" + + p "Your nodes should be ready soon" + kc_worker get nodes + + local bastion="$(kc_mgmt get azurecluster flatcar-capi-demo-azure -o json \ + | jq '.spec.networkSpec.apiServerLB.frontendIPs[0].publicIP.dnsName')" + + p "Cluster is deployed and can now be used ('kc_worker' kubectl wrapper)." + p "You can use 'az serial-console connect -g ${AZURE_RESOURCE_GROUP} -n ' connects via serial console, or\n" \ + "'ssh core@${bastion}' to ssh into the control plane node (if you set AZURE_SSH_PUBLIC_KEY_B64 in azure.env)." +} +# -- + +cleanup() { + kc_mgmt delete cluster ${AZURE_RESOURCE_GROUP} + ./kind delete cluster --kubeconfig=./kind-mgmt.kubeconfig +} +# -- + +help() { + cat < /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: true + dropins: + - name: bootcheck.conf + contents: | + [Timer] + OnBootSec=1min + OnUnitActiveSec=10s + RandomizedDelaySec=1s + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + overwrite: false + partitions: [] + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: true + dropins: + - name: bootcheck.conf + contents: | + [Timer] + OnBootSec=1min + OnUnitActiveSec=10s + RandomizedDelaySec=1s + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + # Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7679. + storage: + disks: + - device: /dev/disk/azure/scsi1/lun0 + partitions: + - number: 1 + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + mounts: + - - etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + version: ${FLATCAR_VERSION} + publisher: kinvolk + offer: flatcar-container-linux-corevm-amd64 + sku: stable-gen2 + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + version: ${FLATCAR_VERSION} + publisher: kinvolk + offer: flatcar-container-linux-corevm-amd64 + sku: stable-gen2 + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE}