diff --git a/CAPZ-sysext/.gitignore b/CAPZ-sysext/.gitignore new file mode 100644 index 0000000..4b32c8e --- /dev/null +++ b/CAPZ-sysext/.gitignore @@ -0,0 +1,3 @@ +*.swp +demo +azure.env diff --git a/CAPZ-sysext/azure.env.template b/CAPZ-sysext/azure.env.template new file mode 100644 index 0000000..21c6359 --- /dev/null +++ b/CAPZ-sysext/azure.env.template @@ -0,0 +1,20 @@ +# Template for Azure CAPZ settings + +export AZURE_SUBSCRIPTION_ID="TODO add subscription ID" + +# From https://capz.sigs.k8s.io/getting-started: +# az ad sp create-for-rbac --role contributor --scopes="/subscriptions/${AZURE_SUBSCRIPTION_ID}" +export AZURE_TENANT_ID="TODO add 'tenant' from output of az command" +export AZURE_CLIENT_ID="TODO add 'appId from output of az command'" +export AZURE_CLIENT_SECRET="TODO add 'password' from output of az command" + +# +# These usually do not need to be touched +# + +export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="${AZURE_CLIENT_ID}" # for compatibility with CAPZ v1.16 templates + +# AZURE_RESOURCE_GROUP is set in capz-demo.env +export AZURE_CLUSTER_IDENTITY_SECRET_NAME="${AZURE_RESOURCE_GROUP}-cluster-identity-secret" +export CLUSTER_IDENTITY_NAME="${AZURE_RESOURCE_GROUP}-cluster-identity" +export AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE="default" diff --git a/CAPZ-sysext/capz-demo.env b/CAPZ-sysext/capz-demo.env new file mode 100644 index 0000000..b86aef3 --- /dev/null +++ b/CAPZ-sysext/capz-demo.env @@ -0,0 +1,213 @@ +#!/bin/bash + +KUBERNETES_VERSION="v1.30.0" + +AZURE_RESOURCE_GROUP="flatcar-capi-demo-azure" +export AZURE_RESOURCE_GROUP + +WORKER_CONTROLPLANE_NODES=1 +WORKER_NODES=2 + +p() { + echo + echo "#####################################" + echo "${@}" + echo "-------------------------------------" +} +# -- + +check_command() { + local cmd="$1" + + if ! command -v "$cmd" &> /dev/null ; then + echo "'$cmd' could not be found. Please install your distro's '$cmd'." + return 1 + fi + echo " - '$cmd'" +} +# -- + +check_file() { + local f="$1" + + if [ ! -f "./$f" ] ; then + echo "prerequisite '$f' could not be found." + return 1 + fi + + echo " - '$f'" +} +# -- + +get_prerequisites() { + p "Prerequisites: Checking for required host commands." + check_command "kubectl" || return + check_command "yq" || return + check_command "wget" || return + + p "Prerequisites: Checking for prerequisite files." + check_file "azure.env" || return + check_file "cluster-template-flatcar-sysext.yaml" || return + + if [ ! -f ./clusterctl ] ; then + p "Prerequisites: fetching clusterctl" + wget https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/clusterctl-linux-amd64 + mv clusterctl-linux-amd64 clusterctl + chmod 755 clusterctl + fi + + if [ ! -f ./kind ] ; then + p "Prerequisites: fetching kind" + wget https://github.com/kubernetes-sigs/kind/releases/latest/download/kind-linux-amd64 + mv kind-linux-amd64 kind + chmod 755 kind + fi + + if [ ! -f ./helm ] ; then + p "Prerequisites: fetching helm" + curl https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz \ + | tar xz linux-amd64/helm -O >helm + chmod 755 helm + mkdir -p helm-cache + fi +} +# -- + +setup_kind_cluster() { + p "Bootsstrapping cluster" + ./kind create cluster --kubeconfig=./kind-mgmt.kubeconfig + export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true +} +# -- + +kc_mgmt() { + kubectl --kubeconfig=./kind-mgmt.kubeconfig "${@}" + +} +# -- + +kc_worker() { + kubectl --kubeconfig=./${AZURE_RESOURCE_GROUP}.kubeconfig "${@}" +} +# -- + +generate_capz_yaml() { + source ./azure.env + + p "Initialising ClusterAPI Azure provider." + ./clusterctl init --infrastructure azure --kubeconfig=./kind-mgmt.kubeconfig + + # FIXME: add + # --infrastructure azure \ + # --flavor flatcar-sysext \ + # and remove + # --from cluster-template-flatcar-sysext.yaml \ + # when https://github.com/kubernetes-sigs/cluster-api-provider-azure/pull/4575 is merged. + + # TODO: remove when PR is merged + export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3" + export AZURE_LOCATION="northeurope" + export AZURE_NODE_MACHINE_TYPE="Standard_D2s_v3" + export FLATCAR_VERSION="latest" + export CI_RG="${AZURE_RESOURCE_GROUP}" + + p "Generating ${AZURE_RESOURCE_GROUP}.yaml." + ./clusterctl generate cluster ${AZURE_RESOURCE_GROUP} \ + --from cluster-template-flatcar-sysext.yaml \ + --kubeconfig=./kind-mgmt.kubeconfig \ + --kubernetes-version "${KUBERNETES_VERSION}" \ + --control-plane-machine-count=${WORKER_CONTROLPLANE_NODES} \ + --worker-machine-count=${WORKER_NODES} \ + > ${AZURE_RESOURCE_GROUP}.yaml + + yq -i "with(. | select(.kind == \"AzureClusterIdentity\"); .spec.type |= \"ServicePrincipal\" | .spec.clientSecret.name |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}\" | .spec.clientSecret.namespace |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}\")" "${AZURE_RESOURCE_GROUP}.yaml" +} +# -- + +deploy_capz_cluster() { + + p "Creating client secrets and workload cluster" + + kc_mgmt create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" \ + --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" \ + --namespace "${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}" + kc_mgmt apply -f ./${AZURE_RESOURCE_GROUP}.yaml + + p "Waiting for cluster to be provisioned." + while ! kc_mgmt get cluster | grep "${AZURE_RESOURCE_GROUP}" | grep -i provisioned ; do + sleep 1 + done + + # Hack alert: sometimes ended up with an empty kubeconfig from the command below, so + # we add a defensive sleep. + sleep 2 + + ./clusterctl get kubeconfig ${AZURE_RESOURCE_GROUP} \ + --kubeconfig=./kind-mgmt.kubeconfig \ + --namespace default \ + > ./${AZURE_RESOURCE_GROUP}.kubeconfig + + p "Waiting for all nodes to come up." + local count=0 + local target_count=$((WORKER_CONTROLPLANE_NODES + WORKER_NODES)) + while [ "$count" -lt "$target_count" ] ; do + count="$(kc_worker --request-timeout=5s get nodes \ + | grep "${AZURE_RESOURCE_GROUP}" \ + | wc -l)" + echo "$count of $target_count nodes are up." + done + + local worker_cidr="$(kc_mgmt get cluster "${AZURE_RESOURCE_GROUP}" \ + -o=jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[0]}')" + + p "Deploying Calico to cluster CIDR '$worker_cidr' so worker nodes can talk to each other" + + ./helm --kubeconfig=./${AZURE_RESOURCE_GROUP}.kubeconfig \ + --registry-config=./${AZURE_RESOURCE_GROUP}.helm-registry.json \ + --repository-cache=./helm-cache \ + --repository-config=./${AZURE_RESOURCE_GROUP}.helm-registry.yaml \ + repo add projectcalico https://docs.tigera.io/calico/charts + + ./helm --kubeconfig=./${AZURE_RESOURCE_GROUP}.kubeconfig \ + --registry-config=./${AZURE_RESOURCE_GROUP}.helm-registry.json \ + --repository-cache=./helm-cache \ + --repository-config=./${AZURE_RESOURCE_GROUP}.helm-registry.yaml \ + install calico projectcalico/tigera-operator \ + --version v3.26.1 \ + -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/addons/calico/values.yaml \ + --set-string "installation.calicoNetwork.ipPools[0].cidr=${worker_cidr}" \ + --namespace tigera-operator \ + --create-namespace + + p "Your nodes should be ready soon" + kc_worker get nodes + + p "Cluster is deployed and can now be used ('kc_worker' kubectl wrapper)." + p "'az serial-console connect -g ${AZURE_RESOURCE_GROUP} -n ' connects via serial console." +} +# -- + +cleanup() { + kc_mgmt delete cluster ${AZURE_RESOURCE_GROUP} + ./kind delete cluster --kubeconfig=./kind-mgmt.kubeconfig +} +# -- + +help() { + cat < /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: bootcheck.conf + contents: | + [Timer] + OnBootSec=1min + OnUnitActiveSec=10s + RandomizedDelaySec=1s + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: true + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + overwrite: false + partitions: [] + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: bootcheck.conf + contents: | + [Timer] + OnBootSec=1min + OnUnitActiveSec=10s + RandomizedDelaySec=1s + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: true + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + # Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7679. + storage: + disks: + - device: /dev/disk/azure/scsi1/lun0 + partitions: + - number: 1 + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + mounts: + - - etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + version: ${FLATCAR_VERSION} + publisher: kinvolk + offer: flatcar-container-linux-corevm-amd64 + sku: stable-gen2 + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + version: ${FLATCAR_VERSION} + publisher: kinvolk + offer: flatcar-container-linux-corevm-amd64 + sku: stable-gen2 + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE}