diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 57cbeab9472..570c782c4e8 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -115,6 +115,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] useExperimentalRetryJoin: true machineTemplate: diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 349f85572b0..ca20882429b 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -112,6 +112,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index ba475462255..1e272b5b693 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -110,6 +110,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-external-cloud-provider.yaml index 98f040c8ec0..51335d14060 100644 --- a/templates/cluster-template-external-cloud-provider.yaml +++ b/templates/cluster-template-external-cloud-provider.yaml @@ -111,6 +111,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-machinepool-multiple-subnets.yaml b/templates/cluster-template-machinepool-multiple-subnets.yaml index b8edcf02c8f..41a65a70acd 100644 --- a/templates/cluster-template-machinepool-multiple-subnets.yaml +++ b/templates/cluster-template-machinepool-multiple-subnets.yaml @@ -117,6 +117,8 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 diff --git a/templates/cluster-template-machinepool-system-assigned-identity.yaml b/templates/cluster-template-machinepool-system-assigned-identity.yaml index d4ee97ee06f..713fe630729 100644 --- a/templates/cluster-template-machinepool-system-assigned-identity.yaml +++ b/templates/cluster-template-machinepool-system-assigned-identity.yaml @@ -106,6 +106,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-machinepool-user-assigned-identity.yaml b/templates/cluster-template-machinepool-user-assigned-identity.yaml index d42f853d6ca..7201571953b 100644 --- a/templates/cluster-template-machinepool-user-assigned-identity.yaml +++ b/templates/cluster-template-machinepool-user-assigned-identity.yaml @@ -106,6 +106,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 8b01512f2bc..b2877869ad1 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -110,6 +110,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-nat-gateway.yaml b/templates/cluster-template-nat-gateway.yaml index 7de0ed5c00c..3d576256fd9 100644 --- a/templates/cluster-template-nat-gateway.yaml +++ b/templates/cluster-template-nat-gateway.yaml @@ -123,6 +123,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 3bd40a0bccf..84fd0dfa9cb 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -111,6 +111,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-system-assigned-identity.yaml b/templates/cluster-template-system-assigned-identity.yaml index 055268c547b..5b31f689292 100644 --- a/templates/cluster-template-system-assigned-identity.yaml +++ b/templates/cluster-template-system-assigned-identity.yaml @@ -106,6 +106,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template-user-assigned-identity.yaml b/templates/cluster-template-user-assigned-identity.yaml index c66aaab1ce3..a1e8bfcf967 100644 --- a/templates/cluster-template-user-assigned-identity.yaml +++ b/templates/cluster-template-user-assigned-identity.yaml @@ -106,6 +106,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index dc57a6d5c1e..3fe7e4e5989 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -110,6 +110,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 85c243ba1d0..d53df34681b 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -110,6 +110,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk preKubeadmCommands: [] + postKubeadmCommands: [] version: "${KUBERNETES_VERSION}" --- kind: AzureMachineTemplate diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index edf7d256050..86ae4c0c9e9 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -181,6 +181,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh useExperimentalRetryJoin: true diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 5d35fa10061..4b6355cf142 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -126,6 +126,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml index a3755cb0ce0..5e7dd05b658 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml @@ -116,6 +116,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-identity-from-env.yaml b/templates/test/ci/cluster-template-prow-identity-from-env.yaml index b79a350c9b5..74443ef6f33 100644 --- a/templates/test/ci/cluster-template-prow-identity-from-env.yaml +++ b/templates/test/ci/cluster-template-prow-identity-from-env.yaml @@ -111,6 +111,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 3eb007a8dbc..b68877015f7 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -181,6 +181,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh useExperimentalRetryJoin: true diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 8ac6708cf35..16e1377ee3c 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -115,6 +115,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 45d9c5af331..1ffd08c14b8 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -116,6 +116,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 2b9d1a905d2..7575fabae98 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -115,6 +115,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml index ca130e5d252..c043a81c240 100644 --- a/templates/test/dev/cluster-template-custom-builds-windows.yaml +++ b/templates/test/dev/cluster-template-custom-builds-windows.yaml @@ -95,6 +95,25 @@ spec: overwrite: false tableType: gpt files: + - content: | + network: + version: 2 + ethernets: + eth0: + mtu: 1400 + match: + macaddress: MACADDRESS + set-name: eth0 + owner: root:root + path: /etc/netplan/60-eth0.yaml + permissions: "0644" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" - content: | #!/bin/bash @@ -144,13 +163,6 @@ spec: owner: root:root path: /tmp/replace-k8s-components.sh permissions: "0744" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" initConfiguration: nodeRegistration: kubeletExtraArgs: @@ -169,6 +181,9 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: + - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') + - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml + - netplan apply - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - bash -c /tmp/replace-k8s-binaries.sh @@ -260,6 +275,25 @@ spec: template: spec: files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + network: + version: 2 + ethernets: + eth0: + mtu: 1400 + match: + macaddress: MACADDRESS + set-name: eth0 + owner: root:root + path: /etc/netplan/60-eth0.yaml + permissions: "0644" - content: | #!/bin/bash @@ -281,13 +315,6 @@ spec: owner: root:root path: /tmp/replace-k8s-binaries.sh permissions: "0744" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" joinConfiguration: nodeRegistration: kubeletExtraArgs: diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b8550137ef4..19761557c0a 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -94,6 +94,13 @@ spec: overwrite: false tableType: gpt files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" - content: | #!/bin/bash @@ -143,13 +150,6 @@ spec: owner: root:root path: /tmp/replace-k8s-components.sh permissions: "0744" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" initConfiguration: nodeRegistration: kubeletExtraArgs: @@ -259,6 +259,13 @@ spec: template: spec: files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" - content: | #!/bin/bash @@ -280,13 +287,6 @@ spec: owner: root:root path: /tmp/replace-k8s-binaries.sh permissions: "0744" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" joinConfiguration: nodeRegistration: kubeletExtraArgs: diff --git a/templates/test/dev/custom-builds-windows/kustomization.yaml b/templates/test/dev/custom-builds-windows/kustomization.yaml index 2b5c5981b3b..64c71579e49 100644 --- a/templates/test/dev/custom-builds-windows/kustomization.yaml +++ b/templates/test/dev/custom-builds-windows/kustomization.yaml @@ -2,8 +2,8 @@ namespace: default resources: - ../../../test/ci/prow-windows patchesStrategicMerge: - - ../custom-builds/patches/custom-builds.yaml - patches/windows-image-update.yaml + - ../custom-builds/patches/machine-deployment-pr-version.yaml patchesJson6902: - target: group: bootstrap.cluster.x-k8s.io @@ -12,3 +12,16 @@ patchesJson6902: name: .*-md-win namespace: default path: patches/custom-builds-windows.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: ../custom-builds/patches/kubeadm-bootstrap.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../custom-builds/patches/kubeadm-controlplane-bootstrap.yaml diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index 6c7a10f7e90..a29eee3f6c5 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -2,5 +2,18 @@ namespace: default resources: - ../../../test/ci/prow patchesStrategicMerge: - - ../patches/control-plane-custom-builds.yaml - - patches/custom-builds.yaml + - patches/machine-deployment-pr-version.yaml +patches: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: patches/kubeadm-bootstrap.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmControlPlane + name: .*-control-plane + path: patches/kubeadm-controlplane-bootstrap.yaml \ No newline at end of file diff --git a/templates/test/dev/custom-builds/patches/custom-builds.yaml b/templates/test/dev/custom-builds/patches/custom-builds.yaml deleted file mode 100644 index 85b4b4833ff..00000000000 --- a/templates/test/dev/custom-builds/patches/custom-builds.yaml +++ /dev/null @@ -1,143 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 -kind: KubeadmControlPlane -metadata: - name: "${CLUSTER_NAME}-control-plane" - annotations: - controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" -spec: - kubeadmConfigSpec: - useExperimentalRetryJoin: true - clusterConfiguration: - kubernetesVersion: "ci/${CI_VERSION}" - preKubeadmCommands: - - bash -c /tmp/replace-k8s-binaries.sh - postKubeadmCommands: - - bash -c /tmp/replace-k8s-components.sh - files: - - path: /tmp/replace-k8s-binaries.sh - owner: "root:root" - permissions: "0744" - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" - done - systemctl restart kubelet - - # prepull images from gcr.io/k8s-staging-ci-images and retag it to - # k8s.gcr.io so kubeadm can fetch correct images no matter what - declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - for IMAGE in "$${IMAGES[@]}"; do - $${SUDO} ctr -n k8s.io images pull "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" - $${SUDO} ctr -n k8s.io images tag "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" "k8s.gcr.io/$${IMAGE}:${CI_VERSION/+/_}" - done - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true --short=true)" - echo "kubelet version: $(kubelet --version)" - - path: /tmp/replace-k8s-components.sh - owner: "root:root" - permissions: "0744" - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz - tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq - rm /tmp/yq_linux_amd64.tar.gz - - export KUBECONFIG=/etc/kubernetes/admin.conf - kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${IMAGE_TAG}" - yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml - - path: /etc/kubernetes/azure.json - owner: "root:root" - permissions: "0644" - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - preKubeadmCommands: - - bash -c /tmp/replace-k8s-binaries.sh - files: - - path: /tmp/replace-k8s-binaries.sh - owner: "root:root" - permissions: "0744" - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" - done - systemctl restart kubelet - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true --short=true)" - echo "kubelet version: $(kubelet --version)" - - path: /etc/kubernetes/azure.json - owner: "root:root" - permissions: "0644" - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - template: - spec: - image: - # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions. - # 1.18.8 binaries and images will get replaced to the desired version by the script above. - marketplace: - publisher: cncf-upstream - offer: capi - sku: k8s-1dot18dot8-ubuntu-1804 - version: "2020.08.17" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - image: - # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions. - # 1.18.8 binaries and images will get replaced to the desired version by the script above. - marketplace: - publisher: cncf-upstream - offer: capi - sku: k8s-1dot18dot8-ubuntu-1804 - version: "2020.08.17" diff --git a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml new file mode 100644 index 00000000000..42be3a83049 --- /dev/null +++ b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml @@ -0,0 +1,28 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true --short=true)" + echo "kubelet version: $(kubelet --version)" + path: /tmp/replace-k8s-binaries.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-binaries.sh diff --git a/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml new file mode 100644 index 00000000000..f9bee901cb4 --- /dev/null +++ b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml @@ -0,0 +1,63 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + done + systemctl restart kubelet + + # prepull images from gcr.io/k8s-staging-ci-images and retag it to + # k8s.gcr.io so kubeadm can fetch correct images no matter what + declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + for IMAGE in "$${IMAGES[@]}"; do + $${SUDO} ctr -n k8s.io images pull "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" + $${SUDO} ctr -n k8s.io images tag "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" "k8s.gcr.io/$${IMAGE}:${CI_VERSION/+/_}" + done + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true --short=true)" + echo "kubelet version: $(kubelet --version)" + path: /tmp/replace-k8s-binaries.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq + rm /tmp/yq_linux_amd64.tar.gz + + export KUBECONFIG=/etc/kubernetes/admin.conf + kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${IMAGE_TAG}" + yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml + path: /tmp/replace-k8s-components.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-binaries.sh +- op: add + path: /spec/kubeadmConfigSpec/postKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-components.sh \ No newline at end of file diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml new file mode 100644 index 00000000000..18afeadae24 --- /dev/null +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml @@ -0,0 +1,44 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: + # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions. + # 1.18.8 binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: k8s-1dot18dot8-ubuntu-1804 + version: "2020.08.17" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions. + # 1.18.8 binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: k8s-1dot18dot8-ubuntu-1804 + version: "2020.08.17" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 +kind: KubeadmControlPlane +metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ci/${CI_VERSION} + useExperimentalRetryJoin: true \ No newline at end of file