From edb48c4242d47f953ea18e9326ad86d253951121 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 26 Jan 2022 10:20:14 -0800 Subject: [PATCH] More external cloud-provider-azure --- .../ccm-resource-set.yaml | 0 .../cloud-controller-manager.yaml | 200 ++++++++++ .../cloud-node-manager.yaml | 0 .../kustomization.yaml | 4 - .../ccm-resource-set.yaml | 27 ++ .../cloud-controller-manager.yaml | 0 .../cloud-node-manager.yaml | 85 +++++ .../cloud-provider-azure/kustomization.yaml | 17 + templates/cluster-template-aad.yaml | 342 ++++++++++++++++- templates/cluster-template-azure-bastion.yaml | 344 ++++++++++++++++- templates/cluster-template-ephemeral.yaml | 344 ++++++++++++++++- ...ster-template-in-tree-cloud-provider.yaml} | 97 +++-- templates/cluster-template-ipv6.yaml | 346 +++++++++++++++++- ...template-machinepool-multiple-subnets.yaml | 302 --------------- ...te-machinepool-user-assigned-identity.yaml | 213 ----------- ...mplate-machinepool-windows-containerd.yaml | 344 ++++++++++++++++- .../cluster-template-machinepool-windows.yaml | 346 +++++++++++++++++- templates/cluster-template-machinepool.yaml | 344 ++++++++++++++++- templates/cluster-template-nvidia-gpu.yaml | 344 ++++++++++++++++- templates/cluster-template-private.yaml | 344 ++++++++++++++++- ...ter-template-system-assigned-identity.yaml | 344 ++++++++++++++++- ...uster-template-user-assigned-identity.yaml | 344 ++++++++++++++++- .../cluster-template-windows-containerd.yaml | 344 ++++++++++++++++- templates/cluster-template-windows.yaml | 342 ++++++++++++++++- templates/cluster-template.yaml | 344 ++++++++++++++++- templates/flavors/aad/kustomization.yaml | 1 + .../aad/patches/kubeadm-controlplane.yaml | 6 +- templates/flavors/base/cluster-template.yaml | 7 +- templates/flavors/default/kustomization.yaml | 1 + .../flavors/default/machine-deployment.yaml | 2 +- .../in-tree-cloud-provider/kustomization.yaml | 9 + .../machine-deployment.yaml | 60 +++ .../patches/in-tree-cloud-provider.yaml} | 21 +- templates/flavors/ipv6/kustomization.yaml | 1 + .../flavors/ipv6/machine-deployment.yaml | 2 +- .../ipv6/patches/kubeadm-controlplane.yaml | 8 +- .../kustomization.yaml | 7 - .../machine-pool-deployment.yaml | 64 ---- .../patches/azurecluster-subnets.yaml | 18 - .../patches/machine-pool-subnet.yaml | 8 - .../kustomization.yaml | 6 - .../machine-pool-deployment.yaml | 63 ---- .../patches/system-assigned-identity.yaml | 7 - .../kustomization.yaml | 6 - .../machine-pool-deployment.yaml | 63 ---- .../patches/user-assigned-identity.yaml | 9 - .../machinepool-windows/kustomization.yaml | 1 + .../machine-pool-deployment-windows.yaml | 2 +- .../machine-pool-deployment.yaml | 2 +- .../flavors/machinepool/kustomization.yaml | 1 + .../machinepool/machine-pool-deployment.yaml | 2 +- .../flavors/nvidia-gpu/kustomization.yaml | 1 + .../nvidia-gpu/machine-deployment.yaml | 2 +- templates/flavors/private/kustomization.yaml | 1 + .../kustomization.yaml | 2 + .../user-assigned-identity/kustomization.yaml | 2 + templates/flavors/windows/kustomization.yaml | 1 + ...ow-ci-version-windows-containerd-2022.yaml | 344 ++++++++++++++++- ...ster-template-prow-ci-version-windows.yaml | 342 ++++++++++++++++- .../ci/cluster-template-prow-ci-version.yaml | 344 ++++++++++++++++- .../ci/cluster-template-prow-custom-vnet.yaml | 344 ++++++++++++++++- ...template-prow-in-tree-cloud-provider.yaml} | 345 +---------------- .../test/ci/cluster-template-prow-ipv6.yaml | 346 +++++++++++++++++- ...template-prow-machine-pool-ci-version.yaml | 344 ++++++++++++++++- ...er-template-prow-machine-pool-windows.yaml | 346 +++++++++++++++++- .../cluster-template-prow-machine-pool.yaml | 344 ++++++++++++++++- .../ci/cluster-template-prow-nvidia-gpu.yaml | 344 ++++++++++++++++- .../ci/cluster-template-prow-private.yaml | 344 ++++++++++++++++- .../ci/cluster-template-prow-windows.yaml | 342 ++++++++++++++++- templates/test/ci/cluster-template-prow.yaml | 344 ++++++++++++++++- .../kustomization.yaml | 2 +- templates/test/ci/prow/kustomization.yaml | 1 + ...r-template-custom-builds-machine-pool.yaml | 344 ++++++++++++++++- ...luster-template-custom-builds-windows.yaml | 342 ++++++++++++++++- .../dev/cluster-template-custom-builds.yaml | 344 ++++++++++++++++- test/e2e/azure_test.go | 4 +- test/e2e/config/azure-dev.yaml | 4 +- 77 files changed, 10369 insertions(+), 1292 deletions(-) rename templates/{flavors/external-cloud-provider => cloud-provider-azure-ipv6}/ccm-resource-set.yaml (100%) create mode 100644 templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml rename templates/{flavors/external-cloud-provider => cloud-provider-azure-ipv6}/cloud-node-manager.yaml (100%) rename templates/{flavors/external-cloud-provider => cloud-provider-azure-ipv6}/kustomization.yaml (81%) create mode 100644 templates/cloud-provider-azure/ccm-resource-set.yaml rename templates/{flavors/external-cloud-provider => cloud-provider-azure}/cloud-controller-manager.yaml (100%) create mode 100644 templates/cloud-provider-azure/cloud-node-manager.yaml create mode 100644 templates/cloud-provider-azure/kustomization.yaml rename templates/{cluster-template-machinepool-system-assigned-identity.yaml => cluster-template-in-tree-cloud-provider.yaml} (73%) delete mode 100644 templates/cluster-template-machinepool-multiple-subnets.yaml delete mode 100644 templates/cluster-template-machinepool-user-assigned-identity.yaml create mode 100644 templates/flavors/in-tree-cloud-provider/kustomization.yaml create mode 100644 templates/flavors/in-tree-cloud-provider/machine-deployment.yaml rename templates/flavors/{external-cloud-provider/patches/external-cloud-provider.yaml => in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml} (66%) delete mode 100644 templates/flavors/machinepool-multiple-subnets/kustomization.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/kustomization.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/kustomization.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml rename templates/test/ci/{cluster-template-prow-external-cloud-provider.yaml => cluster-template-prow-in-tree-cloud-provider.yaml} (96%) rename templates/test/ci/{prow-external-cloud-provider => prow-in-tree-cloud-provider}/kustomization.yaml (91%) diff --git a/templates/flavors/external-cloud-provider/ccm-resource-set.yaml b/templates/cloud-provider-azure-ipv6/ccm-resource-set.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/ccm-resource-set.yaml rename to templates/cloud-provider-azure-ipv6/ccm-resource-set.yaml diff --git a/templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml b/templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml new file mode 100644 index 000000000000..e22798cd24fc --- /dev/null +++ b/templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml @@ -0,0 +1,200 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager +--- +apiVersion: v1 +kind: Pod +metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager +spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=2001:1234:5678:9a40::/58" + - "--bind-address=::" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--node-cidr-mask-size=0" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings diff --git a/templates/flavors/external-cloud-provider/cloud-node-manager.yaml b/templates/cloud-provider-azure-ipv6/cloud-node-manager.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/cloud-node-manager.yaml rename to templates/cloud-provider-azure-ipv6/cloud-node-manager.yaml diff --git a/templates/flavors/external-cloud-provider/kustomization.yaml b/templates/cloud-provider-azure-ipv6/kustomization.yaml similarity index 81% rename from templates/flavors/external-cloud-provider/kustomization.yaml rename to templates/cloud-provider-azure-ipv6/kustomization.yaml index 2e6ae5e2d99c..6c5e52528343 100644 --- a/templates/flavors/external-cloud-provider/kustomization.yaml +++ b/templates/cloud-provider-azure-ipv6/kustomization.yaml @@ -1,11 +1,7 @@ namespace: default resources: - - ../default - ccm-resource-set.yaml -patchesStrategicMerge: - - patches/external-cloud-provider.yaml - configMapGenerator: - name: cloud-controller-manager-addon files: diff --git a/templates/cloud-provider-azure/ccm-resource-set.yaml b/templates/cloud-provider-azure/ccm-resource-set.yaml new file mode 100644 index 000000000000..d475d9e4abaf --- /dev/null +++ b/templates/cloud-provider-azure/ccm-resource-set.yaml @@ -0,0 +1,27 @@ +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + strategy: "ApplyOnce" + clusterSelector: + matchLabels: + ccm: external + resources: + - name: cloud-controller-manager-addon + kind: ConfigMap +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + strategy: "ApplyOnce" + clusterSelector: + matchLabels: + ccm: external + resources: + - name: cloud-node-manager-addon + kind: ConfigMap diff --git a/templates/flavors/external-cloud-provider/cloud-controller-manager.yaml b/templates/cloud-provider-azure/cloud-controller-manager.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/cloud-controller-manager.yaml rename to templates/cloud-provider-azure/cloud-controller-manager.yaml diff --git a/templates/cloud-provider-azure/cloud-node-manager.yaml b/templates/cloud-provider-azure/cloud-node-manager.yaml new file mode 100644 index 000000000000..b5267af1a747 --- /dev/null +++ b/templates/cloud-provider-azure/cloud-node-manager.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager +subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager +spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi diff --git a/templates/cloud-provider-azure/kustomization.yaml b/templates/cloud-provider-azure/kustomization.yaml new file mode 100644 index 000000000000..6c5e52528343 --- /dev/null +++ b/templates/cloud-provider-azure/kustomization.yaml @@ -0,0 +1,17 @@ +namespace: default +resources: + - ccm-resource-set.yaml + +configMapGenerator: + - name: cloud-controller-manager-addon + files: + - cloud-controller-manager.yaml + - name: cloud-node-manager-addon + files: + - cloud-node-manager.yaml +generatorOptions: + disableNameSuffixHash: true + labels: + type: generated + annotations: + note: generated diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index faf72fd939de..c8673e1ddc07 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -112,14 +113,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -233,3 +234,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 6109eef6addb..f478cffeea6e 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,7 +209,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -227,3 +228,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 466ba8dc9a60..57b885c222a6 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -231,3 +232,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-machinepool-system-assigned-identity.yaml b/templates/cluster-template-in-tree-cloud-provider.yaml similarity index 73% rename from templates/cluster-template-machinepool-system-assigned-identity.yaml rename to templates/cluster-template-in-tree-cloud-provider.yaml index f9354a6b7046..4694f4bf0ccf 100644 --- a/templates/cluster-template-machinepool-system-assigned-identity.yaml +++ b/templates/cluster-template-in-tree-cloud-provider.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: none cni: calico name: ${CLUSTER_NAME} namespace: default @@ -25,6 +26,10 @@ metadata: name: ${CLUSTER_NAME} namespace: default spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: subnets: @@ -144,68 +149,80 @@ spec: vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool +kind: MachineDeployment metadata: - name: ${CLUSTER_NAME}-mp-0 + name: ${CLUSTER_NAME}-md-0 namespace: default spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool +kind: AzureMachineTemplate metadata: - name: ${CLUSTER_NAME}-mp-0 + name: ${CLUSTER_NAME}-md-0 namespace: default spec: - identity: SystemAssigned - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig +kind: KubeadmConfigTemplate metadata: - name: ${CLUSTER_NAME}-mp-0 + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} namespace: default spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index e33503d50e80..7c86fa3cab82 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default @@ -77,7 +78,7 @@ spec: allocate-node-cidrs: "true" bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" @@ -127,7 +128,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -140,7 +141,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -197,6 +198,343 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=2001:1234:5678:9a40::/58" + - "--bind-address=::" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--node-cidr-mask-size=0" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: @@ -269,7 +607,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: '[fd00::10]' node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-multiple-subnets.yaml b/templates/cluster-template-machinepool-multiple-subnets.yaml deleted file mode 100644 index 6c990d206382..000000000000 --- a/templates/cluster-template-machinepool-multiple-subnets.yaml +++ /dev/null @@ -1,302 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: ${CLUSTER_NAME}-mp-0 - natGateway: - name: node-natgateway-0 - role: node - - name: ${CLUSTER_NAME}-mp-1 - natGateway: - name: node-natgateway-1 - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: ${CLUSTER_NAME}-mp-0 - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-1 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-1 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: ${CLUSTER_NAME}-mp-1 - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-user-assigned-identity.yaml b/templates/cluster-template-machinepool-user-assigned-identity.yaml deleted file mode 100644 index 811ee478b56e..000000000000 --- a/templates/cluster-template-machinepool-user-assigned-identity.yaml +++ /dev/null @@ -1,213 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: ${USER_ASSIGNED_IDENTITY_PROVIDER_ID} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-windows-containerd.yaml b/templates/cluster-template-machinepool-windows-containerd.yaml index 9d429f874804..f398adef8ddd 100644 --- a/templates/cluster-template-machinepool-windows-containerd.yaml +++ b/templates/cluster-template-machinepool-windows-containerd.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico windows: enabled name: ${CLUSTER_NAME} @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -231,6 +232,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 8a75d813a0b7..57518a3ec66c 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: flannel-windows name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" extraVolumes: @@ -120,14 +121,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -248,7 +249,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') @@ -310,7 +311,7 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - content: |- + - content: | # required as a work around for Flannel and Wins bugs # https://github.com/coreos/flannel/issues/1359 # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 @@ -335,3 +336,338 @@ spec: name: capi sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 4a3b2bedf8c5..a4195d9aa65a 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -210,7 +211,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -228,3 +229,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 3f7a4a58610a..877b19bd2cdf 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico gpu: nvidia name: ${CLUSTER_NAME} @@ -66,7 +67,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -108,14 +109,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -225,7 +226,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: addons.cluster.x-k8s.io/v1beta1 @@ -244,6 +245,341 @@ spec: name: nvidia-gpu-operator-components strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: clusterpolicy-crd.yaml: |+ diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index ab4972d72e8e..7eb79861e87b 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -74,7 +75,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -116,14 +117,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -220,7 +221,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -239,3 +240,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-system-assigned-identity.yaml b/templates/cluster-template-system-assigned-identity.yaml index 8e1c1fd65350..855edacff8f6 100644 --- a/templates/cluster-template-system-assigned-identity.yaml +++ b/templates/cluster-template-system-assigned-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -61,7 +62,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -103,14 +104,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -204,6 +205,341 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-user-assigned-identity.yaml b/templates/cluster-template-user-assigned-identity.yaml index cb2b2e89e75c..7ed6b1cfd670 100644 --- a/templates/cluster-template-user-assigned-identity.yaml +++ b/templates/cluster-template-user-assigned-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -61,7 +62,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -103,14 +104,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,6 +209,341 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-windows-containerd.yaml b/templates/cluster-template-windows-containerd.yaml index 0cac91d6b259..943491a1cf68 100644 --- a/templates/cluster-template-windows-containerd.yaml +++ b/templates/cluster-template-windows-containerd.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico windows: enabled name: ${CLUSTER_NAME} @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,7 +209,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -228,6 +229,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 7d5a173e4461..2b00b48a616d 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: flannel-windows name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" extraVolumes: @@ -120,14 +121,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -346,3 +347,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a6292ee8ee9e..d8d5d5f81679 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -206,7 +207,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -225,3 +226,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/flavors/aad/kustomization.yaml b/templates/flavors/aad/kustomization.yaml index c575fef432e4..5a6c662a9701 100644 --- a/templates/flavors/aad/kustomization.yaml +++ b/templates/flavors/aad/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - patches/kubeadm-controlplane.yaml - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/aad/patches/kubeadm-controlplane.yaml b/templates/flavors/aad/patches/kubeadm-controlplane.yaml index 403abc84c036..64dd8380a736 100644 --- a/templates/flavors/aad/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/aad/patches/kubeadm-controlplane.yaml @@ -9,14 +9,14 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: @@ -32,5 +32,5 @@ spec: oidc-username-prefix: "-" controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 82bfb6f4cdb5..df5298ea6c96 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -5,6 +5,7 @@ metadata: name: ${CLUSTER_NAME} labels: cni: "calico" + ccm: "external" spec: clusterNetwork: pods: @@ -53,14 +54,14 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: @@ -76,7 +77,7 @@ spec: readOnly: true controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json allocate-node-cidrs: "false" cluster-name: ${CLUSTER_NAME} diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index 2afb6f7ff4bf..13d010a66590 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index c2ce6a416a4f..5147e8e09f95 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -47,7 +47,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/in-tree-cloud-provider/kustomization.yaml b/templates/flavors/in-tree-cloud-provider/kustomization.yaml new file mode 100644 index 000000000000..b07cdc8fe844 --- /dev/null +++ b/templates/flavors/in-tree-cloud-provider/kustomization.yaml @@ -0,0 +1,9 @@ +namespace: default +resources: + - ../base + - machine-deployment.yaml + - ../../azure-cluster-identity + +patchesStrategicMerge: + - ../../azure-cluster-identity/azurecluster-identity-ref.yaml + - patches/in-tree-cloud-provider.yaml diff --git a/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml b/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml new file mode 100644 index 000000000000..c2ce6a416a4f --- /dev/null +++ b/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + vmSize: ${AZURE_NODE_MACHINE_TYPE} + osDisk: + osType: "Linux" + diskSizeGB: 128 + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + preKubeadmCommands: [] + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data["local_hostname"] }}' + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + azure-container-registry-config: /etc/kubernetes/azure.json + files: + - contentFrom: + secret: + name: ${CLUSTER_NAME}-md-0-azure-json + key: worker-node-azure.json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" diff --git a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml similarity index 66% rename from templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml rename to templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml index ee196d62c9c1..06a8d4ec771d 100644 --- a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml +++ b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml @@ -4,8 +4,8 @@ kind: Cluster metadata: name: ${CLUSTER_NAME} labels: - cni: "calico" - ccm: "external" + cni: calico + ccm: none --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -16,20 +16,24 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: external + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: external + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: apiServer: - timeoutForControlPlane: 20m + extraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json controllerManager: extraArgs: - cloud-provider: external - version: "${KUBERNETES_VERSION}" + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -41,5 +45,6 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: external + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json diff --git a/templates/flavors/ipv6/kustomization.yaml b/templates/flavors/ipv6/kustomization.yaml index 20b7e499b7bb..c4cc2c5c4aa5 100644 --- a/templates/flavors/ipv6/kustomization.yaml +++ b/templates/flavors/ipv6/kustomization.yaml @@ -2,6 +2,7 @@ namespace: default resources: - ../base - ../../azure-cluster-identity + - ../../cloud-provider-azure-ipv6 - machine-deployment.yaml patchesStrategicMerge: diff --git a/templates/flavors/ipv6/machine-deployment.yaml b/templates/flavors/ipv6/machine-deployment.yaml index b843f63e7c8a..239ad3b9c410 100644 --- a/templates/flavors/ipv6/machine-deployment.yaml +++ b/templates/flavors/ipv6/machine-deployment.yaml @@ -54,7 +54,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" diff --git a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml index 5aa34de5beaf..87e908867032 100644 --- a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml @@ -14,7 +14,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" @@ -26,7 +26,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" @@ -44,11 +44,11 @@ spec: bind-address: "::" controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json #required for ipv6 using calico allocate-node-cidrs: "true" - cluster-cidr: "2001:1234:5678:9a40::/58" + cluster-cidr: "2001:1234:5678:9a40::/58" # TODO we may or may not need this w/ out of tree configure-cloud-routes: "true" bind-address: "::" scheduler: diff --git a/templates/flavors/machinepool-multiple-subnets/kustomization.yaml b/templates/flavors/machinepool-multiple-subnets/kustomization.yaml deleted file mode 100644 index 4cd6ffe71d97..000000000000 --- a/templates/flavors/machinepool-multiple-subnets/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: default -resources: - - ../machinepool - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/machine-pool-subnet.yaml - - patches/azurecluster-subnets.yaml diff --git a/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml b/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml deleted file mode 100644 index c925c051278b..000000000000 --- a/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-1" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-1" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: "${CLUSTER_NAME}-mp-1" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml b/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml deleted file mode 100644 index 3ad2d21e456e..000000000000 --- a/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} -spec: - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: "${CLUSTER_NAME}-mp-0" - role: node - natGateway: - name: node-natgateway-0 - - name: "${CLUSTER_NAME}-mp-1" - role: node - natGateway: - name: node-natgateway-1 diff --git a/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml b/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml deleted file mode 100644 index 095211259844..000000000000 --- a/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - template: - subnetName: "${CLUSTER_NAME}-mp-0" diff --git a/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml b/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml deleted file mode 100644 index 6600b421357d..000000000000 --- a/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../base - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/system-assigned-identity.yaml diff --git a/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml b/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml deleted file mode 100644 index cf3e50b6a6b0..000000000000 --- a/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml b/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml deleted file mode 100644 index e7aae3adefba..000000000000 --- a/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - identity: SystemAssigned diff --git a/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml b/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml deleted file mode 100644 index 2a4f7e06d0ab..000000000000 --- a/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../base - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/user-assigned-identity.yaml diff --git a/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml b/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml deleted file mode 100644 index cf3e50b6a6b0..000000000000 --- a/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml b/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml deleted file mode 100644 index 83c81d94fae0..000000000000 --- a/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - identity: UserAssigned - userAssignedIdentities: - - providerID: ${USER_ASSIGNED_IDENTITY_PROVIDER_ID} diff --git a/templates/flavors/machinepool-windows/kustomization.yaml b/templates/flavors/machinepool-windows/kustomization.yaml index 586f3a62e830..aa83fd1b17f9 100644 --- a/templates/flavors/machinepool-windows/kustomization.yaml +++ b/templates/flavors/machinepool-windows/kustomization.yaml @@ -4,6 +4,7 @@ resources: - ../../azure-cluster-identity - machine-pool-deployment.yaml - machine-pool-deployment-windows.yaml + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml index fdd224f7b94f..2b8f5e30c14d 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml @@ -74,4 +74,4 @@ spec: # https://github.com/coreos/flannel/issues/1359 # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) \ No newline at end of file + New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml index dab4b2719372..2ab9fc078893 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml @@ -53,7 +53,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/machinepool/kustomization.yaml b/templates/flavors/machinepool/kustomization.yaml index ba038c599d5a..70b29875b230 100644 --- a/templates/flavors/machinepool/kustomization.yaml +++ b/templates/flavors/machinepool/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-pool-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/machinepool/machine-pool-deployment.yaml b/templates/flavors/machinepool/machine-pool-deployment.yaml index cf3e50b6a6b0..85ad548ed9ea 100644 --- a/templates/flavors/machinepool/machine-pool-deployment.yaml +++ b/templates/flavors/machinepool/machine-pool-deployment.yaml @@ -50,7 +50,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/nvidia-gpu/kustomization.yaml b/templates/flavors/nvidia-gpu/kustomization.yaml index 4b2741ea6c9c..ded5830dafb9 100644 --- a/templates/flavors/nvidia-gpu/kustomization.yaml +++ b/templates/flavors/nvidia-gpu/kustomization.yaml @@ -4,6 +4,7 @@ resources: - ../../azure-cluster-identity - machine-deployment.yaml - gpu-operator-resources-set.yaml + - ../../cloud-provider-azure patchesStrategicMerge: - patches/cluster.yaml diff --git a/templates/flavors/nvidia-gpu/machine-deployment.yaml b/templates/flavors/nvidia-gpu/machine-deployment.yaml index 88e97237f233..e37b8da3e446 100644 --- a/templates/flavors/nvidia-gpu/machine-deployment.yaml +++ b/templates/flavors/nvidia-gpu/machine-deployment.yaml @@ -48,7 +48,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml index 60660bd25bce..8678ccedb52b 100644 --- a/templates/flavors/private/kustomization.yaml +++ b/templates/flavors/private/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - ../default/machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/system-assigned-identity/kustomization.yaml b/templates/flavors/system-assigned-identity/kustomization.yaml index f61e418f8dd9..f020b5d6b6a3 100644 --- a/templates/flavors/system-assigned-identity/kustomization.yaml +++ b/templates/flavors/system-assigned-identity/kustomization.yaml @@ -2,5 +2,7 @@ namespace: default resources: - ../base - ../default/machine-deployment.yaml + - ../../cloud-provider-azure + patchesStrategicMerge: - patches/system-assigned-identity.yaml diff --git a/templates/flavors/user-assigned-identity/kustomization.yaml b/templates/flavors/user-assigned-identity/kustomization.yaml index 981d43513ed3..b401b6f388eb 100644 --- a/templates/flavors/user-assigned-identity/kustomization.yaml +++ b/templates/flavors/user-assigned-identity/kustomization.yaml @@ -2,5 +2,7 @@ namespace: default resources: - ../base - ../default/machine-deployment.yaml + - ../../cloud-provider-azure + patchesStrategicMerge: - patches/user-assigned-identity.yaml diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index a5902fd9ea85..be33dd740f4f 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -4,6 +4,7 @@ resources: - machine-deployment.yaml - machine-deployment-windows.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml index 763876d55921..9bf99ede2799 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico containerd-logger: enabled metrics-server: enabled @@ -72,7 +73,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} feature-gates: HPAContainerMetrics=true v: "4" @@ -182,14 +183,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -362,7 +363,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -546,6 +547,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml index 2565aa1fca12..d376c0758cc2 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -69,7 +70,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -191,14 +192,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -536,6 +537,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 9d75eac7d284..99b3a796071c 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico containerd-logger: enabled metrics-server: enabled @@ -72,7 +73,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} feature-gates: HPAContainerMetrics=true v: "4" @@ -182,14 +183,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -362,7 +363,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -546,6 +547,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 650f6d1dcae0..dd52b098cce7 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -74,7 +75,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -117,14 +118,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -218,7 +219,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -238,6 +239,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-in-tree-cloud-provider.yaml similarity index 96% rename from templates/test/ci/cluster-template-prow-external-cloud-provider.yaml rename to templates/test/ci/cluster-template-prow-in-tree-cloud-provider.yaml index f7b810a520fe..90450f93304c 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-in-tree-cloud-provider.yaml @@ -2,7 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: - ccm: external + ccm: none cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -71,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -114,14 +114,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -213,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -235,341 +235,6 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet -metadata: - name: crs-ccm - namespace: default -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-node-manager - namespace: default -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-node-manager-addon - strategy: ApplyOnce ---- -apiVersion: v1 -data: - cloud-controller-manager.yaml: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - k8s-app: cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - "*" - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - kind: User - name: cloud-controller-manager - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: system:cloud-controller-manager:extension-apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - apiGroup: "" - kind: User - name: cloud-controller-manager - --- - apiVersion: v1 - kind: Pod - metadata: - name: cloud-controller-manager - namespace: kube-system - labels: - tier: control-plane - component: cloud-controller-manager - spec: - priorityClassName: system-node-critical - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - serviceAccountName: cloud-controller-manager - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - containers: - - name: cloud-controller-manager - image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} - imagePullPolicy: IfNotPresent - command: ["cloud-controller-manager"] - args: - - "--allocate-node-cidrs=true" - - "--cloud-config=/etc/kubernetes/azure.json" - - "--cloud-provider=azure" - - "--cluster-cidr=10.244.0.0/16" - - "--cluster-name=${CLUSTER_NAME}" - - "--controllers=*,-cloud-node" # disable cloud-node controller - - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins - - "--leader-elect=true" - - "--route-reconciliation-period=10s" - - "--v=2" - - "--port=10267" - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: "4" - memory: 2Gi - livenessProbe: - httpGet: - path: /healthz - port: 10267 - initialDelaySeconds: 20 - periodSeconds: 10 - timeoutSeconds: 5 - volumeMounts: - - name: etc-kubernetes - mountPath: /etc/kubernetes - - name: etc-ssl - mountPath: /etc/ssl - readOnly: true - - name: msi - mountPath: /var/lib/waagent/ManagedIdentity-Settings - readOnly: true - volumes: - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes - - name: etc-ssl - hostPath: - path: /etc/ssl - - name: msi - hostPath: - path: /var/lib/waagent/ManagedIdentity-Settings -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cloud-controller-manager-addon - namespace: default ---- -apiVersion: v1 -data: - cloud-node-manager.yaml: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: cloud-node-manager - name: cloud-node-manager - namespace: kube-system - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: cloud-node-manager - labels: - k8s-app: cloud-node-manager - rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["watch", "list", "get", "update", "patch"] - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: cloud-node-manager - labels: - k8s-app: cloud-node-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cloud-node-manager - subjects: - - kind: ServiceAccount - name: cloud-node-manager - namespace: kube-system - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: cloud-node-manager - namespace: kube-system - labels: - component: cloud-node-manager - spec: - selector: - matchLabels: - k8s-app: cloud-node-manager - template: - metadata: - labels: - k8s-app: cloud-node-manager - annotations: - cluster-autoscaler.kubernetes.io/daemonset-pod: "true" - spec: - priorityClassName: system-node-critical - serviceAccountName: cloud-node-manager - hostNetwork: true # required to fetch correct hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule - containers: - - name: cloud-node-manager - image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.1.5} - imagePullPolicy: IfNotPresent - command: - - cloud-node-manager - - --node-name=$(NODE_NAME) - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 2000m - memory: 512Mi -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cloud-node-manager-addon - namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-calico namespace: default diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index af780ee35f8e..041f22ce5cb6 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -81,7 +82,7 @@ spec: allocate-node-cidrs: "true" bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" @@ -132,7 +133,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -145,7 +146,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -202,6 +203,343 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=2001:1234:5678:9a40::/58" + - "--bind-address=::" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--node-cidr-mask-size=0" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: @@ -274,7 +612,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: '[fd00::10]' node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 873f58031844..abbd4cc5ed3d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico windows: enabled name: ${CLUSTER_NAME} @@ -71,7 +72,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -180,14 +181,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -362,7 +363,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -383,6 +384,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml index 080040d90d62..9a70e2a342c7 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -69,7 +70,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -125,14 +126,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -253,7 +254,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') @@ -315,7 +316,7 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - content: |- + - content: | # required as a work around for Flannel and Wins bugs # https://github.com/coreos/flannel/issues/1359 # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 @@ -349,6 +350,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 59dc9d5ee404..e6af642baefa 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico windows: enabled name: ${CLUSTER_NAME} @@ -71,7 +72,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -114,14 +115,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -217,7 +218,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -236,6 +237,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index d6a78a57abc8..954f8658b086 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico gpu: nvidia name: ${CLUSTER_NAME} @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -113,14 +114,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -230,7 +231,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: addons.cluster.x-k8s.io/v1beta1 @@ -249,6 +250,341 @@ spec: name: nvidia-gpu-operator-components strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: clusterpolicy-crd.yaml: |+ diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index b1a0c4ec4031..e6b6d1b9532c 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -96,7 +97,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -139,14 +140,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -243,7 +244,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -265,6 +266,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-calico namespace: default diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml index c35e2cad2421..a16bbc1e92e9 100644 --- a/templates/test/ci/cluster-template-prow-windows.yaml +++ b/templates/test/ci/cluster-template-prow-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -69,7 +70,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -125,14 +126,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -358,6 +359,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 4dcce5b49383..61e812f03bd5 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -113,14 +114,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -214,7 +215,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -353,6 +354,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: proxy: "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n k8s-app: diff --git a/templates/test/ci/prow-external-cloud-provider/kustomization.yaml b/templates/test/ci/prow-in-tree-cloud-provider/kustomization.yaml similarity index 91% rename from templates/test/ci/prow-external-cloud-provider/kustomization.yaml rename to templates/test/ci/prow-in-tree-cloud-provider/kustomization.yaml index 7375e2f89f11..c85b80ceda07 100644 --- a/templates/test/ci/prow-external-cloud-provider/kustomization.yaml +++ b/templates/test/ci/prow-in-tree-cloud-provider/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../../flavors/external-cloud-provider + - ../../../flavors/in-tree-cloud-provider - ../prow/cni-resource-set.yaml patchesStrategicMerge: - ../patches/tags.yaml diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index a74c8ea98dde..6bf143aa8b16 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -8,6 +8,7 @@ resources: - mhc.yaml - cni-resource-set.yaml - ../../../azure-cluster-identity + - ../../../cloud-provider-azure patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 195f1043283d..d83a9b77c29a 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico windows: enabled name: ${CLUSTER_NAME} @@ -73,7 +74,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -166,14 +167,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -305,7 +306,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/replace-k8s-binaries.sh @@ -326,6 +327,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml index eb1c3ae8733a..5b32b45b08f2 100644 --- a/templates/test/dev/cluster-template-custom-builds-windows.yaml +++ b/templates/test/dev/cluster-template-custom-builds-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -71,7 +72,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -177,14 +178,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -481,6 +482,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 7c35e1c86471..99411cc0862b 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -72,7 +73,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -165,14 +166,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -302,7 +303,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/replace-k8s-binaries.sh @@ -442,6 +443,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.7} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: proxy: "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n k8s-app: diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 072d7a47bdcd..5f57d28ffe75 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -452,7 +452,7 @@ var _ = Describe("Workload cluster creation", func() { // ci-e2e.sh and Prow CI skip this test by default. // To include this test, set `GINKGO_SKIP=""`. - Context("Creating a cluster that uses the external cloud provider", func() { + Context("Creating a cluster that uses the in-tree cloud provider", func() { It("with a 1 control plane nodes and 2 worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "oot") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -462,7 +462,7 @@ var _ = Describe("Workload cluster creation", func() { ClusterctlConfigPath: clusterctlConfigPath, KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: "external-cloud-provider", + Flavor: "in-tree-cloud-provider", Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 22953dbaf42d..4f8b5535cbe9 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -171,8 +171,8 @@ providers: targetName: "cluster-template-windows.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml" targetName: "cluster-template-machine-pool-windows.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml" - targetName: "cluster-template-external-cloud-provider.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-in-tree-cloud-provider.yaml" + targetName: "cluster-template-in-tree-cloud-provider.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml" targetName: "cluster-template-aks-multi-tenancy.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml"