From 66e73a95177ceb1b3495b379105d1eee795e6434 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 24 Nov 2021 10:28:35 -0800 Subject: [PATCH 1/5] Create in-tree-cloud-provider.yaml --- .../patches/in-tree-cloud-provider.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml diff --git a/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml new file mode 100644 index 00000000000..b6f31bb1100 --- /dev/null +++ b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + cni: "calico" +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + azure-container-registry-config: /etc/kubernetes/azure.json + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + azure-container-registry-config: /etc/kubernetes/azure.json + clusterConfiguration: + apiServer: + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + azure-container-registry-config: /etc/kubernetes/azure.json From ce00940082b71941ffa2a00743b771314d1cb81a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 24 Nov 2021 10:38:00 -0800 Subject: [PATCH 2/5] Create cluster-template-in-tree-cloud-provider.yaml --- ...uster-template-in-tree-cloud-provider.yaml | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 templates/cluster-template-in-tree-cloud-provider.yaml diff --git a/templates/cluster-template-in-tree-cloud-provider.yaml b/templates/cluster-template-in-tree-cloud-provider.yaml new file mode 100644 index 00000000000..a13c7aa37ca --- /dev/null +++ b/templates/cluster-template-in-tree-cloud-provider.yaml @@ -0,0 +1,227 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal \ No newline at end of file From 352b40b55e428ef3f0d15d50333bef8d9030e18a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 24 Nov 2021 11:11:28 -0800 Subject: [PATCH 3/5] Switch default flavors to use OOT cloud-provider --- .../ccm-resource-set.yaml | 0 .../cloud-controller-manager.yaml | 0 .../cloud-node-manager.yaml | 0 .../cloud-provider-azure/kustomization.yaml | 17 + templates/cluster-template-aad.yaml | 344 ++++++++++++++++- templates/cluster-template-azure-bastion.yaml | 346 ++++++++++++++++- templates/cluster-template-ephemeral.yaml | 346 ++++++++++++++++- ...ster-template-external-cloud-provider.yaml | 2 +- ...uster-template-in-tree-cloud-provider.yaml | 3 +- templates/cluster-template-ipv6.yaml | 346 ++++++++++++++++- ...template-machinepool-multiple-subnets.yaml | 302 --------------- ...-machinepool-system-assigned-identity.yaml | 211 ----------- ...te-machinepool-user-assigned-identity.yaml | 213 ----------- ...mplate-machinepool-windows-containerd.yaml | 348 +++++++++++++++++- .../cluster-template-machinepool-windows.yaml | 348 +++++++++++++++++- templates/cluster-template-machinepool.yaml | 346 ++++++++++++++++- templates/cluster-template-nvidia-gpu.yaml | 346 ++++++++++++++++- templates/cluster-template-private.yaml | 346 ++++++++++++++++- ...ter-template-system-assigned-identity.yaml | 346 ++++++++++++++++- ...uster-template-user-assigned-identity.yaml | 346 ++++++++++++++++- .../cluster-template-windows-containerd.yaml | 346 ++++++++++++++++- templates/cluster-template-windows.yaml | 344 ++++++++++++++++- templates/cluster-template.yaml | 346 ++++++++++++++++- templates/flavors/aad/kustomization.yaml | 1 + .../aad/patches/kubeadm-controlplane.yaml | 8 +- templates/flavors/base/cluster-template.yaml | 9 +- templates/flavors/default/kustomization.yaml | 1 + .../flavors/default/machine-deployment.yaml | 2 +- .../kustomization.yaml | 18 - .../patches/external-cloud-provider.yaml | 45 --- .../in-tree-cloud-provider/kustomization.yaml | 9 + .../machine-deployment.yaml | 60 +++ .../patches/in-tree-cloud-provider.yaml | 7 +- templates/flavors/ipv6/kustomization.yaml | 1 + .../flavors/ipv6/machine-deployment.yaml | 2 +- .../ipv6/patches/kubeadm-controlplane.yaml | 8 +- .../kustomization.yaml | 7 - .../machine-pool-deployment.yaml | 64 ---- .../patches/azurecluster-subnets.yaml | 18 - .../patches/machine-pool-subnet.yaml | 8 - .../kustomization.yaml | 6 - .../machine-pool-deployment.yaml | 63 ---- .../patches/system-assigned-identity.yaml | 7 - .../kustomization.yaml | 6 - .../machine-pool-deployment.yaml | 63 ---- .../patches/user-assigned-identity.yaml | 9 - .../machine-pool-deployment-windows.yaml | 2 +- .../machinepool-windows/kustomization.yaml | 1 + .../machine-pool-deployment-windows.yaml | 2 +- .../machine-pool-deployment.yaml | 2 +- .../flavors/machinepool/kustomization.yaml | 1 + .../machinepool/machine-pool-deployment.yaml | 2 +- .../flavors/nvidia-gpu/kustomization.yaml | 1 + .../nvidia-gpu/machine-deployment.yaml | 2 +- templates/flavors/private/kustomization.yaml | 1 + .../kustomization.yaml | 2 + .../user-assigned-identity/kustomization.yaml | 2 + templates/flavors/windows/kustomization.yaml | 1 + 58 files changed, 4900 insertions(+), 1133 deletions(-) rename templates/{flavors/external-cloud-provider => cloud-provider-azure}/ccm-resource-set.yaml (100%) rename templates/{flavors/external-cloud-provider => cloud-provider-azure}/cloud-controller-manager.yaml (100%) rename templates/{flavors/external-cloud-provider => cloud-provider-azure}/cloud-node-manager.yaml (100%) create mode 100644 templates/cloud-provider-azure/kustomization.yaml delete mode 100644 templates/cluster-template-machinepool-multiple-subnets.yaml delete mode 100644 templates/cluster-template-machinepool-system-assigned-identity.yaml delete mode 100644 templates/cluster-template-machinepool-user-assigned-identity.yaml delete mode 100644 templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml create mode 100644 templates/flavors/in-tree-cloud-provider/kustomization.yaml create mode 100644 templates/flavors/in-tree-cloud-provider/machine-deployment.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/kustomization.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml delete mode 100644 templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/kustomization.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/kustomization.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml delete mode 100644 templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml diff --git a/templates/flavors/external-cloud-provider/ccm-resource-set.yaml b/templates/cloud-provider-azure/ccm-resource-set.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/ccm-resource-set.yaml rename to templates/cloud-provider-azure/ccm-resource-set.yaml diff --git a/templates/flavors/external-cloud-provider/cloud-controller-manager.yaml b/templates/cloud-provider-azure/cloud-controller-manager.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/cloud-controller-manager.yaml rename to templates/cloud-provider-azure/cloud-controller-manager.yaml diff --git a/templates/flavors/external-cloud-provider/cloud-node-manager.yaml b/templates/cloud-provider-azure/cloud-node-manager.yaml similarity index 100% rename from templates/flavors/external-cloud-provider/cloud-node-manager.yaml rename to templates/cloud-provider-azure/cloud-node-manager.yaml diff --git a/templates/cloud-provider-azure/kustomization.yaml b/templates/cloud-provider-azure/kustomization.yaml new file mode 100644 index 00000000000..6c5e5252834 --- /dev/null +++ b/templates/cloud-provider-azure/kustomization.yaml @@ -0,0 +1,17 @@ +namespace: default +resources: + - ccm-resource-set.yaml + +configMapGenerator: + - name: cloud-controller-manager-addon + files: + - cloud-controller-manager.yaml + - name: cloud-node-manager-addon + files: + - cloud-node-manager.yaml +generatorOptions: + disableNameSuffixHash: true + labels: + type: generated + annotations: + note: generated diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index faf72fd939d..63a95c7e140 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external oidc-client-id: ${AZURE_SERVER_APP_ID} oidc-groups-claim: groups oidc-issuer-url: https://sts.windows.net/${AZURE_TENANT_ID}/ @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -112,14 +113,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -233,3 +234,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 6109eef6add..eb6cf87d8cf 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -56,7 +57,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,7 +209,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -227,3 +228,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 466ba8dc9a6..a9179d44f1e 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -231,3 +232,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-external-cloud-provider.yaml index 10368877b9d..61d7bced076 100644 --- a/templates/cluster-template-external-cloud-provider.yaml +++ b/templates/cluster-template-external-cloud-provider.yaml @@ -55,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json diff --git a/templates/cluster-template-in-tree-cloud-provider.yaml b/templates/cluster-template-in-tree-cloud-provider.yaml index a13c7aa37ca..4694f4bf0cc 100644 --- a/templates/cluster-template-in-tree-cloud-provider.yaml +++ b/templates/cluster-template-in-tree-cloud-provider.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: none cni: calico name: ${CLUSTER_NAME} namespace: default @@ -224,4 +225,4 @@ spec: name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal \ No newline at end of file + type: ServicePrincipal diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index e33503d50e8..7ad91dd8bac 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default @@ -65,7 +66,7 @@ spec: extraArgs: bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -77,7 +78,7 @@ spec: allocate-node-cidrs: "true" bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" @@ -127,7 +128,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -140,7 +141,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -197,6 +198,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: @@ -269,7 +605,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: '[fd00::10]' node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-multiple-subnets.yaml b/templates/cluster-template-machinepool-multiple-subnets.yaml deleted file mode 100644 index 6c990d20638..00000000000 --- a/templates/cluster-template-machinepool-multiple-subnets.yaml +++ /dev/null @@ -1,302 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: ${CLUSTER_NAME}-mp-0 - natGateway: - name: node-natgateway-0 - role: node - - name: ${CLUSTER_NAME}-mp-1 - natGateway: - name: node-natgateway-1 - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: ${CLUSTER_NAME}-mp-0 - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-1 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-1 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: ${CLUSTER_NAME}-mp-1 - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-1 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-system-assigned-identity.yaml b/templates/cluster-template-machinepool-system-assigned-identity.yaml deleted file mode 100644 index f9354a6b704..00000000000 --- a/templates/cluster-template-machinepool-system-assigned-identity.yaml +++ /dev/null @@ -1,211 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - identity: SystemAssigned - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-user-assigned-identity.yaml b/templates/cluster-template-machinepool-user-assigned-identity.yaml deleted file mode 100644 index 811ee478b56..00000000000 --- a/templates/cluster-template-machinepool-user-assigned-identity.yaml +++ /dev/null @@ -1,213 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: ${USER_ASSIGNED_IDENTITY_PROVIDER_ID} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-windows-containerd.yaml b/templates/cluster-template-machinepool-windows-containerd.yaml index 9d429f87480..e2af185282a 100644 --- a/templates/cluster-template-machinepool-windows-containerd.yaml +++ b/templates/cluster-template-machinepool-windows-containerd.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico windows: enabled name: ${CLUSTER_NAME} @@ -55,7 +56,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -231,6 +232,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -294,7 +630,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 8a75d813a0b..092be8d560d 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: flannel-windows name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" extraVolumes: @@ -120,14 +121,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -248,7 +249,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') @@ -323,7 +324,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -335,3 +336,338 @@ spec: name: capi sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 4a3b2bedf8c..dd7d46cb0a0 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -210,7 +211,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -228,3 +229,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 5579c9b46f9..62171e7a2fd 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico gpu: nvidia name: ${CLUSTER_NAME} @@ -55,7 +56,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -66,7 +67,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -108,14 +109,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -225,7 +226,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: addons.cluster.x-k8s.io/v1beta1 @@ -244,6 +245,341 @@ spec: name: nvidia-gpu-operator-components strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: clusterpolicy-crd.yaml: | diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index ab4972d72e8..bbf9d51e7d4 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -63,7 +64,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -74,7 +75,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -116,14 +117,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -220,7 +221,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -239,3 +240,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-system-assigned-identity.yaml b/templates/cluster-template-system-assigned-identity.yaml index 8e1c1fd6535..c5372dcf7d4 100644 --- a/templates/cluster-template-system-assigned-identity.yaml +++ b/templates/cluster-template-system-assigned-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -50,7 +51,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -61,7 +62,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -103,14 +104,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -204,6 +205,341 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-user-assigned-identity.yaml b/templates/cluster-template-user-assigned-identity.yaml index cb2b2e89e75..c85d3fee14a 100644 --- a/templates/cluster-template-user-assigned-identity.yaml +++ b/templates/cluster-template-user-assigned-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -50,7 +51,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -61,7 +62,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -103,14 +104,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,6 +209,341 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template-windows-containerd.yaml b/templates/cluster-template-windows-containerd.yaml index 32a340732be..f86c366e46d 100644 --- a/templates/cluster-template-windows-containerd.yaml +++ b/templates/cluster-template-windows-containerd.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico windows: enabled name: ${CLUSTER_NAME} @@ -55,7 +56,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -67,7 +68,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -109,14 +110,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,7 +209,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -228,6 +229,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 7d5a173e446..854c5dc4c44 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: flannel-windows name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" extraVolumes: @@ -120,14 +121,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -346,3 +347,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a6292ee8ee9..61d7bced076 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default @@ -54,7 +55,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -65,7 +66,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -107,14 +108,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -206,7 +207,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -225,3 +226,338 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default diff --git a/templates/flavors/aad/kustomization.yaml b/templates/flavors/aad/kustomization.yaml index c575fef432e..5a6c662a970 100644 --- a/templates/flavors/aad/kustomization.yaml +++ b/templates/flavors/aad/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - patches/kubeadm-controlplane.yaml - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/aad/patches/kubeadm-controlplane.yaml b/templates/flavors/aad/patches/kubeadm-controlplane.yaml index 403abc84c03..d82d7dfdfbe 100644 --- a/templates/flavors/aad/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/aad/patches/kubeadm-controlplane.yaml @@ -9,21 +9,21 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json oidc-username-claim: oid oidc-groups-claim: groups @@ -32,5 +32,5 @@ spec: oidc-username-prefix: "-" controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 82bfb6f4cdb..79d08b52616 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -5,6 +5,7 @@ metadata: name: ${CLUSTER_NAME} labels: cni: "calico" + ccm: "external" spec: clusterNetwork: pods: @@ -53,21 +54,21 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -76,7 +77,7 @@ spec: readOnly: true controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json allocate-node-cidrs: "false" cluster-name: ${CLUSTER_NAME} diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index 2afb6f7ff4b..13d010a6659 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index c2ce6a416a4..5147e8e09f9 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -47,7 +47,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/external-cloud-provider/kustomization.yaml b/templates/flavors/external-cloud-provider/kustomization.yaml index 2e6ae5e2d99..9181258413a 100644 --- a/templates/flavors/external-cloud-provider/kustomization.yaml +++ b/templates/flavors/external-cloud-provider/kustomization.yaml @@ -1,21 +1,3 @@ namespace: default resources: - ../default - - ccm-resource-set.yaml - -patchesStrategicMerge: - - patches/external-cloud-provider.yaml - -configMapGenerator: - - name: cloud-controller-manager-addon - files: - - cloud-controller-manager.yaml - - name: cloud-node-manager-addon - files: - - cloud-node-manager.yaml -generatorOptions: - disableNameSuffixHash: true - labels: - type: generated - annotations: - note: generated diff --git a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml b/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml deleted file mode 100644 index ee196d62c9c..00000000000 --- a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - labels: - cni: "calico" - ccm: "external" ---- -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - azure-container-registry-config: /etc/kubernetes/azure.json - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - azure-container-registry-config: /etc/kubernetes/azure.json - clusterConfiguration: - apiServer: - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - cloud-provider: external - version: "${KUBERNETES_VERSION}" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - azure-container-registry-config: /etc/kubernetes/azure.json diff --git a/templates/flavors/in-tree-cloud-provider/kustomization.yaml b/templates/flavors/in-tree-cloud-provider/kustomization.yaml new file mode 100644 index 00000000000..b07cdc8fe84 --- /dev/null +++ b/templates/flavors/in-tree-cloud-provider/kustomization.yaml @@ -0,0 +1,9 @@ +namespace: default +resources: + - ../base + - machine-deployment.yaml + - ../../azure-cluster-identity + +patchesStrategicMerge: + - ../../azure-cluster-identity/azurecluster-identity-ref.yaml + - patches/in-tree-cloud-provider.yaml diff --git a/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml b/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml new file mode 100644 index 00000000000..c2ce6a416a4 --- /dev/null +++ b/templates/flavors/in-tree-cloud-provider/machine-deployment.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + vmSize: ${AZURE_NODE_MACHINE_TYPE} + osDisk: + osType: "Linux" + diskSizeGB: 128 + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + preKubeadmCommands: [] + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data["local_hostname"] }}' + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + azure-container-registry-config: /etc/kubernetes/azure.json + files: + - contentFrom: + secret: + name: ${CLUSTER_NAME}-md-0-azure-json + key: worker-node-azure.json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" diff --git a/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml index b6f31bb1100..06a8d4ec771 100644 --- a/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml +++ b/templates/flavors/in-tree-cloud-provider/patches/in-tree-cloud-provider.yaml @@ -4,7 +4,8 @@ kind: Cluster metadata: name: ${CLUSTER_NAME} labels: - cni: "calico" + cni: calico + ccm: none --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -26,7 +27,9 @@ spec: azure-container-registry-config: /etc/kubernetes/azure.json clusterConfiguration: apiServer: - timeoutForControlPlane: 20m + extraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json controllerManager: extraArgs: cloud-provider: azure diff --git a/templates/flavors/ipv6/kustomization.yaml b/templates/flavors/ipv6/kustomization.yaml index 20b7e499b7b..0f213ae0ea7 100644 --- a/templates/flavors/ipv6/kustomization.yaml +++ b/templates/flavors/ipv6/kustomization.yaml @@ -2,6 +2,7 @@ namespace: default resources: - ../base - ../../azure-cluster-identity + - ../../cloud-provider-azure - machine-deployment.yaml patchesStrategicMerge: diff --git a/templates/flavors/ipv6/machine-deployment.yaml b/templates/flavors/ipv6/machine-deployment.yaml index b843f63e7c8..239ad3b9c41 100644 --- a/templates/flavors/ipv6/machine-deployment.yaml +++ b/templates/flavors/ipv6/machine-deployment.yaml @@ -54,7 +54,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" diff --git a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml index 5aa34de5bea..980f17e5764 100644 --- a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml @@ -14,7 +14,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" @@ -26,7 +26,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" @@ -39,12 +39,12 @@ spec: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json bind-address: "::" controllerManager: extraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json #required for ipv6 using calico allocate-node-cidrs: "true" diff --git a/templates/flavors/machinepool-multiple-subnets/kustomization.yaml b/templates/flavors/machinepool-multiple-subnets/kustomization.yaml deleted file mode 100644 index 4cd6ffe71d9..00000000000 --- a/templates/flavors/machinepool-multiple-subnets/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: default -resources: - - ../machinepool - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/machine-pool-subnet.yaml - - patches/azurecluster-subnets.yaml diff --git a/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml b/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml deleted file mode 100644 index c925c051278..00000000000 --- a/templates/flavors/machinepool-multiple-subnets/machine-pool-deployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-1" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-1" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subnetName: "${CLUSTER_NAME}-mp-1" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-1" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml b/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml deleted file mode 100644 index 3ad2d21e456..00000000000 --- a/templates/flavors/machinepool-multiple-subnets/patches/azurecluster-subnets.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} -spec: - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: "${CLUSTER_NAME}-mp-0" - role: node - natGateway: - name: node-natgateway-0 - - name: "${CLUSTER_NAME}-mp-1" - role: node - natGateway: - name: node-natgateway-1 diff --git a/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml b/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml deleted file mode 100644 index 09521125984..00000000000 --- a/templates/flavors/machinepool-multiple-subnets/patches/machine-pool-subnet.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - template: - subnetName: "${CLUSTER_NAME}-mp-0" diff --git a/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml b/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml deleted file mode 100644 index 6600b421357..00000000000 --- a/templates/flavors/machinepool-system-assigned-identity/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../base - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/system-assigned-identity.yaml diff --git a/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml b/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml deleted file mode 100644 index cf3e50b6a6b..00000000000 --- a/templates/flavors/machinepool-system-assigned-identity/machine-pool-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml b/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml deleted file mode 100644 index e7aae3adefb..00000000000 --- a/templates/flavors/machinepool-system-assigned-identity/patches/system-assigned-identity.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - identity: SystemAssigned diff --git a/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml b/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml deleted file mode 100644 index 2a4f7e06d0a..00000000000 --- a/templates/flavors/machinepool-user-assigned-identity/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../base - - machine-pool-deployment.yaml -patchesStrategicMerge: - - patches/user-assigned-identity.yaml diff --git a/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml b/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml deleted file mode 100644 index cf3e50b6a6b..00000000000 --- a/templates/flavors/machinepool-user-assigned-identity/machine-pool-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 1 - deletePolicy: Oldest - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml b/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml deleted file mode 100644 index 83c81d94fae..00000000000 --- a/templates/flavors/machinepool-user-assigned-identity/patches/user-assigned-identity.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: AzureMachinePool -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - identity: UserAssigned - userAssignedIdentities: - - providerID: ${USER_ASSIGNED_IDENTITY_PROVIDER_ID} diff --git a/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml index 7f90b786d32..19356895e70 100644 --- a/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml +++ b/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml @@ -57,7 +57,7 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:3.4.1" diff --git a/templates/flavors/machinepool-windows/kustomization.yaml b/templates/flavors/machinepool-windows/kustomization.yaml index 586f3a62e83..aa83fd1b17f 100644 --- a/templates/flavors/machinepool-windows/kustomization.yaml +++ b/templates/flavors/machinepool-windows/kustomization.yaml @@ -4,6 +4,7 @@ resources: - ../../azure-cluster-identity - machine-pool-deployment.yaml - machine-pool-deployment-windows.yaml + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml index fdd224f7b94..5eafc520e6f 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml @@ -55,7 +55,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:1.4.1" diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml index dab4b271937..2ab9fc07889 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml @@ -53,7 +53,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/machinepool/kustomization.yaml b/templates/flavors/machinepool/kustomization.yaml index ba038c599d5..70b29875b23 100644 --- a/templates/flavors/machinepool/kustomization.yaml +++ b/templates/flavors/machinepool/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - machine-pool-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/machinepool/machine-pool-deployment.yaml b/templates/flavors/machinepool/machine-pool-deployment.yaml index cf3e50b6a6b..85ad548ed9e 100644 --- a/templates/flavors/machinepool/machine-pool-deployment.yaml +++ b/templates/flavors/machinepool/machine-pool-deployment.yaml @@ -50,7 +50,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/nvidia-gpu/kustomization.yaml b/templates/flavors/nvidia-gpu/kustomization.yaml index 4b2741ea6c9..ded5830dafb 100644 --- a/templates/flavors/nvidia-gpu/kustomization.yaml +++ b/templates/flavors/nvidia-gpu/kustomization.yaml @@ -4,6 +4,7 @@ resources: - ../../azure-cluster-identity - machine-deployment.yaml - gpu-operator-resources-set.yaml + - ../../cloud-provider-azure patchesStrategicMerge: - patches/cluster.yaml diff --git a/templates/flavors/nvidia-gpu/machine-deployment.yaml b/templates/flavors/nvidia-gpu/machine-deployment.yaml index 88e97237f23..e37b8da3e44 100644 --- a/templates/flavors/nvidia-gpu/machine-deployment.yaml +++ b/templates/flavors/nvidia-gpu/machine-deployment.yaml @@ -48,7 +48,7 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure + cloud-provider: external cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json files: diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml index 60660bd25bc..8678ccedb52 100644 --- a/templates/flavors/private/kustomization.yaml +++ b/templates/flavors/private/kustomization.yaml @@ -3,6 +3,7 @@ resources: - ../base - ../default/machine-deployment.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/system-assigned-identity/kustomization.yaml b/templates/flavors/system-assigned-identity/kustomization.yaml index f61e418f8dd..f020b5d6b6a 100644 --- a/templates/flavors/system-assigned-identity/kustomization.yaml +++ b/templates/flavors/system-assigned-identity/kustomization.yaml @@ -2,5 +2,7 @@ namespace: default resources: - ../base - ../default/machine-deployment.yaml + - ../../cloud-provider-azure + patchesStrategicMerge: - patches/system-assigned-identity.yaml diff --git a/templates/flavors/user-assigned-identity/kustomization.yaml b/templates/flavors/user-assigned-identity/kustomization.yaml index 981d43513ed..b401b6f388e 100644 --- a/templates/flavors/user-assigned-identity/kustomization.yaml +++ b/templates/flavors/user-assigned-identity/kustomization.yaml @@ -2,5 +2,7 @@ namespace: default resources: - ../base - ../default/machine-deployment.yaml + - ../../cloud-provider-azure + patchesStrategicMerge: - patches/user-assigned-identity.yaml diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index a5902fd9ea8..be33dd740f4 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -4,6 +4,7 @@ resources: - machine-deployment.yaml - machine-deployment-windows.yaml - ../../azure-cluster-identity + - ../../cloud-provider-azure patchesStrategicMerge: - ../../azure-cluster-identity/azurecluster-identity-ref.yaml From bc7144a3f2c8e18e7c1319172c4c6e3596a39f90 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 24 Nov 2021 11:57:56 -0800 Subject: [PATCH 4/5] test templates --- .../ci/cluster-template-prow-custom-vnet.yaml | 346 ++++++++++++++++- ...template-prow-external-cloud-provider.yaml | 2 +- .../test/ci/cluster-template-prow-ipv6.yaml | 346 ++++++++++++++++- ...er-template-prow-machine-pool-windows.yaml | 348 +++++++++++++++++- .../cluster-template-prow-machine-pool.yaml | 348 +++++++++++++++++- .../ci/cluster-template-prow-nvidia-gpu.yaml | 346 ++++++++++++++++- .../ci/cluster-template-prow-private.yaml | 346 ++++++++++++++++- .../ci/cluster-template-prow-windows.yaml | 344 ++++++++++++++++- templates/test/ci/cluster-template-prow.yaml | 346 ++++++++++++++++- templates/test/ci/prow/kustomization.yaml | 1 + 10 files changed, 2731 insertions(+), 42 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index b60d88c7308..8be3de69b16 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -57,7 +58,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -68,7 +69,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -111,14 +112,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,7 +213,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -232,6 +233,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml index 5b0b15ea0e3..71418373b2e 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml @@ -59,7 +59,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: MixedProtocolLBService=true extraVolumes: - hostPath: /etc/kubernetes/azure.json diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index af780ee35f8..65188110679 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -69,7 +70,7 @@ spec: extraArgs: bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -81,7 +82,7 @@ spec: allocate-node-cidrs: "true" bind-address: '::' cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" @@ -132,7 +133,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -145,7 +146,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: fd00::10 node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' @@ -202,6 +203,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: @@ -274,7 +610,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-dns: '[fd00::10]' node-ip: '::' name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml index 080040d90d6..a333f1daab9 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -58,7 +59,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -69,7 +70,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -125,14 +126,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -253,7 +254,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') @@ -333,7 +334,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -349,6 +350,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 59dc9d5ee40..585cfa6c0a0 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico windows: enabled name: ${CLUSTER_NAME} @@ -59,7 +60,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -71,7 +72,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -114,14 +115,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -217,7 +218,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -236,6 +237,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -299,7 +635,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 540f11a0094..62b8530a08c 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico gpu: nvidia name: ${CLUSTER_NAME} @@ -59,7 +60,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -113,14 +114,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -230,7 +231,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: addons.cluster.x-k8s.io/v1beta1 @@ -249,6 +250,341 @@ spec: name: nvidia-gpu-operator-components strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: clusterpolicy-crd.yaml: | diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index ffdbf70caa6..35252fd2162 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -77,7 +78,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -88,7 +89,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -131,14 +132,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -235,7 +236,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -257,6 +258,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-calico namespace: default diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml index c35e2cad242..f41fe1cef14 100644 --- a/templates/test/ci/cluster-template-prow-windows.yaml +++ b/templates/test/ci/cluster-template-prow-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-flannel name: ${CLUSTER_NAME} namespace: default @@ -58,7 +59,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -69,7 +70,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "false" v: "4" @@ -125,14 +126,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -358,6 +359,341 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-flannel namespace: default diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 0328deee419..fb41eff88e4 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-calico name: ${CLUSTER_NAME} namespace: default @@ -58,7 +59,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -70,7 +71,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" extraVolumes: @@ -113,14 +114,14 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -214,7 +215,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -349,6 +350,341 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-node-manager + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-node-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + kind: Pod + metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager + spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cloud-controller-manager + image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - "--allocate-node-cidrs=true" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--cloud-provider=azure" + - "--cluster-cidr=10.244.0.0/16" + - "--cluster-name=${CLUSTER_NAME}" + - "--controllers=*,-cloud-node" # disable cloud-node controller + - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins + - "--leader-elect=true" + - "--route-reconciliation-period=10s" + - "--v=2" + - "--port=10267" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + livenessProbe: + httpGet: + path: /healthz + port: 10267 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: msi + mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon + namespace: default +--- +apiVersion: v1 +data: + cloud-node-manager.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: cloud-node-manager + name: cloud-node-manager + namespace: kube-system + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update", "patch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cloud-node-manager + labels: + k8s-app: cloud-node-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-node-manager + subjects: + - kind: ServiceAccount + name: cloud-node-manager + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cloud-node-manager + namespace: kube-system + labels: + component: cloud-node-manager + spec: + selector: + matchLabels: + k8s-app: cloud-node-manager + template: + metadata: + labels: + k8s-app: cloud-node-manager + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: cloud-node-manager + hostNetwork: true # required to fetch correct hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cloud-node-manager + image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1} + imagePullPolicy: IfNotPresent + command: + - cloud-node-manager + - --node-name=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 2000m + memory: 512Mi +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-node-manager-addon + namespace: default +--- apiVersion: v1 data: proxy: "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n k8s-app: diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index a74c8ea98dd..6bf143aa8b1 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -8,6 +8,7 @@ resources: - mhc.yaml - cni-resource-set.yaml - ../../../azure-cluster-identity + - ../../../cloud-provider-azure patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml From b77ead44cc4232dca4c8d2491d6d1d76e86f1dfe Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 24 Nov 2021 12:09:00 -0800 Subject: [PATCH 5/5] Switch external cloud provider test to in tree --- test/e2e/azure_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 98b30059911..6fad9aea6ff 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -451,7 +451,7 @@ var _ = Describe("Workload cluster creation", func() { // ci-e2e.sh and Prow CI skip this test by default. // To include this test, set `GINKGO_SKIP=""`. - Context("Creating a cluster that uses the external cloud provider", func() { + Context("Creating a cluster that uses the in-tree cloud provider", func() { It("with a 1 control plane nodes and 2 worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "oot") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -461,7 +461,7 @@ var _ = Describe("Workload cluster creation", func() { ClusterctlConfigPath: clusterctlConfigPath, KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: "external-cloud-provider", + Flavor: "in-tree-cloud-provider", Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion),