Use deployment for CCM in example yaml

Signed-off-by: Zhecheng Li <[email protected]>
kubernetes-sigs · Mar 16, 2022 · c2a84b2 · c2a84b2
1 parent 10a510e
commit c2a84b2
Show file tree

Hide file tree

Showing 8 changed files with 198 additions and 125 deletions.
diff --git a/README.md b/README.md
@@ -58,7 +58,7 @@ azure-cloud-controller-manager \
     --cloud-provider=azure \
     --cluster-name=kubernetes \
     --controllers=*,-cloud-node \
-    --cloud-config=/etc/kubernetes/azure.json \
+    --cloud-config=/etc/kubernetes/cloud-config/azure.json \
     --kubeconfig=/etc/kubernetes/kubeconfig \
     --allocate-node-cidrs=true \
     --configure-cloud-routes=true \
@@ -77,7 +77,7 @@ azure-cloud-node-manager \
     --wait-routes=true
 ```
 
-It is recommended to run azure-cloud-controller-manager as Pods on master nodes. See [here](examples/out-of-tree/cloud-controller-manager.yaml) for the example.
+It is recommended to run azure-cloud-controller-manager as Deployment with multiple replicas or Kubelet static Pods on each master Node. See [here](examples/out-of-tree/cloud-controller-manager.yaml) for the example.
 
 Please checkout more details at [Deploy Cloud Controller Manager](http://kubernetes-sigs.github.io/cloud-provider-azure/install/azure-ccm/).
 

diff --git a/examples/out-of-tree/cloud-controller-manager.yaml b/examples/out-of-tree/cloud-controller-manager.yaml
@@ -129,68 +129,128 @@ subjects:
     kind: User
     name: cloud-controller-manager
 ---
-apiVersion: v1
-kind: Pod
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   name: cloud-controller-manager
   namespace: kube-system
   labels:
-    tier: control-plane
     component: cloud-controller-manager
 spec:
-  priorityClassName: system-node-critical
-  hostNetwork: true
-  serviceAccountName: cloud-controller-manager
-  tolerations:
-    - key: node-role.kubernetes.io/master
-      effect: NoSchedule
-  containers:
-    - name: cloud-controller-manager
-      image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.5
-      imagePullPolicy: IfNotPresent
-      command: ["cloud-controller-manager"]
-      args:
-        - "--allocate-node-cidrs=true" # "false" for Azure CNI and "true" for other network plugins
-        - "--cloud-config=/etc/kubernetes/azure.json"
-        - "--cloud-provider=azure"
-        - "--cluster-cidr=10.244.0.0/16"
-        - "--cluster-name=k8s"
-        - "--controllers=*,-cloud-node" # disable cloud-node controller
-        - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
-        - "--leader-elect=true"
-        - "--route-reconciliation-period=10s"
-        - "--v=2"
-        - "--port=10267"
-      resources:
-        requests:
-          cpu: 100m
-          memory: 128Mi
-        limits:
-          cpu: "4"
-          memory: 2Gi
-      livenessProbe:
-        httpGet:
-          path: /healthz
-          port: 10267
-        initialDelaySeconds: 20
-        periodSeconds: 10
-        timeoutSeconds: 5
-      volumeMounts:
+  replicas: 1
+  selector:
+    matchLabels:
+      tier: control-plane
+      component: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        tier: control-plane
+        component: cloud-controller-manager
+    spec:
+      priorityClassName: system-node-critical
+      hostNetwork: true
+      serviceAccountName: cloud-controller-manager
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: cloud-controller-manager
+          image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.5
+          imagePullPolicy: IfNotPresent
+          command: ["cloud-controller-manager"]
+          args:
+            - "--allocate-node-cidrs=true" # "false" for Azure CNI and "true" for other network plugins
+            - "--cloud-config=/etc/kubernetes/cloud-config/azure.json"
+            - "--cloud-provider=azure"
+            - "--cluster-cidr=10.244.0.0/16"
+            - "--cluster-name=k8s"
+            - "--controllers=*,-cloud-node" # disable cloud-node controller
+            - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
+            - "--leader-elect=true"
+            - "--route-reconciliation-period=10s"
+            - "--v=4"
+            - "--port=10267"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "4"
+              memory: 2Gi
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10267
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: etc-kubernetes
+              mountPath: /etc/kubernetes
+            - name: cloud-config
+              mountPath: /etc/kubernetes/cloud-config
+              readOnly: true
+            - name: etc-ssl
+              mountPath: /etc/ssl
+              readOnly: true
+            - name: msi
+              mountPath: /var/lib/waagent/ManagedIdentity-Settings
+              readOnly: true
+      volumes:
         - name: etc-kubernetes
-          mountPath: /etc/kubernetes
+          hostPath:
+            path: /etc/kubernetes
+        - name: cloud-config
+          secret:
+            secretName: azure-cloud-config
         - name: etc-ssl
-          mountPath: /etc/ssl
-          readOnly: true
+          hostPath:
+            path: /etc/ssl
         - name: msi
-          mountPath: /var/lib/waagent/ManagedIdentity-Settings
-          readOnly: true
-  volumes:
-    - name: etc-kubernetes
-      hostPath:
-        path: /etc/kubernetes
-    - name: etc-ssl
-      hostPath:
-        path: /etc/ssl
-    - name: msi
-      hostPath:
-        path: /var/lib/waagent/ManagedIdentity-Settings
+          hostPath:
+            path: /var/lib/waagent/ManagedIdentity-Settings
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-cloud-config
+  namespace: kube-system
+type: Opaque
+stringData:
+  azure.json: |-
+    {
+      "cloud": "AzurePublicCloud",
+      "tenantId": "<tenant-id>",
+      "subscriptionId": "<subscription-id>",
+      "aadClientId": "<client-id>",
+      "aadClientSecret": "<client-secret>",
+      "resourceGroup": "<resource-group-name>",
+      "location": "<location>",
+      "vmType": "<vm-type>",
+      "subnetName": "<subnet-name>",
+      "securityGroupName": "<security-group-name>",
+      "vnetName": "<vnet-name>",
+      "vnetResourceGroup": "<vnet-resource-group>",
+      "routeTableName": "<route-table-name>",
+      "primaryAvailabilitySetName": "<primary-as-name>",
+      "primaryScaleSetName": "<primary-ss-name>",
+      "cloudProviderBackoff": true,
+      "cloudProviderBackoffRetries": 6,
+      "cloudProviderBackoffExponent": 1.5,
+      "cloudProviderBackoffDuration": 5,
+      "cloudProviderBackoffJitter": 1,
+      "cloudProviderRatelimit": true,
+      "cloudProviderRateLimitQPS": 6,
+      "cloudProviderRateLimitBucket": 20,
+      "useManagedIdentityExtension": false,
+      "userAssignedIdentityID": "",
+      "useInstanceMetadata": true,
+      "loadBalancerSku": "<loadbalancer-sku>",
+      "excludeMasterFromStandardLB": false,
+      "maximumLoadBalancerRuleCount": 250,
+      "enableMultipleStandardLoadBalancers": false,
+      "tags": "a=b,c=d"
+    }
diff --git a/site/content/en/install/azure-ccm.md b/site/content/en/install/azure-ccm.md
@@ -22,7 +22,7 @@ To deploy Azure cloud controller manager, the following components need to be co
 |Flag|Value|Remark|
 |----|-----|------|
 |`--cloud-provider`|external|cloud-provider should be set external|
-|`--azure-container-registry-config`|/etc/kubernetes/azure.json|Used for Azure credential provider|
+|`--azure-container-registry-config`|/etc/kubernetes/cloud-config/azure.json|Used for Azure credential provider|
 
 ### kube-controller-manager
 
@@ -39,10 +39,12 @@ Do not set flag `--cloud-provider`.
 
 ### azure-cloud-controller-manager
 
+azure-cloud-controller-manager should be run as Deployment with multiple replicas or Kubelet static Pods on each master Node.
+
 |Flag|Value|Remark|
 |---|---|---|
 |`--cloud-provider`|azure|cloud-provider should be set azure|
-|`--cloud-config`|/etc/kubernetes/azure.json|Path for [cloud provider config](../configs)|
+|`--cloud-config`|/etc/kubernetes/cloud-config/azure.json|Path for [cloud provider config](../configs.md)|
 |`--controllers`|*,-cloud-node | cloud node controller should be disabled|
 |`--configure-cloud-routes`| "false" for Azure CNI and "true" for other network plugins| Used for non-AzureCNI clusters |
 

diff --git a/site/content/en/install/configs.md b/site/content/en/install/configs.md
@@ -225,7 +225,7 @@ Here is an example of per-client config:
 
 When running Kubelet with kube-controller-manager, it also supports running without Azure identity since v1.15.0.
 
-Both kube-controller-manager and kubelet should configure `--cloud-provider=azure --cloud-config=/etc/kubernetes/azure.json`, but the contents for `azure.json` are different:
+Both kube-controller-manager and kubelet should configure `--cloud-provider=azure --cloud-config=/etc/kubernetes/cloud-config/azure.json`, but the contents for `azure.json` are different:
 
 (1) For kube-controller-manager, refer the above part for setting `azure.json`.
 

diff --git a/site/content/en/topics/availability-zones.md b/site/content/en/topics/availability-zones.md
@@ -29,7 +29,7 @@ Kubernetes v1.12 adds support for [Azure availability zones (AZ)](https://azure.
 
 ## Pre-requirements
 
-Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/azure.json`):
+Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/cloud-config/azure.json`):
 
 ```json
 {

diff --git a/site/content/en/topics/credential-provider.md b/site/content/en/topics/credential-provider.md
@@ -7,7 +7,7 @@ description: >
     Detailed steps to setup out-of-tree Kubelet Credential Provider.
 ---
 
-> Note: The Kubelet credential provider feature is still in alpha and shouldn't be used in production environments. Please use `--azure-container-registry-config=/etc/kubernetes/azure.json` if you need pulling images from ACR in production.
+> Note: The Kubelet credential provider feature is still in alpha and shouldn't be used in production environments. Please use `--azure-container-registry-config=/etc/kubernetes/cloud-config/azure.json` if you need pulling images from ACR in production.
 
 As part of [Out-of-Tree Credential Providers]([enhancements/keps/sig-cloud-provider/2133-out-of-tree-credential-provider at master · kubernetes/enhancements (github.com)](https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2133-out-of-tree-credential-provider)), the kubelet builtin image pulling from ACR (which could be enabled by setting `kubelet --azure-container-registry-config=<config-file>`) would be moved out-of-tree credential plugin `acr-credential-provider`. Please refer the original [KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2133-out-of-tree-credential-provider) for details.
 
@@ -33,6 +33,6 @@ providers:
   - "*.azurecr.us"
   - "*.azurecr.*" # Only required for custom Azure cloud.
   args:
-  - /etc/kubernetes/azure.json
+  - /etc/kubernetes/cloud-config/azure.json
 ```
 
diff --git a/site/content/en/topics/cross-resource-group-nodes.md b/site/content/en/topics/cross-resource-group-nodes.md
@@ -48,7 +48,7 @@ For example,
 ```shell script
 kubelet ... \
   --cloud-provider=azure \
-  --cloud-config=/etc/kubernetes/azure.json \
+  --cloud-config=/etc/kubernetes/cloud-config/azure.json \
   --node-labels=node.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/resource-group=<rg-name>
 ```
-Original file line number
+Diff line change
@@ Expand Up @@
     ## Pre-requirements
-    Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/azure.json`):
+    Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/cloud-config/azure.json`):
     ```json
     {
@@ Expand Down @@