More external cloud-provider-azure

kubernetes-sigs · Feb 16, 2022 · 73b3cd3 · 73b3cd3
1 parent fdb1d43
commit 73b3cd3
Show file tree

Hide file tree

Showing 68 changed files with 9,862 additions and 687 deletions.
diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh
@@ -82,11 +82,12 @@ select_cluster_template() {
         export CI_VERSION="${CI_VERSION:-$(curl -sSL ${CI_VERSION_URL})}"
         export KUBERNETES_VERSION="${CI_VERSION}"
     else
-        export CLUSTER_TEMPLATE="test/ci/cluster-template-prow.yaml"
+        export CLUSTER_TEMPLATE="test/ci/cluster-template-prow-in-tree-cloud-provider.yaml"
     fi
 
     if [[ -n "${TEST_CCM:-}" ]]; then
-        export CLUSTER_TEMPLATE="test/ci/cluster-template-prow-external-cloud-provider.yaml"
+        export CLUSTER_TEMPLATE="test/ci/cluster-template-prow.yaml"
+        K8S_FEATURE_GATES="MixedProtocolLBService=true"
         # shellcheck source=scripts/ci-build-azure-ccm.sh
         source "${REPO_ROOT}/scripts/ci-build-azure-ccm.sh"
         echo "Using CCM image ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG} and CNM image ${AZURE_CLOUD_NODE_MANAGER_IMG} to build external cloud provider cluster"
@@ -103,8 +104,13 @@ select_cluster_template() {
     # this requires k8s 1.22+
     if [[ -n "${TEST_WINDOWS:-}" ]]; then
         export WINDOWS_WORKER_MACHINE_COUNT="${WINDOWS_WORKER_MACHINE_COUNT:-2}"
-        export K8S_FEATURE_GATES="WindowsHostProcessContainers=true"
+        if [ "${K8S_FEATURE_GATES:-}" != "" ]; then
+          K8S_FEATURE_GATES+=",WindowsHostProcessContainers=true"
+        else
+          K8S_FEATURE_GATES="WindowsHostProcessContainers=true"
+        fi
     fi
+    export K8S_FEATURE_GATES
 }
 
 create_cluster() {

diff --git a/...rnal-cloud-provider/ccm-resource-set.yaml → ...provider-azure-ipv6/ccm-resource-set.yaml b/...rnal-cloud-provider/ccm-resource-set.yaml → ...provider-azure-ipv6/ccm-resource-set.yaml
diff --git a/templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml b/templates/cloud-provider-azure-ipv6/cloud-controller-manager.yaml
@@ -0,0 +1,200 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    k8s-app: cloud-controller-manager
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - "*"
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - services/status
+    verbs:
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - serviceaccounts
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - create
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+  - kind: User
+    name: cloud-controller-manager
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:cloud-controller-manager:extension-apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+  - apiGroup: ""
+    kind: User
+    name: cloud-controller-manager
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+  labels:
+    tier: control-plane
+    component: cloud-controller-manager
+spec:
+  priorityClassName: system-node-critical
+  hostNetwork: true
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+  serviceAccountName: cloud-controller-manager
+  tolerations:
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+  containers:
+    - name: cloud-controller-manager
+      image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.5}
+      imagePullPolicy: IfNotPresent
+      command: ["cloud-controller-manager"]
+      args:
+        - "--allocate-node-cidrs=true"
+        - "--cloud-config=/etc/kubernetes/azure.json"
+        - "--cloud-provider=azure"
+        - "--cluster-cidr=2001:1234:5678:9a40::/58"
+        - "--bind-address=::"
+        - "--cluster-name=${CLUSTER_NAME}"
+        - "--controllers=*,-cloud-node" # disable cloud-node controller
+        - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
+        - "--leader-elect=true"
+        - "--node-cidr-mask-size=0"
+        - "--route-reconciliation-period=10s"
+        - "--v=2"
+        - "--port=10267"
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: "4"
+          memory: 2Gi
+      livenessProbe:
+        httpGet:
+          path: /healthz
+          port: 10267
+        initialDelaySeconds: 20
+        periodSeconds: 10
+        timeoutSeconds: 5
+      volumeMounts:
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+        - name: etc-ssl
+          mountPath: /etc/ssl
+          readOnly: true
+        - name: msi
+          mountPath: /var/lib/waagent/ManagedIdentity-Settings
+          readOnly: true
+  volumes:
+    - name: etc-kubernetes
+      hostPath:
+        path: /etc/kubernetes
+    - name: etc-ssl
+      hostPath:
+        path: /etc/ssl
+    - name: msi
+      hostPath:
+        path: /var/lib/waagent/ManagedIdentity-Settings
diff --git a/...al-cloud-provider/cloud-node-manager.yaml → ...ovider-azure-ipv6/cloud-node-manager.yaml b/...al-cloud-provider/cloud-node-manager.yaml → ...ovider-azure-ipv6/cloud-node-manager.yaml
diff --git a/...xternal-cloud-provider/kustomization.yaml → ...ud-provider-azure-ipv6/kustomization.yaml b/...xternal-cloud-provider/kustomization.yaml → ...ud-provider-azure-ipv6/kustomization.yaml
@@ -1,11 +1,7 @@
 namespace: default
 resources:
-  - ../default
   - ccm-resource-set.yaml
 
-patchesStrategicMerge:
-  - patches/external-cloud-provider.yaml
-
 configMapGenerator:
   - name: cloud-controller-manager-addon
     files:

diff --git a/templates/cloud-provider-azure/ccm-resource-set.yaml b/templates/cloud-provider-azure/ccm-resource-set.yaml
@@ -0,0 +1,27 @@
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: crs-ccm
+  namespace: default
+spec:
+  strategy: "ApplyOnce"
+  clusterSelector:
+    matchLabels:
+      ccm: external
+  resources:
+    - name: cloud-controller-manager-addon
+      kind: ConfigMap
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: crs-node-manager
+  namespace: default
+spec:
+  strategy: "ApplyOnce"
+  clusterSelector:
+    matchLabels:
+      ccm: external
+  resources:
+    - name: cloud-node-manager-addon
+      kind: ConfigMap
diff --git a/...ud-provider/cloud-controller-manager.yaml → ...vider-azure/cloud-controller-manager.yaml b/...ud-provider/cloud-controller-manager.yaml → ...vider-azure/cloud-controller-manager.yaml
diff --git a/templates/cloud-provider-azure/cloud-node-manager.yaml b/templates/cloud-provider-azure/cloud-node-manager.yaml
@@ -0,0 +1,85 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: cloud-node-manager
+  name: cloud-node-manager
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cloud-node-manager
+  labels:
+    k8s-app: cloud-node-manager
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cloud-node-manager
+  labels:
+    k8s-app: cloud-node-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cloud-node-manager
+subjects:
+  - kind: ServiceAccount
+    name: cloud-node-manager
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-node-manager
+  namespace: kube-system
+  labels:
+    component: cloud-node-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-node-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-node-manager
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: cloud-node-manager
+      hostNetwork: true # required to fetch correct hostname
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - operator: "Exists"
+          effect: NoExecute
+        - operator: "Exists"
+          effect: NoSchedule
+      containers:
+        - name: cloud-node-manager
+          image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.1.5}
+          imagePullPolicy: IfNotPresent
+          command:
+            - cloud-node-manager
+            - --node-name=$(NODE_NAME)
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: 50m
+              memory: 50Mi
+            limits:
+              cpu: 2000m
+              memory: 512Mi
diff --git a/templates/cloud-provider-azure/kustomization.yaml b/templates/cloud-provider-azure/kustomization.yaml
@@ -0,0 +1,17 @@
+namespace: default
+resources:
+  - ccm-resource-set.yaml
+
+configMapGenerator:
+  - name: cloud-controller-manager-addon
+    files:
+      - cloud-controller-manager.yaml
+  - name: cloud-node-manager-addon
+    files:
+      - cloud-node-manager.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    type: generated
+  annotations:
+    note: generated