debug

kubernetes-sigs · Oct 20, 2023 · 4468fb0 · 4468fb0
1 parent 2726786
commit 4468fb0
Showing 1 changed file with 146 additions and 133 deletions.
diff --git a/test/e2e/data/ccm/gce-cloud-controller-manager.yaml b/test/e2e/data/ccm/gce-cloud-controller-manager.yaml
@@ -1,129 +1,130 @@
-# GCP CCM DaemonSet
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: cloud-controller-manager
   namespace: kube-system
+  labels:
+    component: cloud-controller-manager
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 spec:
-  revisionHistoryLimit: 2
   selector:
     matchLabels:
-      app: gce-cloud-controller-manager
+      component: cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       labels:
-        app: gce-cloud-controller-manager
+        tier: control-plane
+        component: cloud-controller-manager
     spec:
-      dnsPolicy: Default
-      hostNetwork: true
-      priorityClassName: system-cluster-critical
-      serviceAccountName: cloud-controller-manager
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: ""
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #             - key: node-role.kubernetes.io/control-plane
-      #               operator: Exists
-      #         - matchExpressions:
-      #             - key: node-role.kubernetes.io/master
-      #               operator: Exists
+      nodeSelector: null
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
       tolerations:
-        # this taint is set by all kubelets running `--cloud-provider=external`
-        # so we should tolerate it to schedule the gce ccm
-        - key: "node.cloudprovider.kubernetes.io/uninitialized"
-          value: "true"
-          effect: "NoSchedule"
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
-        # cloud controller manages should be able to run on masters
-        # TODO: remove this when ccm is not supported on k8s <= 1.23
-        - key: "node-role.kubernetes.io/master"
-          effect: NoSchedule
-        # k8s clusters 1.24+ uses control-plane name instead of master
-        - key: "node-role.kubernetes.io/control-plane"
-          effect: NoSchedule
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node.kubernetes.io/not-ready
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
       serviceAccountName: cloud-controller-manager
       containers:
-        - name: cloud-controller-manager
-          image: gcr.io/k8s-staging-cloud-provider-gcp/cloud-controller-manager@sha256:b98242f767595c3c137e63bd270ce4de625abcac398fcc105848f0d7dcd30c02
-          # command: ['/cloud-controller-manager']
-          args:
-            # The --help output of the controller binary suggests that profiling is enabled by default
-            - --profiling=false
-            - --v=4
-            - --leader-elect=true
-            # We generate a ConfigMap for this file using Kustomize and apply it together with the CAPI manifests in the
-            # management cluster, then use it in KubeadmControlPlane.spec.kubeadmConfigSpec.files to have cloud-init
-            # write its contents to a file on controlplane nodes. See below for contents but I'm fairly sure we only needed 
-            # to explicitly provide it to make Shared VPC work.
-            # - --cloud-config=/etc/kubernetes/cloud.config
-            # Default stuff
-            - --cloud-provider=gce
-            - --use-service-account-credentials=true
-            - --bind-address=127.0.0.1
-            - --secure-port=10258
-            # These took a bit of trial and error, most of them probably aren't universally applicable, as we run cilium without
-            #  kube-proxy and use Shared VPC + Secondary VPC Ranges for "native" routing (https://docs.cilium.io/en/stable/network/concepts/routing/#google-cloud)
-            # - --cluster-name=my-cluster
-            - --cluster-cidr=10.0.0.0/8
-            - --allocate-node-cidrs=true
-            - --configure-cloud-routes=false
-            - --cidr-allocator-type=CloudAllocator
-            - --controllers=cloud-node,cloud-node-lifecycle,nodeipam,service
-          env:
-            # This probably won't work when running HA controlplanes, but without kube-proxy we don't get DNS resolution
-            # for services until cilium is up and running, which doesn't happen until after CCM itself is deployed.
-            # - name: KUBERNETES_SERVICE_HOST
-            #   value: "127.0.0.1"
-            # - name: KUBERNETES_SERVICE_PORT
-            #   value: "6443"
-          # volumeMounts:
-          #   - mountPath: /etc/kubernetes/cloud.config
-          #     name: cloudconfig
-          #     readOnly: true
-      # volumes:
-      #   - hostPath:
-      #       path: /etc/kubernetes/cloud.config
-      #       type: ""
-      #     name: cloudconfig
+      - name: cloud-controller-manager
+        image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v27.1.6@sha256:f057f6c934d6afa73a38f94b71d7da2f99033e9a6e689d59b4ee1e689031ef00
+        imagePullPolicy: IfNotPresent
+        args:
+          # The --help output of the controller binary suggests that profiling is enabled by default
+          - --profiling=false
+          - --v=4
+          - --leader-elect=true
+          - --cloud-provider=gce
+          - --use-service-account-credentials=true
+          - --bind-address=127.0.0.1
+          - --secure-port=10258
+          - --cluster-name=${CLUSTER_NAME}
+          - --cluster-cidr=192.168.0.0/16
+          - --allocate-node-cidrs=true
+          - --configure-cloud-routes=false
+          - --cidr-allocator-type=CloudAllocator
+          - --controllers=cloud-node,cloud-node-lifecycle,nodeipam,service
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: "127.0.0.1"
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 10258
+            scheme: HTTPS
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 15
+        resources:
+          requests:
+            cpu: "200m"
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cloud-controller-manager
   namespace: kube-system
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+
 ---
-apiVersion: v1
-kind: ServiceAccount
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
 metadata:
-  name: system:cloud-controller-manager
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
   namespace: kube-system
 ---
+
+# https://github.com/kubernetes/cloud-provider-gcp/blob/master/deploy/cloud-node-controller-role.yaml
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  name: system:cloud-controller-manager
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
-  name: system:cloud-controller-manager
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 rules:
-- apiGroups:
-  - networking.gke.io
-  resources:
-  - network
-  verbs:
-  - get
-- apiGroups:
-  - networking.gke.io
-  resources:
-  - network/status
-  - gkenetworkparamset
-  - gkenetworkparamset/status
-  verbs:
-  - update
-  - get
 - apiGroups:
   - ""
   - events.k8s.io
@@ -139,6 +140,10 @@ rules:
   - leases
   verbs:
   - create
+  - get
+  - list
+  - watch
+  - update
 - apiGroups:
   - coordination.k8s.io
   resourceNames:
@@ -148,8 +153,6 @@ rules:
   verbs:
   - get
   - update
-  - create
-  - delete
 - apiGroups:
   - ""
   resources:
@@ -166,7 +169,7 @@ rules:
   verbs:
   - get
   - update
-  - patch
+  - patch # until #393 lands
 - apiGroups:
   - ""
   resources:
@@ -184,7 +187,6 @@ rules:
   - ""
   resources:
   - secrets
-  - configmaps
   verbs:
   - create
   - delete
@@ -213,10 +215,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
   name: system::leader-locking-cloud-controller-manager
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 rules:
 - apiGroups:
   - ""
@@ -237,9 +240,10 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  name: system:controller:cloud-node-controller
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
-  name: system:controller:cloud-node-controller
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 rules:
 - apiGroups:
   - ""
@@ -285,37 +289,16 @@ rules:
   - list
   - delete
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
-  name: system:controller:pvl-controller
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  - persistentvolumes
-  verbs:
-  - list
-  - watch
----
+
+# https://github.com/kubernetes/cloud-provider-gcp/blob/master/deploy/cloud-node-controller-binding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
   name: system::leader-locking-cloud-controller-manager
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -328,24 +311,27 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  name: system:cloud-controller-manager
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
-  name: system:cloud-controller-manager
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: system:cloud-controller-manager
 subjects:
-- kind: User
-  apiGroup: rbac.authorization.k8s.io
-  name: system:cloud-controller-manager
+- kind: ServiceAccount
+  apiGroup: ""
+  name: cloud-controller-manager
+  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  name: system:controller:cloud-node-controller
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
-  name: system:controller:cloud-node-controller
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -354,3 +340,30 @@ subjects:
 - kind: ServiceAccount
   name: cloud-node-controller
   namespace: kube-system
+---
+
+# https://github.com/kubernetes/cloud-provider-gcp/blob/master/deploy/pvl-controller-role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:controller:pvl-controller
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  - persistentvolumes
+  verbs:
+  - list
+  - watch