diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index ed0593b8e9..f9029b002b 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -12,6 +12,7 @@ - [Power VS Cluster](./topics/powervs/index.md) - [Prerequisites](./topics/powervs/prerequisites.md) - [Creating a cluster](./topics/powervs/creating-a-cluster.md) + - [External Cloud Provider](./topics/powervs/external-cloud-provider.md) - [Developer Guide](./developer/index.md) - [Rapid iterative development with Tilt](./developer/tilt.md) - [Guide for API conversions](./developer/conversion.md) diff --git a/docs/book/src/topics/powervs/creating-a-cluster.md b/docs/book/src/topics/powervs/creating-a-cluster.md index a4d1d4d535..558054e7a6 100644 --- a/docs/book/src/topics/powervs/creating-a-cluster.md +++ b/docs/book/src/topics/powervs/creating-a-cluster.md @@ -34,6 +34,8 @@ following the steps below. 2. Use clusterctl to render the yaml through templates and deploy the cluster + **Note:** To deploy workload cluster with Power VS cloud controller manager follow [these](/topics/powervs/external-cloud-provider.html) steps . + **Note:** the `IBMPOWERVS_IMAGE_ID` value below should reflect the ID of the custom qcow2 image, the `kubernetes-version` value below should reflect the kubernetes version of the custom qcow2 image. ```console diff --git a/docs/book/src/topics/powervs/external-cloud-provider.md b/docs/book/src/topics/powervs/external-cloud-provider.md new file mode 100644 index 0000000000..ab42145a33 --- /dev/null +++ b/docs/book/src/topics/powervs/external-cloud-provider.md @@ -0,0 +1,30 @@ +# IBM Power VS External Cloud Provider + +## Steps + +- To deploy a Power VS workload cluster with IBM Power VS external cloud provider, create a cluster configuration with the [external cloud provider template](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/templates/cluster-template-powervs-cloud-provider.yaml) +- The [external cloud provider template](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/templates/cluster-template-powervs-cloud-provider.yaml) will use [clusterresourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) and will create the necessary config map, secret and roles to run the cloud controller manager +- As a prerequisite set the [`powervs-provider-id-fmt`](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/64c9e1d17f1733c721f45a559edba3f4b712bcb0/main.go#L220) flag with value v2 + + ``` + IBMPOWERVS_SSHKEY_NAME="my-pub-key" \ + IBMPOWERVS_VIP="192.168.151.22" \ + IBMPOWERVS_VIP_EXTERNAL="158.175.162.22" \ + IBMPOWERVS_VIP_CIDR="29" \ + IBMPOWERVS_IMAGE_NAME="capibm-powervs-centos-8-1-22-4" \ + IBMPOWERVS_SERVICE_INSTANCE_ID="7845d372-d4e1-46b8-91fc-41051c984601" \ + IBMPOWERVS_NETWORK_NAME="capi-test-3" \ + IBMACCOUNT_ID="1234567889" \ + IBMPOWERVS_RESOURCE_GROUP="powervs-resource-group-name" \ + IBMVPC_SUBNET_NAMES="subnet-name" \ + IBMVPC_NAME="vpc-name" \ + IBMVPC_REGION="vpc-region" \ + IBMPOWERVS_REGION="powervs-region" \ + IBMPOWERVS_ZONE="powervs-zone" \ + BASE64_API_KEY=$(echo -n $IBMCLOUD_API_KEY | base64) \ + clusterctl generate cluster ibm-powervs-1 --kubernetes-version v1.22.4 \ + --target-namespace default \ + --control-plane-machine-count=3 \ + --worker-machine-count=1 \ + --from ./cluster-template-powervs-external-cloud-provider.yaml | kubectl apply -f - + ``` diff --git a/hack/ccm/Dockerfile b/hack/ccm/Dockerfile new file mode 100644 index 0000000000..65e3023147 --- /dev/null +++ b/hack/ccm/Dockerfile @@ -0,0 +1,42 @@ +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG GOLANG_IMAGE=golang:1.17 +ARG TARGETPLATFORM=linux/amd64 +ARG ARCH=amd64 + +# Build vpcctl binary +FROM ${GOLANG_IMAGE} as vpc-builder +ARG ARCH +ARG VPC_CONTROLLER_COMMIT +WORKDIR /build +RUN git clone https://github.com/openshift/cloud-provider-vpc-controller +RUN cd cloud-provider-vpc-controller/cmd && git checkout $VPC_CONTROLLER_COMMIT && CGO_ENABLED=0 GOARCH=$ARCH go build \ + -ldflags "-s -w" -o /build/vpcctl . + +# Build IBM cloud controller manager binary +FROM ${GOLANG_IMAGE} AS ccm-builder +ARG ARCH +ARG POWERVS_CLOUD_CONTROLLER_COMMIT +WORKDIR /build +RUN git clone https://github.com/openshift/cloud-provider-powervs +RUN cd cloud-provider-powervs && git checkout $POWERVS_CLOUD_CONTROLLER_COMMIT && CGO_ENABLED=0 GOARCH=$ARCH go build \ + -ldflags "-s -w" -o /build/ibm-cloud-controller-manager . + +# Assemble the final image +FROM --platform=$TARGETPLATFORM quay.io/centos/centos:stream8 AS centos-base +LABEL description="IBM PowerVS Cloud Controller Manager" +COPY --from=vpc-builder /build/vpcctl /bin/vpcctl +COPY --from=ccm-builder /build/ibm-cloud-controller-manager /bin/ibm-cloud-controller-manager +ENTRYPOINT [ "/bin/ibm-cloud-controller-manager" ] diff --git a/hack/ccm/Makefile b/hack/ccm/Makefile new file mode 100644 index 0000000000..f6139234c0 --- /dev/null +++ b/hack/ccm/Makefile @@ -0,0 +1,54 @@ +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REGISTRY=gcr.io/k8s-staging-capi-ibmcloud +IMG=powervs-cloud-controller-manager + +# VPC_CONTROLLER_COMMIT can be fetched from here https://github.com/openshift/cloud-provider-vpc-controller/commits/master +VPC_CONTROLLER_COMMIT?=9b99b4e +# POWERVS_CLOUD_CONTROLLER_COMMIT can be fetched from here https://github.com/openshift/cloud-provider-powervs/commits/main +POWERVS_CLOUD_CONTROLLER_COMMIT?=a6bfa07 +TAG?=$(VPC_CONTROLLER_COMMIT)_$(POWERVS_CLOUD_CONTROLLER_COMMIT) + +build-image-and-push-linux-amd64: init-buildx + { \ + set -e ; \ + docker buildx build \ + --build-arg TARGETPLATFORM=linux/amd64 --build-arg ARCH=amd64 \ + --build-arg VPC_CONTROLLER_COMMIT=$(VPC_CONTROLLER_COMMIT) --build-arg POWERVS_CLOUD_CONTROLLER_COMMIT=$(POWERVS_CLOUD_CONTROLLER_COMMIT)\ + -t $(REGISTRY)/$(IMG):$(TAG)_linux_amd64 . --push --target centos-base; \ + } + +build-image-and-push-linux-ppc64le: init-buildx + { \ + set -e ; \ + docker buildx build \ + --build-arg TARGETPLATFORM=linux/ppc64le --build-arg ARCH=ppc64le\ + --build-arg VPC_CONTROLLER_COMMIT=$(VPC_CONTROLLER_COMMIT) --build-arg POWERVS_CLOUD_CONTROLLER_COMMIT=$(POWERVS_CLOUD_CONTROLLER_COMMIT)\ + -t $(REGISTRY)/$(IMG):$(TAG)_linux_ppc64le . --push --target centos-base; \ + } + +init-buildx: + # Ensure we use a builder that can leverage it (the default on linux will not) + docker buildx rm multiarch-multiplatform-builder + docker buildx create --use --name=multiarch-multiplatform-builder + docker run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes + # Register gcloud as a Docker credential helper. + # Required for "docker buildx build --push". + gcloud auth configure-docker --quiet + + +build-and-push-multi-arch: build-image-and-push-linux-amd64 build-image-and-push-linux-ppc64le + docker manifest create --amend $(REGISTRY)/$(IMG):$(TAG) $(REGISTRY)/$(IMG):$(TAG)_linux_amd64 $(REGISTRY)/$(IMG):$(TAG)_linux_ppc64le + docker manifest push -p $(REGISTRY)/$(IMG):$(TAG) diff --git a/templates/cluster-template-powervs-cloud-provider.yaml b/templates/cluster-template-powervs-cloud-provider.yaml new file mode 100644 index 0000000000..7f68f4d468 --- /dev/null +++ b/templates/cluster-template-powervs-cloud-provider.yaml @@ -0,0 +1,496 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IBMPowerVSCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + controlPlaneEndpoint: + host: "${IBMPOWERVS_VIP_EXTERNAL}" + port: 6443 +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IBMPowerVSMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - "${IBMPOWERVS_VIP}" + - "${IBMPOWERVS_VIP_EXTERNAL}" + extraArgs: + cloud-provider: external + controlPlaneEndpoint: ${IBMPOWERVS_VIP}:6443 + controllerManager: + extraArgs: + cloud-provider: external + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:6443 + token: "" + caCertHashes: [ ] + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + useExperimentalRetryJoin: true + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + files: + - path: /etc/kubernetes/manifests/kube-vip.yaml + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: vip_interface + value: "env2" + - name: port + value: "6443" + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: "${IBMPOWERVS_VIP}" + image: docker.io/plndr/kube-vip:v0.3.7 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_TIME + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: "root:root" + permissions: "0744" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IBMPowerVSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/control-plane: "" + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:6443 + token: "" + caCertHashes: [] + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cloud-conf +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ibmpowervs-cfg +data: + ibmpowervs-cloud-conf.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: ibmpowervs-cloud-config + namespace: kube-system + data: + ibmpowervs.conf: | + [global] + version = 1.1.0 + [kubernetes] + config-file = "" + [provider] + cluster-default-provider = g2 + accountID = ${IBMACCOUNT_ID} + clusterID = ${CLUSTER_NAME} + g2workerServiceAccountID = ${IBMACCOUNT_ID} + g2Credentials = /etc/ibm-secret/ibmcloud_api_key + g2ResourceGroupName = ${IBMPOWERVS_RESOURCE_GROUP} + g2VpcSubnetNames = ${IBMVPC_SUBNET_NAMES} + g2VpcName = ${IBMVPC_NAME} + region = ${IBMVPC_REGION} + powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID} + powerVSRegion = ${IBMPOWERVS_REGION} + powerVSZone = ${IBMPOWERVS_ZONE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ibmpowervs-credential +type: addons.cluster.x-k8s.io/resource-set +stringData: + ibmpowervs-credential.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: ibmpowervs-cloud-credential + namespace: kube-system + data: + ibmcloud_api_key: ${BASE64_API_KEY} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon +data: + ibmpowervs-ccm-external.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + resources: + - serviceaccounts/token + verbs: + - create + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ibmpowervs-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: ibmpowervs-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: ibmpowervs-cloud-controller-manager + #TODO: Replace the image name + image: quay.io/kabhat/powervs-cloud-controller-manager:v1 + args: + - --v=2 + - --cloud-provider=ibm + - --cloud-config=/etc/cloud/ibmpowervs.conf + - --use-service-account-credentials=true + env: + - name: VPCCTL_CLOUD_CONFIG + value: /etc/cloud/ibmpowervs.conf + - name: VPCCTL_PUBLIC_ENDPOINT + value: "true" + volumeMounts: + - mountPath: /etc/cloud + name: ibmpowervs-config-volume + readOnly: true + - mountPath: /etc/ibm-secret + name: ibm-secret + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: ibmpowervs-config-volume + configMap: + name: ibmpowervs-cloud-config + - name: ibm-secret + secret: + secretName: ibmpowervs-cloud-credential