From d0b273bb5addd52196f67a7a01d858824353903c Mon Sep 17 00:00:00 2001 From: Karthik Bhat Date: Tue, 22 Nov 2022 11:44:13 +0530 Subject: [PATCH] Add clusterclass support for Power VS (#967) --- PROJECT | 3 + .../ibmpowervsclustertemplate_types.go | 62 +++ api/v1beta1/zz_generated.deepcopy.go | 91 +++ ...r.x-k8s.io_ibmpowervsclustertemplates.yaml | 130 +++++ config/crd/kustomization.yaml | 3 + ...jection_in_ibmpowervsclustertemplates.yaml | 8 + ...webhook_in_ibmpowervsclustertemplates.yaml | 17 + ...ibmpowervsclustertemplate_editor_role.yaml | 24 + ...ibmpowervsclustertemplate_viewer_role.yaml | 20 + ...ure_v1beta1_ibmpowervsclustertemplate.yaml | 6 + ...-template-simple-powervs-clusterclass.yaml | 518 ++++++++++++++++++ 11 files changed, 882 insertions(+) create mode 100644 api/v1beta1/ibmpowervsclustertemplate_types.go create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml create mode 100644 config/crd/patches/cainjection_in_ibmpowervsclustertemplates.yaml create mode 100644 config/crd/patches/webhook_in_ibmpowervsclustertemplates.yaml create mode 100644 config/rbac/ibmpowervsclustertemplate_editor_role.yaml create mode 100644 config/rbac/ibmpowervsclustertemplate_viewer_role.yaml create mode 100644 config/samples/infrastructure_v1beta1_ibmpowervsclustertemplate.yaml create mode 100644 templates/cluster-template-simple-powervs-clusterclass.yaml diff --git a/PROJECT b/PROJECT index b08fa3a75..5c3816232 100644 --- a/PROJECT +++ b/PROJECT @@ -40,4 +40,7 @@ resources: - group: infrastructure kind: IBMPowerVSImage version: v1beta1 +- group: infrastructure + kind: IBMPowerVSClusterTemplate + version: v1beta1 version: "2" diff --git a/api/v1beta1/ibmpowervsclustertemplate_types.go b/api/v1beta1/ibmpowervsclustertemplate_types.go new file mode 100644 index 000000000..f83136e2c --- /dev/null +++ b/api/v1beta1/ibmpowervsclustertemplate_types.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// IBMPowerVSClusterTemplateSpec defines the desired state of IBMPowerVSClusterTemplate. +type IBMPowerVSClusterTemplateSpec struct { + Template IBMPowerVSClusterTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=ibmpowervsclustertemplates,scope=Namespaced,categories=cluster-api,shortName=ibmpowervsct +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of IBMPowerVSClusterTemplate" + +// IBMPowerVSClusterTemplate is the schema for IBM Power VS Kubernetes Cluster Templates. +type IBMPowerVSClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec IBMPowerVSClusterTemplateSpec `json:"spec,omitempty"` +} + +//+kubebuilder:object:root=true + +// IBMPowerVSClusterTemplateList contains a list of IBMPowerVSClusterTemplate. +type IBMPowerVSClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IBMPowerVSClusterTemplate `json:"items"` +} + +// IBMPowerVSClusterTemplateResource describes the data needed to create an IBMPowerVSCluster from a template. +type IBMPowerVSClusterTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta capiv1beta1.ObjectMeta `json:"metadata,omitempty"` + Spec IBMPowerVSClusterSpec `json:"spec"` +} + +func init() { + SchemeBuilder.Register(&IBMPowerVSClusterTemplate{}, &IBMPowerVSClusterTemplateList{}) +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5a0339a36..8616988e2 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -119,6 +119,97 @@ func (in *IBMPowerVSClusterStatus) DeepCopy() *IBMPowerVSClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IBMPowerVSClusterTemplate) DeepCopyInto(out *IBMPowerVSClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMPowerVSClusterTemplate. +func (in *IBMPowerVSClusterTemplate) DeepCopy() *IBMPowerVSClusterTemplate { + if in == nil { + return nil + } + out := new(IBMPowerVSClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IBMPowerVSClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IBMPowerVSClusterTemplateList) DeepCopyInto(out *IBMPowerVSClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IBMPowerVSClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMPowerVSClusterTemplateList. +func (in *IBMPowerVSClusterTemplateList) DeepCopy() *IBMPowerVSClusterTemplateList { + if in == nil { + return nil + } + out := new(IBMPowerVSClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IBMPowerVSClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IBMPowerVSClusterTemplateResource) DeepCopyInto(out *IBMPowerVSClusterTemplateResource) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMPowerVSClusterTemplateResource. +func (in *IBMPowerVSClusterTemplateResource) DeepCopy() *IBMPowerVSClusterTemplateResource { + if in == nil { + return nil + } + out := new(IBMPowerVSClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IBMPowerVSClusterTemplateSpec) DeepCopyInto(out *IBMPowerVSClusterTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMPowerVSClusterTemplateSpec. +func (in *IBMPowerVSClusterTemplateSpec) DeepCopy() *IBMPowerVSClusterTemplateSpec { + if in == nil { + return nil + } + out := new(IBMPowerVSClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IBMPowerVSImage) DeepCopyInto(out *IBMPowerVSImage) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml new file mode 100644 index 000000000..ab172e91b --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: ibmpowervsclustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: IBMPowerVSClusterTemplate + listKind: IBMPowerVSClusterTemplateList + plural: ibmpowervsclustertemplates + shortNames: + - ibmpowervsct + singular: ibmpowervsclustertemplate + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Time duration since creation of IBMPowerVSClusterTemplate + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: IBMPowerVSClusterTemplate is the schema for IBM Power VS Kubernetes + Cluster Templates. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IBMPowerVSClusterTemplateSpec defines the desired state of + IBMPowerVSClusterTemplate. + properties: + template: + description: IBMPowerVSClusterTemplateResource describes the data + needed to create an IBMPowerVSCluster from a template. + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: IBMPowerVSClusterSpec defines the desired state of + IBMPowerVSCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint + used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + network: + description: Network is the reference to the Network to use + for this cluster. + properties: + id: + description: ID of resource + minLength: 1 + type: string + name: + description: Name of resource + minLength: 1 + type: string + regex: + description: Regular expression to match resource, In + case of multiple resources matches the provided regular + expression the first matched resource will be selected + minLength: 1 + type: string + type: object + serviceInstanceID: + description: ServiceInstanceID is the id of the power cloud + instance where the vsi instance will get deployed. + minLength: 1 + type: string + required: + - network + - serviceInstanceID + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d0913625d..4bacca39b 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -16,6 +16,7 @@ resources: - bases/infrastructure.cluster.x-k8s.io_ibmpowervsmachines.yaml - bases/infrastructure.cluster.x-k8s.io_ibmpowervsmachinetemplates.yaml - bases/infrastructure.cluster.x-k8s.io_ibmpowervsimages.yaml +- bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -28,6 +29,7 @@ patchesStrategicMerge: - patches/webhook_in_ibmpowervsmachinetemplates.yaml - patches/webhook_in_ibmvpcmachinetemplates.yaml - patches/webhook_in_ibmpowervsimages.yaml +#- patches/webhook_in_ibmpowervsclustertemplates.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -39,6 +41,7 @@ patchesStrategicMerge: - patches/cainjection_in_ibmpowervsmachinetemplates.yaml - patches/cainjection_in_ibmvpcmachinetemplates.yaml - patches/cainjection_in_ibmpowervsimages.yaml +#- patches/cainjection_in_ibmpowervsclustertemplates.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_ibmpowervsclustertemplates.yaml b/config/crd/patches/cainjection_in_ibmpowervsclustertemplates.yaml new file mode 100644 index 000000000..87919b2f9 --- /dev/null +++ b/config/crd/patches/cainjection_in_ibmpowervsclustertemplates.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: ibmpowervsclustertemplates.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/webhook_in_ibmpowervsclustertemplates.yaml b/config/crd/patches/webhook_in_ibmpowervsclustertemplates.yaml new file mode 100644 index 000000000..b7f27f32f --- /dev/null +++ b/config/crd/patches/webhook_in_ibmpowervsclustertemplates.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ibmpowervsclustertemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/rbac/ibmpowervsclustertemplate_editor_role.yaml b/config/rbac/ibmpowervsclustertemplate_editor_role.yaml new file mode 100644 index 000000000..c039f8c40 --- /dev/null +++ b/config/rbac/ibmpowervsclustertemplate_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit ibmpowervsclustertemplates. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ibmpowervsclustertemplate-editor-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - ibmpowervsclustertemplates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - ibmpowervsclustertemplates/status + verbs: + - get diff --git a/config/rbac/ibmpowervsclustertemplate_viewer_role.yaml b/config/rbac/ibmpowervsclustertemplate_viewer_role.yaml new file mode 100644 index 000000000..8516ac673 --- /dev/null +++ b/config/rbac/ibmpowervsclustertemplate_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view ibmpowervsclustertemplates. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ibmpowervsclustertemplate-viewer-role +rules: +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - ibmpowervsclustertemplates + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - ibmpowervsclustertemplates/status + verbs: + - get diff --git a/config/samples/infrastructure_v1beta1_ibmpowervsclustertemplate.yaml b/config/samples/infrastructure_v1beta1_ibmpowervsclustertemplate.yaml new file mode 100644 index 000000000..c928151c3 --- /dev/null +++ b/config/samples/infrastructure_v1beta1_ibmpowervsclustertemplate.yaml @@ -0,0 +1,6 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSClusterTemplate +metadata: + name: ibmpowervsclustertemplate-sample +spec: + # TODO(user): Add fields here diff --git a/templates/cluster-template-simple-powervs-clusterclass.yaml b/templates/cluster-template-simple-powervs-clusterclass.yaml new file mode 100644 index 000000000..75e57e234 --- /dev/null +++ b/templates/cluster-template-simple-powervs-clusterclass.yaml @@ -0,0 +1,518 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + topology: + class: "${CLUSTER_CLASS_NAME}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 2 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME:="powervs-cc"} +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IBMPowerVSMachineTemplate + name: "${CLUSTER_CLASS_NAME}-control-plane-machinetemplate" + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: "${CLUSTER_CLASS_NAME}-control-plane" + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IBMPowerVSClusterTemplate + name: "${CLUSTER_CLASS_NAME}-cluster-template" + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "${CLUSTER_CLASS_NAME}-md-0" + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IBMPowerVSMachineTemplate + name: "${CLUSTER_CLASS_NAME}-worker-machinetemplate" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSClusterTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-cluster-template" +spec: + template: + spec: + controlPlaneEndpoint: + host: "${IBMPOWERVS_VIP_EXTERNAL}" + port: 6443 + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-control-plane" +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - "${IBMPOWERVS_VIP}" + - "${IBMPOWERVS_VIP_EXTERNAL}" + extraArgs: + cloud-provider: external + controlPlaneEndpoint: ${IBMPOWERVS_VIP}:6443 + controllerManager: + extraArgs: + cloud-provider: external + enable-hostpath-provisioner: "true" + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0744" + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: 192.168.167.85:6443 + caCertHashes: [] + token: "" + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + useExperimentalRetryJoin: true +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/control-plane: "" + name: "${CLUSTER_CLASS_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:6443 + caCertHashes: [] + token: "" + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-control-plane-machinetemplate" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memory: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:="4"} + processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + sysType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} + procType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="shared"} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-worker-machinetemplate" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memory: ${IBMPOWERVS_COMPUTE_MEMORY:="4"} + processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"} + sysType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"} + procType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="shared"} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cloud-conf +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ibmpowervs-cfg +data: + ibmpowervs-cloud-conf.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: ibmpowervs-cloud-config + namespace: kube-system + data: + ibmpowervs.conf: | + [global] + version = 1.1.0 + [kubernetes] + config-file = "" + [provider] + cluster-default-provider = g2 + accountID = ${IBMACCOUNT_ID} + clusterID = ${CLUSTER_NAME} + g2workerServiceAccountID = ${IBMACCOUNT_ID} + g2Credentials = /etc/ibm-secret/ibmcloud_api_key + g2ResourceGroupName = ${IBMVPC_RESOURCE_GROUP:=""} + g2VpcSubnetNames = ${IBMVPC_SUBNET_NAMES:=""} + g2VpcName = ${IBMVPC_NAME:=""} + region = ${IBMVPC_REGION:=""} + powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID} + powerVSRegion = ${IBMPOWERVS_REGION} + powerVSZone = ${IBMPOWERVS_ZONE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ibmpowervs-credential +type: addons.cluster.x-k8s.io/resource-set +stringData: + ibmpowervs-credential.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: ibmpowervs-cloud-credential + namespace: kube-system + data: + ibmcloud_api_key: ${BASE64_API_KEY} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon +data: + ibmpowervs-ccm-external.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + resources: + - serviceaccounts/token + verbs: + - create + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ibmpowervs-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: ibmpowervs-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: ibmpowervs-cloud-controller-manager + image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:9b99b4e_a6bfa07 + args: + - --v=2 + - --cloud-provider=ibm + - --cloud-config=/etc/cloud/ibmpowervs.conf + - --use-service-account-credentials=true + env: + - name: VPCCTL_CLOUD_CONFIG + value: /etc/cloud/ibmpowervs.conf + - name: VPCCTL_PUBLIC_ENDPOINT + value: "true" + volumeMounts: + - mountPath: /etc/cloud + name: ibmpowervs-config-volume + readOnly: true + - mountPath: /etc/ibm-secret + name: ibm-secret + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: ibmpowervs-config-volume + configMap: + name: ibmpowervs-cloud-config + - name: ibm-secret + secret: + secretName: ibmpowervs-cloud-credential