diff --git a/Makefile b/Makefile index 2ec41ab34..ffbb025c1 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin GO_INSTALL = ./scripts/go_install.sh E2E_CONF_FILE_ENVSUBST := $(REPO_ROOT)/test/e2e/config/ibmcloud-e2e-envsubst.yaml E2E_TEMPLATES := $(REPO_ROOT)/test/e2e/data/templates +TEMPLATES_DIR := $(REPO_ROOT)/templates GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint @@ -174,6 +175,14 @@ generate-go-conversions: $(CONVERSION_GEN) ## Generate conversions go code --output-file-base=zz_generated.conversion $(CONVERSION_GEN_OUTPUT_BASE) \ --go-header-file=./hack/boilerplate/boilerplate.generatego.txt +.PHONY: generate-templates +generate-templates: $(KUSTOMIZE) + $(KUSTOMIZE) build $(TEMPLATES_DIR)/cluster-template --load-restrictor LoadRestrictionsNone > $(TEMPLATES_DIR)/cluster-template.yaml + $(KUSTOMIZE) build $(TEMPLATES_DIR)/cluster-template-powervs --load-restrictor LoadRestrictionsNone > $(TEMPLATES_DIR)/cluster-template-powervs.yaml + $(KUSTOMIZE) build $(TEMPLATES_DIR)/cluster-template-powervs-cloud-provider --load-restrictor LoadRestrictionsNone > $(TEMPLATES_DIR)/cluster-template-powervs-cloud-provider.yaml + $(KUSTOMIZE) build $(TEMPLATES_DIR)/cluster-template-powervs-clusterclass --load-restrictor LoadRestrictionsNone > $(TEMPLATES_DIR)/cluster-template-powervs-clusterclass.yaml + $(KUSTOMIZE) build $(TEMPLATES_DIR)/cluster-template-vpc-load-balancer --load-restrictor LoadRestrictionsNone > $(TEMPLATES_DIR)/cluster-template-vpc-load-balancer.yaml + .PHONY: generate-e2e-templates generate-e2e-templates: $(KUSTOMIZE) ifeq ($(E2E_FLAVOR), powervs-md-remediation) diff --git a/templates/addons/crs-powervs.yaml b/templates/addons/crs-powervs.yaml new file mode 100644 index 000000000..d7a4d8539 --- /dev/null +++ b/templates/addons/crs-powervs.yaml @@ -0,0 +1,262 @@ +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cloud-conf +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ibmpowervs-cfg +data: + ibmpowervs-cloud-conf.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: ibmpowervs-cloud-config + namespace: kube-system + data: + ibmpowervs.conf: | + [global] + version = 1.1.0 + [kubernetes] + config-file = "" + [provider] + cluster-default-provider = g2 + accountID = ${IBMACCOUNT_ID} + clusterID = ${CLUSTER_NAME} + g2workerServiceAccountID = ${IBMACCOUNT_ID} + g2Credentials = /etc/ibm-secret/ibmcloud_api_key + g2ResourceGroupName = ${IBMVPC_RESOURCE_GROUP:=""} + g2VpcSubnetNames = ${IBMVPC_SUBNET_NAMES:=""} + g2VpcName = ${IBMVPC_NAME:=""} + region = ${IBMVPC_REGION:=""} + powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID} + powerVSRegion = ${IBMPOWERVS_REGION} + powerVSZone = ${IBMPOWERVS_ZONE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ibmpowervs-credential +type: addons.cluster.x-k8s.io/resource-set +stringData: + ibmpowervs-credential.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: ibmpowervs-cloud-credential + namespace: kube-system + data: + ibmcloud_api_key: ${BASE64_API_KEY} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon +data: + ibmpowervs-ccm-external.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + resources: + - serviceaccounts/token + verbs: + - create + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ibmpowervs-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: ibmpowervs-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: ibmpowervs-cloud-controller-manager + image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:07d19bf + args: + - --v=2 + - --cloud-provider=ibm + - --cloud-config=/etc/cloud/ibmpowervs.conf + - --use-service-account-credentials=true + env: + - name: ENABLE_VPC_PUBLIC_ENDPOINT + value: "true" + volumeMounts: + - mountPath: /etc/cloud + name: ibmpowervs-config-volume + readOnly: true + - mountPath: /etc/ibm-secret + name: ibm-secret + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: ibmpowervs-config-volume + configMap: + name: ibmpowervs-cloud-config + - name: ibm-secret + secret: + secretName: ibmpowervs-cloud-credential diff --git a/templates/addons/crs.yaml b/templates/addons/crs.yaml new file mode 100644 index 000000000..0d92d25d2 --- /dev/null +++ b/templates/addons/crs.yaml @@ -0,0 +1,256 @@ +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cloud-conf +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: Secret + name: ibm-credential + - kind: ConfigMap + name: ibm-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ibm-cfg +data: + ibm-cloud-conf.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: ibm-cloud-config + namespace: kube-system + data: + ibm.conf: | + [global] + version = 1.1.0 + [kubernetes] + config-file = "" + [provider] + cluster-default-provider = g2 + accountID = ${IBMACCOUNT_ID} + clusterID = ${CLUSTER_NAME} + g2workerServiceAccountID = ${IBMACCOUNT_ID} + g2Credentials = /etc/ibm-secret/ibmcloud_api_key + g2ResourceGroupName = ${IBMVPC_RESOURCEGROUP_NAME:=""} + g2VpcSubnetNames = "${CLUSTER_NAME}-subnet" + g2VpcName = ${IBMVPC_NAME:=""} + region = ${IBMVPC_REGION:=""} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ibm-credential +type: addons.cluster.x-k8s.io/resource-set +stringData: + ibm-credential.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: ibm-cloud-credential + namespace: kube-system + data: + ibmcloud_api_key: ${BASE64_API_KEY} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon +data: + ibm-ccm-external.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + resources: + - serviceaccounts/token + verbs: + - create + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ibm-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: ibm-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: ibm-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: ibm-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: ibm-cloud-controller-manager + image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:07d19bf + args: + - --v=2 + - --cloud-provider=ibm + - --cloud-config=/etc/cloud/ibm.conf + - --use-service-account-credentials=true + volumeMounts: + - mountPath: /etc/cloud + name: ibm-config-volume + readOnly: true + - mountPath: /etc/ibm-secret + name: ibm-secret + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: ibm-config-volume + configMap: + name: ibm-cloud-config + - name: ibm-secret + secret: + secretName: ibm-cloud-credentia diff --git a/templates/bases/powervs/cluster.yaml b/templates/bases/powervs/cluster.yaml new file mode 100644 index 000000000..9983247b7 --- /dev/null +++ b/templates/bases/powervs/cluster.yaml @@ -0,0 +1,37 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + controlPlaneEndpoint: + host: "${IBMPOWERVS_VIP_EXTERNAL}" + port: ${API_SERVER_PORT:=6443} diff --git a/templates/bases/powervs/kcp.yaml b/templates/bases/powervs/kcp.yaml new file mode 100644 index 000000000..cbff6452c --- /dev/null +++ b/templates/bases/powervs/kcp.yaml @@ -0,0 +1,131 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IBMPowerVSMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - "${IBMPOWERVS_VIP}" + - "${IBMPOWERVS_VIP_EXTERNAL}" + controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + token: "" + caCertHashes: [ ] + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + useExperimentalRetryJoin: true + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + files: + - path: /etc/kubernetes/manifests/kube-vip.yaml + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "${API_SERVER_PORT:=6443}" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: "root:root" + permissions: "0744" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} diff --git a/templates/bases/powervs/kubeadm-config.yaml b/templates/bases/powervs/kubeadm-config.yaml new file mode 100644 index 000000000..629780e19 --- /dev/null +++ b/templates/bases/powervs/kubeadm-config.yaml @@ -0,0 +1,28 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + token: "" + caCertHashes: [] + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname diff --git a/templates/bases/powervs/kustomization.yaml b/templates/bases/powervs/kustomization.yaml new file mode 100644 index 000000000..5a202a95d --- /dev/null +++ b/templates/bases/powervs/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- cluster.yaml +- kcp.yaml +- md.yaml +- kubeadm-config.yaml diff --git a/templates/bases/powervs/md.yaml b/templates/bases/powervs/md.yaml new file mode 100644 index 000000000..49f4f5173 --- /dev/null +++ b/templates/bases/powervs/md.yaml @@ -0,0 +1,38 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memoryGiB: ${IBMPOWERVS_COMPUTE_MEMORY:=4} + processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"} + systemType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"} + processorType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="Shared"} diff --git a/templates/bases/vpc/cluster.yaml b/templates/bases/vpc/cluster.yaml new file mode 100644 index 000000000..e7267fb5f --- /dev/null +++ b/templates/bases/vpc/cluster.yaml @@ -0,0 +1,38 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCCluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + region: "${IBMVPC_REGION}" + zone: "${IBMVPC_ZONE}" + resourceGroup: "${IBMVPC_RESOURCEGROUP}" + vpc: "${IBMVPC_NAME}" diff --git a/templates/bases/vpc/kcp.yaml b/templates/bases/vpc/kcp.yaml new file mode 100644 index 000000000..217a83d97 --- /dev/null +++ b/templates/bases/vpc/kcp.yaml @@ -0,0 +1,52 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IBMVPCMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + controllerManager: + extraArgs: {enable-hostpath-provisioner: 'true'} + apiServer: + certSANs: [localhost, 127.0.0.1] + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' +--- +kind: IBMVPCMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + image: + name: "${IBMVPC_IMAGE_NAME}" + zone: "${IBMVPC_ZONE}" + profile: "${IBMVPC_PROFILE}" + sshKeys: + - name: "${IBMVPC_SSHKEY_NAME}" diff --git a/templates/bases/vpc/kubeadm-config.yaml b/templates/bases/vpc/kubeadm-config.yaml new file mode 100644 index 000000000..bcde9149e --- /dev/null +++ b/templates/bases/vpc/kubeadm-config.yaml @@ -0,0 +1,12 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% diff --git a/templates/bases/vpc/kustomization.yaml b/templates/bases/vpc/kustomization.yaml new file mode 100644 index 000000000..5a202a95d --- /dev/null +++ b/templates/bases/vpc/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- cluster.yaml +- kcp.yaml +- md.yaml +- kubeadm-config.yaml diff --git a/templates/bases/vpc/md.yaml b/templates/bases/vpc/md.yaml new file mode 100644 index 000000000..67fb7b60f --- /dev/null +++ b/templates/bases/vpc/md.yaml @@ -0,0 +1,34 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + image: + name: "${IBMVPC_IMAGE_NAME}" + zone: "${IBMVPC_ZONE}" + profile: "${IBMVPC_PROFILE}" + sshKeys: + - name: "${IBMVPC_SSHKEY_NAME}" diff --git a/templates/cluster-template-powervs-cloud-provider.yaml b/templates/cluster-template-powervs-cloud-provider.yaml index fc1167d7a..debab20e4 100644 --- a/templates/cluster-template-powervs-cloud-provider.yaml +++ b/templates/cluster-template-powervs-cloud-provider.yaml @@ -3,58 +3,51 @@ kind: Cluster metadata: labels: ccm: external - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: clusterNetwork: pods: cidrBlocks: - - ${POD_CIDR:="192.168.0.0/16"} + - ${POD_CIDR:="192.168.0.0/16"} serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} services: cidrBlocks: - - ${SERVICE_CIDR:="10.128.0.0/12"} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: IBMPowerVSCluster - name: "${CLUSTER_NAME}" + - ${SERVICE_CIDR:="10.128.0.0/12"} controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSCluster + name: ${CLUSTER_NAME} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSCluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" controlPlaneEndpoint: - host: "${IBMPOWERVS_VIP_EXTERNAL}" + host: ${IBMPOWERVS_VIP_EXTERNAL} port: ${API_SERVER_PORT:=6443} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} --- -kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: - version: "${KUBERNETES_VERSION}" - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: IBMPowerVSMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: "${CLUSTER_NAME}-control-plane" kubeadmConfigSpec: clusterConfiguration: apiServer: certSANs: - - "${IBMPOWERVS_VIP}" - - "${IBMPOWERVS_VIP_EXTERNAL}" + - ${IBMPOWERVS_VIP} + - ${IBMPOWERVS_VIP_EXTERNAL} extraArgs: cloud-provider: external controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} @@ -62,6 +55,70 @@ spec: extraArgs: cloud-provider: external enable-hostpath-provisioner: "true" + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "${API_SERVER_PORT:=6443}" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0744" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -73,8 +130,8 @@ spec: discovery: bootstrapToken: apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + caCertHashes: [] token: "" - caCertHashes: [ ] unsafeSkipCAVerification: false nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -82,136 +139,77 @@ spec: cloud-provider: external eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% name: '{{ v1.local_hostname }}' - useExperimentalRetryJoin: true preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname - files: - - path: /etc/kubernetes/manifests/kube-vip.yaml - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "${API_SERVER_PORT:=6443}" - - name: vip_interface - value: env2 - - name: vip_cidr - value: "${IBMPOWERVS_VIP_CIDR}" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "${IBMPOWERVS_VIP}" - image: ghcr.io/kube-vip/kube-vip:v0.4.4 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig - status: {} - owner: "root:root" - permissions: "0744" + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + useExperimentalRetryJoin: true + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: - clusterName: "${CLUSTER_NAME}" + clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: template: spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" bootstrap: configRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} infrastructureRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} + memoryGiB: ${IBMPOWERVS_COMPUTE_MEMORY:=4} network: - name: "${IBMPOWERVS_NETWORK_NAME}" - memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} - processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} - systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="Shared"} + processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} + systemType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -219,7 +217,7 @@ metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} cluster.x-k8s.io/control-plane: "" - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: @@ -227,8 +225,8 @@ spec: discovery: bootstrapToken: apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} - token: "" caCertHashes: [] + token: "" unsafeSkipCAVerification: false nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -237,11 +235,11 @@ spec: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% name: '{{ v1.local_hostname }}' preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -252,18 +250,15 @@ spec: matchLabels: ccm: external resources: - - kind: Secret - name: ibmpowervs-credential - - kind: ConfigMap - name: ibmpowervs-cfg - - kind: ConfigMap - name: cloud-controller-manager-addon + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon strategy: ApplyOnce --- apiVersion: v1 -kind: ConfigMap -metadata: - name: ibmpowervs-cfg data: ibmpowervs-cloud-conf.yaml: |- apiVersion: v1 @@ -290,12 +285,14 @@ data: powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID} powerVSRegion = ${IBMPOWERVS_REGION} powerVSZone = ${IBMPOWERVS_ZONE} +kind: ConfigMap +metadata: + name: ibmpowervs-cfg --- apiVersion: v1 kind: Secret metadata: name: ibmpowervs-credential -type: addons.cluster.x-k8s.io/resource-set stringData: ibmpowervs-credential.yaml: |- apiVersion: v1 @@ -305,11 +302,9 @@ stringData: namespace: kube-system data: ibmcloud_api_key: ${BASE64_API_KEY} +type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon data: ibmpowervs-ccm-external.yaml: |- apiVersion: v1 @@ -505,3 +500,6 @@ data: - name: ibm-secret secret: secretName: ibmpowervs-cloud-credential +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon diff --git a/templates/cluster-template-powervs-cloud-provider/cluster.yaml b/templates/cluster-template-powervs-cloud-provider/cluster.yaml new file mode 100644 index 000000000..2c4cfdd59 --- /dev/null +++ b/templates/cluster-template-powervs-cloud-provider/cluster.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + ccm: external diff --git a/templates/cluster-template-powervs-cloud-provider/kcp.yaml b/templates/cluster-template-powervs-cloud-provider/kcp.yaml new file mode 100644 index 000000000..e252e978c --- /dev/null +++ b/templates/cluster-template-powervs-cloud-provider/kcp.yaml @@ -0,0 +1,13 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external diff --git a/templates/cluster-template-powervs-cloud-provider/kubeadm-config.yaml b/templates/cluster-template-powervs-cloud-provider/kubeadm-config.yaml new file mode 100644 index 000000000..dbe04179a --- /dev/null +++ b/templates/cluster-template-powervs-cloud-provider/kubeadm-config.yaml @@ -0,0 +1,6 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/control-plane: "" + name: "${CLUSTER_NAME}-md-0" diff --git a/templates/cluster-template-powervs-cloud-provider/kustomization.yaml b/templates/cluster-template-powervs-cloud-provider/kustomization.yaml new file mode 100644 index 000000000..a347ae125 --- /dev/null +++ b/templates/cluster-template-powervs-cloud-provider/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- ../bases/powervs +- ../addons/crs-powervs.yaml + +patches: +- path: cluster.yaml +- path: kcp.yaml +- path: kubeadm-config.yaml + +sortOptions: + order: fifo diff --git a/templates/cluster-template-simple-powervs-clusterclass.yaml b/templates/cluster-template-powervs-clusterclass.yaml similarity index 70% rename from templates/cluster-template-simple-powervs-clusterclass.yaml rename to templates/cluster-template-powervs-clusterclass.yaml index 1a7d16f7e..3624ad02d 100644 --- a/templates/cluster-template-simple-powervs-clusterclass.yaml +++ b/templates/cluster-template-powervs-clusterclass.yaml @@ -3,27 +3,27 @@ kind: Cluster metadata: labels: ccm: external - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: clusterNetwork: pods: cidrBlocks: - - ${POD_CIDR:="192.168.0.0/16"} + - ${POD_CIDR:="192.168.0.0/16"} serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} services: cidrBlocks: - - ${SERVICE_CIDR:="10.128.0.0/12"} + - ${SERVICE_CIDR:="10.128.0.0/12"} topology: - class: "${IBMPOWERVS_CLUSTER_CLASS_NAME}" + class: ${IBMPOWERVS_CLUSTER_CLASS_NAME} controlPlane: replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} workers: machineDeployments: - - class: default-worker - name: md-0 - replicas: 2 + - class: default-worker + name: md-0 + replicas: 2 --- apiVersion: cluster.x-k8s.io/v1beta1 kind: ClusterClass @@ -35,49 +35,49 @@ spec: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSClusterTemplate - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template workers: machineDeployments: - - class: default-worker - template: - bootstrap: - ref: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0" - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: IBMPowerVSMachineTemplate - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate" + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSClusterTemplate metadata: - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template spec: template: spec: controlPlaneEndpoint: - host: "${IBMPOWERVS_VIP_EXTERNAL}" + host: ${IBMPOWERVS_VIP_EXTERNAL} port: ${API_SERVER_PORT:=6443} network: - name: "${IBMPOWERVS_NETWORK_NAME}" - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + name: ${IBMPOWERVS_NETWORK_NAME} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane spec: template: spec: @@ -85,8 +85,8 @@ spec: clusterConfiguration: apiServer: certSANs: - - "${IBMPOWERVS_VIP}" - - "${IBMPOWERVS_VIP_EXTERNAL}" + - ${IBMPOWERVS_VIP} + - ${IBMPOWERVS_VIP_EXTERNAL} extraArgs: cloud-provider: external controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} @@ -95,69 +95,69 @@ spec: cloud-provider: external enable-hostpath-provisioner: "true" files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "${API_SERVER_PORT:=6443}" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "${API_SERVER_PORT:=6443}" - - name: vip_interface - value: env2 - - name: vip_cidr - value: "${IBMPOWERVS_VIP_CIDR}" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "${IBMPOWERVS_VIP}" - image: ghcr.io/kube-vip/kube-vip:v0.4.4 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - permissions: "0744" + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0744" initConfiguration: nodeRegistration: criSocket: unix:///var/run/containerd/containerd.sock @@ -179,11 +179,11 @@ spec: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% name: '{{ v1.local_hostname }}' preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname useExperimentalRetryJoin: true --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -192,7 +192,7 @@ metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} cluster.x-k8s.io/control-plane: "" - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0 spec: template: spec: @@ -210,47 +210,47 @@ spec: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% name: '{{ v1.local_hostname }}' preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate" + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -261,18 +261,15 @@ spec: matchLabels: ccm: external resources: - - kind: Secret - name: ibmpowervs-credential - - kind: ConfigMap - name: ibmpowervs-cfg - - kind: ConfigMap - name: cloud-controller-manager-addon + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon strategy: ApplyOnce --- apiVersion: v1 -kind: ConfigMap -metadata: - name: ibmpowervs-cfg data: ibmpowervs-cloud-conf.yaml: |- apiVersion: v1 @@ -299,12 +296,14 @@ data: powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID} powerVSRegion = ${IBMPOWERVS_REGION} powerVSZone = ${IBMPOWERVS_ZONE} +kind: ConfigMap +metadata: + name: ibmpowervs-cfg --- apiVersion: v1 kind: Secret metadata: name: ibmpowervs-credential -type: addons.cluster.x-k8s.io/resource-set stringData: ibmpowervs-credential.yaml: |- apiVersion: v1 @@ -314,11 +313,9 @@ stringData: namespace: kube-system data: ibmcloud_api_key: ${BASE64_API_KEY} +type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon data: ibmpowervs-ccm-external.yaml: |- apiVersion: v1 @@ -488,16 +485,14 @@ data: serviceAccountName: cloud-controller-manager containers: - name: ibmpowervs-cloud-controller-manager - image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:9b99b4e_a6bfa07 + image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:07d19bf args: - --v=2 - --cloud-provider=ibm - --cloud-config=/etc/cloud/ibmpowervs.conf - --use-service-account-credentials=true env: - - name: VPCCTL_CLOUD_CONFIG - value: /etc/cloud/ibmpowervs.conf - - name: VPCCTL_PUBLIC_ENDPOINT + - name: ENABLE_VPC_PUBLIC_ENDPOINT value: "true" volumeMounts: - mountPath: /etc/cloud @@ -516,3 +511,6 @@ data: - name: ibm-secret secret: secretName: ibmpowervs-cloud-credential +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon diff --git a/templates/cluster-template-powervs-clusterclass/cluster-with-kcp.yaml b/templates/cluster-template-powervs-clusterclass/cluster-with-kcp.yaml new file mode 100644 index 000000000..6e87cad87 --- /dev/null +++ b/templates/cluster-template-powervs-clusterclass/cluster-with-kcp.yaml @@ -0,0 +1,187 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + topology: + class: "${IBMPOWERVS_CLUSTER_CLASS_NAME}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 2 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${IBMPOWERVS_CLUSTER_CLASS_NAME:="powervs-cc"} +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate" + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane" + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSClusterTemplate + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template" + workers: + machineDeployments: + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0" + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSClusterTemplate +metadata: + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-cluster-template" +spec: + template: + spec: + controlPlaneEndpoint: + host: "${IBMPOWERVS_VIP_EXTERNAL}" + port: ${API_SERVER_PORT:=6443} + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane" +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - "${IBMPOWERVS_VIP}" + - "${IBMPOWERVS_VIP_EXTERNAL}" + extraArgs: + cloud-provider: external + controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + controllerManager: + extraArgs: + cloud-provider: external + enable-hostpath-provisioner: "true" + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "${API_SERVER_PORT:=6443}" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0744" + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: 192.168.167.85:${API_SERVER_PORT:=6443} + caCertHashes: [] + token: "" + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + useExperimentalRetryJoin: true diff --git a/templates/cluster-template-powervs-clusterclass/kustomization.yaml b/templates/cluster-template-powervs-clusterclass/kustomization.yaml new file mode 100644 index 000000000..4a8bb9dfc --- /dev/null +++ b/templates/cluster-template-powervs-clusterclass/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- cluster-with-kcp.yaml +- md.yaml +- ../addons/crs-powervs.yaml + +sortOptions: + order: fifo diff --git a/templates/cluster-template-powervs-clusterclass/md.yaml b/templates/cluster-template-powervs-clusterclass/md.yaml new file mode 100644 index 000000000..25b4dfbb2 --- /dev/null +++ b/templates/cluster-template-powervs-clusterclass/md.yaml @@ -0,0 +1,65 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/control-plane: "" + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + discovery: + bootstrapToken: + apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + caCertHashes: [] + token: "" + unsafeSkipCAVerification: false + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-control-plane-machinetemplate" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: "${IBMPOWERVS_CLUSTER_CLASS_NAME}-worker-machinetemplate" +spec: + template: + spec: + serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" + sshKey: "${IBMPOWERVS_SSHKEY_NAME}" + image: + name: "${IBMPOWERVS_IMAGE_NAME}" + network: + name: "${IBMPOWERVS_NETWORK_NAME}" + memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} diff --git a/templates/cluster-template-powervs.yaml b/templates/cluster-template-powervs.yaml index 16ba5d63a..0e1949c04 100644 --- a/templates/cluster-template-powervs.yaml +++ b/templates/cluster-template-powervs.yaml @@ -2,221 +2,219 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: clusterNetwork: pods: cidrBlocks: - - ${POD_CIDR:="192.168.0.0/16"} + - ${POD_CIDR:="192.168.0.0/16"} serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} services: cidrBlocks: - - ${SERVICE_CIDR:="10.128.0.0/12"} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: IBMPowerVSCluster - name: "${CLUSTER_NAME}" + - ${SERVICE_CIDR:="10.128.0.0/12"} controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSCluster + name: ${CLUSTER_NAME} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSCluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" controlPlaneEndpoint: - host: "${IBMPOWERVS_VIP_EXTERNAL}" + host: ${IBMPOWERVS_VIP_EXTERNAL} port: ${API_SERVER_PORT:=6443} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} --- -kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: - version: "${KUBERNETES_VERSION}" - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: IBMPowerVSMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: "${CLUSTER_NAME}-control-plane" kubeadmConfigSpec: clusterConfiguration: apiServer: certSANs: - - "${IBMPOWERVS_VIP}" - - "${IBMPOWERVS_VIP_EXTERNAL}" + - ${IBMPOWERVS_VIP} + - ${IBMPOWERVS_VIP_EXTERNAL} controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} controllerManager: extraArgs: enable-hostpath-provisioner: "true" + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "${API_SERVER_PORT:=6443}" + - name: vip_interface + value: env2 + - name: vip_cidr + value: "${IBMPOWERVS_VIP_CIDR}" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "${IBMPOWERVS_VIP}" + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0744" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: cloud-provider: external - provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' name: '{{ v1.local_hostname }}' joinConfiguration: discovery: bootstrapToken: apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} + caCertHashes: [] token: "" - caCertHashes: [ ] unsafeSkipCAVerification: false nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: cloud-provider: external - provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' name: '{{ v1.local_hostname }}' - useExperimentalRetryJoin: true preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname - files: - - path: /etc/kubernetes/manifests/kube-vip.yaml - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "${API_SERVER_PORT:=6443}" - - name: vip_interface - value: env2 - - name: vip_cidr - value: "${IBMPOWERVS_VIP_CIDR}" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "${IBMPOWERVS_VIP}" - image: ghcr.io/kube-vip/kube-vip:v0.4.4 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig - status: {} - owner: "root:root" - permissions: "0744" + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + useExperimentalRetryJoin: true + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: - clusterName: "${CLUSTER_NAME}" + clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: template: spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" bootstrap: configRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} infrastructureRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMPowerVSMachineTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: - serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}" - sshKey: "${IBMPOWERVS_SSHKEY_NAME}" image: - name: "${IBMPOWERVS_IMAGE_NAME}" - network: - name: "${IBMPOWERVS_NETWORK_NAME}" + name: ${IBMPOWERVS_IMAGE_NAME} memoryGiB: ${IBMPOWERVS_COMPUTE_MEMORY:=4} + network: + name: ${IBMPOWERVS_NETWORK_NAME} + processorType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="Shared"} processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"} + serviceInstanceID: ${IBMPOWERVS_SERVICE_INSTANCE_ID} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} systemType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"} - processorType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="Shared"} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: @@ -224,19 +222,19 @@ spec: discovery: bootstrapToken: apiServerEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443} - token: "" caCertHashes: [] + token: "" unsafeSkipCAVerification: false nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: cloud-provider: external - provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' name: '{{ v1.local_hostname }}' preKubeadmCommands: - - hostname "{{ v1.local_hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts - - echo "{{ v1.local_hostname }}" >/etc/hostname + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname diff --git a/templates/cluster-template-powervs/kcp.yaml b/templates/cluster-template-powervs/kcp.yaml new file mode 100644 index 000000000..d5200d353 --- /dev/null +++ b/templates/cluster-template-powervs/kcp.yaml @@ -0,0 +1,16 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' + name: '{{ v1.local_hostname }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' + diff --git a/templates/cluster-template-powervs/kubeadm-config.yaml b/templates/cluster-template-powervs/kubeadm-config.yaml new file mode 100644 index 000000000..d690de6c2 --- /dev/null +++ b/templates/cluster-template-powervs/kubeadm-config.yaml @@ -0,0 +1,11 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}' diff --git a/templates/cluster-template-powervs/kustomization.yaml b/templates/cluster-template-powervs/kustomization.yaml new file mode 100644 index 000000000..158a96dac --- /dev/null +++ b/templates/cluster-template-powervs/kustomization.yaml @@ -0,0 +1,9 @@ +resources: +- ../bases/powervs + +patches: +- path: kcp.yaml +- path: kubeadm-config.yaml + +sortOptions: + order: fifo diff --git a/templates/cluster-template-vpc-load-balancer.yaml b/templates/cluster-template-vpc-load-balancer.yaml index a57584f20..07c0af7c3 100644 --- a/templates/cluster-template-vpc-load-balancer.yaml +++ b/templates/cluster-template-vpc-load-balancer.yaml @@ -3,9 +3,9 @@ kind: Cluster metadata: labels: ccm: external - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" - namespace: "${NAMESPACE}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} spec: clusterNetwork: pods: @@ -15,58 +15,52 @@ spec: services: cidrBlocks: - ${SERVICE_CIDR:="10.128.0.0/12"} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: IBMVPCCluster - name: "${CLUSTER_NAME}" - namespace: "${NAMESPACE}" controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCCluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCCluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: - region: "${IBMVPC_REGION}" - zone: "${IBMVPC_ZONE}" - resourceGroup: "${IBMVPC_RESOURCEGROUP}" - vpc: "${IBMVPC_NAME}" controlPlaneLoadBalancer: - name: "${CLUSTER_NAME}-load-balancer" + name: ${CLUSTER_NAME}-load-balancer + region: ${IBMVPC_REGION} + resourceGroup: ${IBMVPC_RESOURCEGROUP} + vpc: ${IBMVPC_NAME} + zone: ${IBMVPC_ZONE} --- -kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane metadata: - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} spec: - version: "${KUBERNETES_VERSION}" - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: IBMVPCMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" kubeadmConfigSpec: clusterConfiguration: - kubernetesVersion: ${KUBERNETES_VERSION} - controllerManager: + apiServer: + certSANs: + - localhost + - 127.0.0.1 extraArgs: - enable-hostpath-provisioner: "true" cloud-provider: external - apiServer: - certSANs: [localhost, 127.0.0.1] + controllerManager: extraArgs: cloud-provider: external + enable-hostpath-provisioner: "true" dns: {} etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} networking: {} scheduler: {} initConfiguration: @@ -74,74 +68,80 @@ spec: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: cloud-provider: external - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% joinConfiguration: discovery: {} nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: cloud-provider: external - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} --- -kind: IBMVPCMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCMachineTemplate metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: template: spec: - image: - name: "${IBMVPC_IMAGE_NAME}" - zone: "${IBMVPC_ZONE}" - profile: "${IBMVPC_PROFILE}" - sshKeys: - - name: "${IBMVPC_SSHKEY_NAME}" bootVolume: sizeGiB: ${IBMVPC_CONTROLPLANE_BOOT_VOLUME_SIZEGIB:=20} + image: + name: ${IBMVPC_IMAGE_NAME} + profile: ${IBMVPC_PROFILE} + sshKeys: + - name: ${IBMVPC_SSHKEY_NAME} + zone: ${IBMVPC_ZONE} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: - clusterName: "${CLUSTER_NAME}" + clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: template: spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" bootstrap: configRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} infrastructureRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCMachineTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: - image: - name: "${IBMVPC_IMAGE_NAME}" - zone: "${IBMVPC_ZONE}" - profile: "${IBMVPC_PROFILE}" - sshKeys: - - name: "${IBMVPC_SSHKEY_NAME}" bootVolume: sizeGiB: ${IBMVPC_WORKER_BOOT_VOLUME_SIZEGIB:=20} + image: + name: ${IBMVPC_IMAGE_NAME} + profile: ${IBMVPC_PROFILE} + sshKeys: + - name: ${IBMVPC_SSHKEY_NAME} + zone: ${IBMVPC_ZONE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: @@ -160,18 +160,15 @@ spec: matchLabels: ccm: external resources: - - kind: Secret - name: ibm-credential - - kind: ConfigMap - name: ibm-cfg - - kind: ConfigMap - name: cloud-controller-manager-addon + - kind: Secret + name: ibm-credential + - kind: ConfigMap + name: ibm-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon strategy: ApplyOnce --- apiVersion: v1 -kind: ConfigMap -metadata: - name: ibm-cfg data: ibm-cloud-conf.yaml: |- apiVersion: v1 @@ -195,12 +192,14 @@ data: g2VpcSubnetNames = "${CLUSTER_NAME}-subnet" g2VpcName = ${IBMVPC_NAME:=""} region = ${IBMVPC_REGION:=""} +kind: ConfigMap +metadata: + name: ibm-cfg --- apiVersion: v1 kind: Secret metadata: name: ibm-credential -type: addons.cluster.x-k8s.io/resource-set stringData: ibm-credential.yaml: |- apiVersion: v1 @@ -210,11 +209,9 @@ stringData: namespace: kube-system data: ibmcloud_api_key: ${BASE64_API_KEY} +type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 -kind: ConfigMap -metadata: - name: cloud-controller-manager-addon data: ibm-ccm-external.yaml: |- apiVersion: v1 @@ -406,4 +403,7 @@ data: name: ibm-cloud-config - name: ibm-secret secret: - secretName: ibm-cloud-credential + secretName: ibm-cloud-credentia +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon diff --git a/templates/cluster-template-vpc-load-balancer/cluster.yaml b/templates/cluster-template-vpc-load-balancer/cluster.yaml new file mode 100644 index 000000000..a89e4d0c2 --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/cluster.yaml @@ -0,0 +1,8 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: external + diff --git a/templates/cluster-template-vpc-load-balancer/kcp.yaml b/templates/cluster-template-vpc-load-balancer/kcp.yaml new file mode 100644 index 000000000..acc2b3141 --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/kcp.yaml @@ -0,0 +1,15 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + cloud-provider: external + apiServer: + extraArgs: + cloud-provider: external + diff --git a/templates/cluster-template-vpc-load-balancer/kustomization.yaml b/templates/cluster-template-vpc-load-balancer/kustomization.yaml new file mode 100644 index 000000000..65049ca19 --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/kustomization.yaml @@ -0,0 +1,13 @@ +resources: +- ../bases/vpc +- ../addons/crs.yaml + +patches: +- path: cluster.yaml +- path: vpc-cluster.yaml +- path: kcp.yaml +- path: vpc-cp-machine-template.yaml +- path: md.yaml + +sortOptions: + order: fifo diff --git a/templates/cluster-template-vpc-load-balancer/md.yaml b/templates/cluster-template-vpc-load-balancer/md.yaml new file mode 100644 index 000000000..b85ff009b --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/md.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + bootVolume: + sizeGiB: ${IBMVPC_WORKER_BOOT_VOLUME_SIZEGIB:=20} diff --git a/templates/cluster-template-vpc-load-balancer/vpc-cluster.yaml b/templates/cluster-template-vpc-load-balancer/vpc-cluster.yaml new file mode 100644 index 000000000..58450edae --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/vpc-cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneLoadBalancer: + name: "${CLUSTER_NAME}-load-balancer" diff --git a/templates/cluster-template-vpc-load-balancer/vpc-cp-machine-template.yaml b/templates/cluster-template-vpc-load-balancer/vpc-cp-machine-template.yaml new file mode 100644 index 000000000..b52ae588d --- /dev/null +++ b/templates/cluster-template-vpc-load-balancer/vpc-cp-machine-template.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + bootVolume: + sizeGiB: ${IBMVPC_CONTROLPLANE_BOOT_VOLUME_SIZEGIB:=20} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 66ad48e2d..9a0f52e42 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -2,9 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" - namespace: "${NAMESPACE}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} spec: clusterNetwork: pods: @@ -14,125 +14,126 @@ spec: services: cidrBlocks: - ${SERVICE_CIDR:="10.128.0.0/12"} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: IBMVPCCluster - name: "${CLUSTER_NAME}" - namespace: "${NAMESPACE}" controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCCluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCCluster metadata: labels: - cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - name: "${CLUSTER_NAME}" + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} spec: - region: "${IBMVPC_REGION}" - zone: "${IBMVPC_ZONE}" - resourceGroup: "${IBMVPC_RESOURCEGROUP}" - vpc: "${IBMVPC_NAME}" + region: ${IBMVPC_REGION} + resourceGroup: ${IBMVPC_RESOURCEGROUP} + vpc: ${IBMVPC_NAME} + zone: ${IBMVPC_ZONE} --- -kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane metadata: - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} spec: - version: "${KUBERNETES_VERSION}" - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: IBMVPCMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: "${CLUSTER_NAME}-control-plane" - namespace: "${NAMESPACE}" kubeadmConfigSpec: clusterConfiguration: - kubernetesVersion: ${KUBERNETES_VERSION} - controllerManager: - extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: + - localhost + - 127.0.0.1 + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" dns: {} etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} networking: {} scheduler: {} initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: + kubeletExtraArgs: cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: discovery: {} nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: + kubeletExtraArgs: cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMVPCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} --- -kind: IBMVPCMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMVPCMachineTemplate metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-control-plane spec: template: spec: image: - name: "${IBMVPC_IMAGE_NAME}" - zone: "${IBMVPC_ZONE}" - profile: "${IBMVPC_PROFILE}" + name: ${IBMVPC_IMAGE_NAME} + profile: ${IBMVPC_PROFILE} sshKeys: - - name: "${IBMVPC_SSHKEY_NAME}" + - name: ${IBMVPC_SSHKEY_NAME} + zone: ${IBMVPC_ZONE} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: - clusterName: "${CLUSTER_NAME}" + clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: template: spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" bootstrap: configRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} infrastructureRef: - name: "${CLUSTER_NAME}-md-0" apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: IBMVPCMachineTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: image: - name: "${IBMVPC_IMAGE_NAME}" - zone: "${IBMVPC_ZONE}" - profile: "${IBMVPC_PROFILE}" + name: ${IBMVPC_IMAGE_NAME} + profile: ${IBMVPC_PROFILE} sshKeys: - - name: "${IBMVPC_SSHKEY_NAME}" + - name: ${IBMVPC_SSHKEY_NAME} + zone: ${IBMVPC_ZONE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_NAME}-md-0" + name: ${CLUSTER_NAME}-md-0 spec: template: spec: @@ -140,5 +141,5 @@ spec: nodeRegistration: kubeletExtraArgs: cloud-provider: external - provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' diff --git a/templates/cluster-template/kcp.yaml b/templates/cluster-template/kcp.yaml new file mode 100644 index 000000000..b46dfa9ed --- /dev/null +++ b/templates/cluster-template/kcp.yaml @@ -0,0 +1,15 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' diff --git a/templates/cluster-template/kubeadm-config.yaml b/templates/cluster-template/kubeadm-config.yaml new file mode 100644 index 000000000..d7cd02bc9 --- /dev/null +++ b/templates/cluster-template/kubeadm-config.yaml @@ -0,0 +1,12 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: ibmvpc://${CLUSTER_NAME}/'{{ v1.local_hostname }}' + diff --git a/templates/cluster-template/kustomization.yaml b/templates/cluster-template/kustomization.yaml new file mode 100644 index 000000000..15f9c341e --- /dev/null +++ b/templates/cluster-template/kustomization.yaml @@ -0,0 +1,9 @@ +resources: +- ../bases/vpc + +patches: +- path: kcp.yaml +- path: kubeadm-config.yaml + +sortOptions: + order: fifo