From 13d00b4de33ba296ebddfbc8287d9580d2b71e96 Mon Sep 17 00:00:00 2001 From: willie-yao Date: Wed, 18 Oct 2023 21:48:17 +0000 Subject: [PATCH] Add templates for clusterclass machinepool --- .../cluster-template-aks-clusterclass.yaml | 155 ++++++++++ templates/cluster-template-aks-topology.yaml | 21 ++ templates/cluster-template-clusterclass.yaml | 24 -- templates/cluster-template-topology.yaml | 23 ++ .../azure-managed-cluster-template.yaml | 9 + .../azure-managed-controlplane-template.yaml | 15 + .../azure-managed-machinepool-template.yaml | 23 ++ .../aks-clusterclass/clusterclass.yaml | 43 +++ .../kubeadm-config-template.yaml | 47 +++ .../aks-clusterclass/kustomization.yaml | 10 + .../managedazurecluster-identity-ref.yaml | 9 + templates/flavors/aks-topology/cluster.yaml | 21 ++ .../flavors/aks-topology/kustomization.yaml | 5 + .../flavors/clusterclass/kustomization.yaml | 1 - .../{clusterclass => topology}/cluster.yaml | 0 templates/flavors/topology/kustomization.yaml | 3 + ...luster-template-prow-aks-clusterclass.yaml | 273 ++++++++++++++++++ .../prow-aks-clusterclass/kustomization.yaml | 13 + .../ci/prow-aks-clusterclass/patches.yaml | 54 ++++ .../prow-aks-clusterclass/patches/addons.yaml | 11 + .../patches/aks-clusterclass-pool0.yaml | 15 + .../patches/aks-clusterclass-pool1.yaml | 67 +++++ .../patches/cluster.yaml | 7 + .../patches/tags-aks-clusterclass.yaml | 12 + .../test/ci/prow-topology/kustomization.yaml | 2 +- 25 files changed, 837 insertions(+), 26 deletions(-) create mode 100644 templates/cluster-template-aks-clusterclass.yaml create mode 100644 templates/cluster-template-aks-topology.yaml create mode 100644 templates/cluster-template-topology.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml create mode 100644 templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml create mode 100644 templates/flavors/aks-clusterclass/clusterclass.yaml create mode 100644 templates/flavors/aks-clusterclass/kubeadm-config-template.yaml create mode 100644 templates/flavors/aks-clusterclass/kustomization.yaml create mode 100644 templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml create mode 100644 templates/flavors/aks-topology/cluster.yaml create mode 100644 templates/flavors/aks-topology/kustomization.yaml rename templates/flavors/{clusterclass => topology}/cluster.yaml (100%) create mode 100644 templates/flavors/topology/kustomization.yaml create mode 100644 templates/test/ci/cluster-template-prow-aks-clusterclass.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/kustomization.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/addons.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml create mode 100644 templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml diff --git a/templates/cluster-template-aks-clusterclass.yaml b/templates/cluster-template-aks-clusterclass.yaml new file mode 100644 index 00000000000..7c91181f416 --- /dev/null +++ b/templates/cluster-template-aks-clusterclass.yaml @@ -0,0 +1,155 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + mode: User + name: pool1 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] diff --git a/templates/cluster-template-aks-topology.yaml b/templates/cluster-template-aks-topology.yaml new file mode 100644 index 00000000000..c78efb4b389 --- /dev/null +++ b/templates/cluster-template-aks-topology.yaml @@ -0,0 +1,21 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml index 7c9c39da4a2..c68df586f34 100644 --- a/templates/cluster-template-clusterclass.yaml +++ b/templates/cluster-template-clusterclass.yaml @@ -79,30 +79,6 @@ spec: kind: AzureMachineTemplate name: ${CLUSTER_NAME}-md-0 --- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - containerd-logger: enabled - csi-proxy: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - topology: - class: ${CLUSTER_CLASS_NAME} - controlPlane: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} - workers: - machineDeployments: - - class: ${CLUSTER_NAME}-worker - name: md-0 - replicas: ${WORKER_MACHINE_COUNT} ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterTemplate metadata: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml new file mode 100644 index 00000000000..28ad70f5a82 --- /dev/null +++ b/templates/cluster-template-topology.yaml @@ -0,0 +1,23 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + containerd-logger: enabled + csi-proxy: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: ${CLUSTER_NAME}-worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} diff --git a/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml new file mode 100644 index 00000000000..6ad8daa397e --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-cluster-template.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} + \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml new file mode 100644 index 00000000000..12e31a5c904 --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-controlplane-template.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} diff --git a/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml b/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml new file mode 100644 index 00000000000..0c637c91507 --- /dev/null +++ b/templates/flavors/aks-clusterclass/azure-managed-machinepool-template.yaml @@ -0,0 +1,23 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + mode: User + name: pool1 + sku: ${AZURE_NODE_MACHINE_TYPE} \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/clusterclass.yaml b/templates/flavors/aks-clusterclass/clusterclass.yaml new file mode 100644 index 00000000000..f20813ac9c3 --- /dev/null +++ b/templates/flavors/aks-clusterclass/clusterclass.yaml @@ -0,0 +1,43 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 + \ No newline at end of file diff --git a/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml b/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml new file mode 100644 index 00000000000..8b36253b628 --- /dev/null +++ b/templates/flavors/aks-clusterclass/kubeadm-config-template.yaml @@ -0,0 +1,47 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] diff --git a/templates/flavors/aks-clusterclass/kustomization.yaml b/templates/flavors/aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..98083341a84 --- /dev/null +++ b/templates/flavors/aks-clusterclass/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- clusterclass.yaml +- azure-managed-controlplane-template.yaml +- azure-managed-cluster-template.yaml +- azure-managed-machinepool-template.yaml +- ../../azure-cluster-identity +- kubeadm-config-template.yaml diff --git a/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml new file mode 100644 index 00000000000..37d7ac29191 --- /dev/null +++ b/templates/flavors/aks-clusterclass/patches/managedazurecluster-identity-ref.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: "${CLUSTER_IDENTITY_NAME}" \ No newline at end of file diff --git a/templates/flavors/aks-topology/cluster.yaml b/templates/flavors/aks-topology/cluster.yaml new file mode 100644 index 00000000000..c78efb4b389 --- /dev/null +++ b/templates/flavors/aks-topology/cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/flavors/aks-topology/kustomization.yaml b/templates/flavors/aks-topology/kustomization.yaml new file mode 100644 index 00000000000..7a5648beb9c --- /dev/null +++ b/templates/flavors/aks-topology/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- cluster.yaml diff --git a/templates/flavors/clusterclass/kustomization.yaml b/templates/flavors/clusterclass/kustomization.yaml index d32392fc71f..30b4276d555 100644 --- a/templates/flavors/clusterclass/kustomization.yaml +++ b/templates/flavors/clusterclass/kustomization.yaml @@ -1,7 +1,6 @@ namespace: default resources: - clusterclass.yaml - - cluster.yaml - azure-cluster-template.yaml - azure-machine-template-controlplane.yaml - azure-machine-template-worker.yaml diff --git a/templates/flavors/clusterclass/cluster.yaml b/templates/flavors/topology/cluster.yaml similarity index 100% rename from templates/flavors/clusterclass/cluster.yaml rename to templates/flavors/topology/cluster.yaml diff --git a/templates/flavors/topology/kustomization.yaml b/templates/flavors/topology/kustomization.yaml new file mode 100644 index 00000000000..fd7811e1145 --- /dev/null +++ b/templates/flavors/topology/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: default +resources: +- cluster.yaml \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml new file mode 100644 index 00000000000..34da32fe61a --- /dev/null +++ b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml @@ -0,0 +1,273 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + patches: + - definitions: + - jsonPatches: + - op: replace + path: /spec/template/spec/files + valueFrom: + template: | + - contentFrom: + secret: + key: worker-node-azure.json + name: "{{ .builtin.machinePool.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machinePoolClass: + names: + - default-system + - default-worker + name: workerAzureJsonSecretName + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + addonProfiles: + - enabled: true + name: azurepolicy + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + availabilityZones: + - "1" + - "2" + enableNodePublicIP: false + enableUltraSSD: true + maxPods: 30 + mode: System + name: pool0 + osDiskSizeGB: 30 + osDiskType: Managed + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + enableNodePublicIP: false + kubeletConfig: + allowedUnsafeSysctls: + - net.* + - kernel.msg* + containerLogMaxFiles: 50 + containerLogMaxSizeMB: 500 + cpuCfsQuota: true + cpuCfsQuotaPeriod: 110ms + cpuManagerPolicy: static + failSwapOn: false + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + podMaxPids: 2048 + topologyManagerPolicy: best-effort + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: 32768 60999 + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: madvise + transparentHugePageEnabled: always + maxPods: 64 + mode: User + name: pool1 + nodeLabels: + type: shared + osDiskSizeGB: 40 + osDiskType: Ephemeral + scaleSetPriority: Regular + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} + taints: + - effect: NoSchedule + key: type + value: shared +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: default + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/test/ci/prow-aks-clusterclass/kustomization.yaml b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..f72a0910979 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../../flavors/aks-clusterclass + - ../../../flavors/aks-topology +patchesStrategicMerge: + - patches/tags-aks-clusterclass.yaml + - patches/aks-clusterclass-pool0.yaml + - patches/aks-clusterclass-pool1.yaml + - patches/cluster.yaml + - patches/addons.yaml + - patches.yaml diff --git a/templates/test/ci/prow-aks-clusterclass/patches.yaml b/templates/test/ci/prow-aks-clusterclass/patches.yaml new file mode 100644 index 00000000000..74d7e896aa7 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches.yaml @@ -0,0 +1,54 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} +spec: + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 + patches: + - name: workerAzureJsonSecretName + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machinePoolClass: + names: + - default-system + - default-worker + jsonPatches: + - op: replace + path: "/spec/template/spec/files" + valueFrom: + template: | + - contentFrom: + secret: + key: worker-node-azure.json + name: "{{ .builtin.machinePool.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml new file mode 100644 index 00000000000..51d2e93ca8a --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + addonProfiles: + - enabled: true + name: azurepolicy diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml new file mode 100644 index 00000000000..0c7b26b42bf --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool0" +spec: + template: + spec: + maxPods: 30 + osDiskType: "Managed" + osDiskSizeGB: 30 + enableNodePublicIP: false + enableUltraSSD: true + availabilityZones: ["1", "2"] + name: pool0 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml new file mode 100644 index 00000000000..a7e69ba69a2 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml @@ -0,0 +1,67 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool1" +spec: + template: + spec: + maxPods: 64 + osDiskType: "Ephemeral" + osDiskSizeGB: 40 + enableNodePublicIP: false + scaleSetPriority: Regular + taints: + - effect: NoSchedule + key: type + value: shared + nodeLabels: + "type": "shared" + name: pool1 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" + kubeletConfig: + cpuManagerPolicy: "static" + cpuCfsQuota: true + cpuCfsQuotaPeriod: "110ms" + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + topologyManagerPolicy: "best-effort" + allowedUnsafeSysctls: + - "net.*" + - "kernel.msg*" + failSwapOn: false + containerLogMaxSizeMB: 500 + containerLogMaxFiles: 50 + podMaxPids: 2048 + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: "32768 60999" + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: "madvise" + transparentHugePageEnabled: "always" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml b/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml new file mode 100644 index 00000000000..a1a4b8f8e39 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + topology: + class: default diff --git a/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml new file mode 100644 index 00000000000..9a6f76edb6f --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + jobName: ${JOB_NAME} + creationTimestamp: ${TIMESTAMP} + buildProvenance: ${BUILD_PROVENANCE} \ No newline at end of file diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index 60a8aa47928..e4da9c2e30e 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../../flavors/clusterclass/cluster.yaml + - ../../../flavors/topology/cluster.yaml - cni-resource-set.yaml - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml