diff --git a/Makefile b/Makefile index 73eb90fb842..8b6101db638 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ PULL_POLICY ?= Always # Allow overriding the e2e configurations GINKGO_FOCUS ?= Workload cluster creation +GINKGO_SKIP ?= Creating a GPU-enabled cluster GINKGO_NODES ?= 3 GINKGO_NOCOLOR ?= false ARTIFACTS ?= $(ROOT_DIR)/_artifacts @@ -162,7 +163,7 @@ test-e2e: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests PULL_POLICY=IfNotPresent $(MAKE) docker-build docker-push MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ - $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ + $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) @@ -172,7 +173,7 @@ test-e2e-local: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests PULL_POLICY=IfNotPresent $(MAKE) docker-build MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ - $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ + $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) @@ -182,7 +183,7 @@ test-conformance: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests PULL_POLICY=IfNotPresent $(MAKE) docker-build docker-push MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ - $(GINKGO) -v -trace -stream -progress -v -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ + $(GINKGO) -v -trace -stream -progress -v -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \ diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index b45837c1b27..9f240773be0 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -9,6 +9,7 @@ - [Ephemeral OS](./topics/ephemeral-os.md) - [External Cloud Provider](./topics/external-cloud-provider.md) - [Failure Domains](./topics/failure-domains.md) + - [GPU-enabled Clusters](../topics/gpu.md) - [Identity](./topics/identity.md) - [IPv6](./topics/ipv6.md) - [Machine Pools (VMSS)](./topics/machinepools.md) diff --git a/docs/book/src/topics/gpu.md b/docs/book/src/topics/gpu.md new file mode 100644 index 00000000000..a10ec27308f --- /dev/null +++ b/docs/book/src/topics/gpu.md @@ -0,0 +1,129 @@ +# GPU-enabled clusters + +## Overview + +With CAPZ you can create GPU-enabled Kubernetes clusters on Microsoft Azure. + +Before you begin, be aware that: + +- [Scheduling GPUs](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/) is a Kubernetes beta feature +- [NVIDIA GPUs](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-gpu) are supported on Azure NC-series, NV-series, and NVv3-series VMs + +To deploy a cluster with support for GPU nodes, use the [nvidia-gpu flavor](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/master/templates/cluster-template-nvidia-gpu.yaml). + +## An example GPU cluster + +Let's create a CAPZ cluster with an N-series node and run a GPU-powered vector calculation. + +### Generate an nvidia-gpu cluster template + +Use the `clusterctl config cluster` command to generate a manifest that defines your GPU-enabled +workload cluster. + +Remember to use the `nvidia-gpu` flavor with N-series nodes. + +```bash +AZURE_CONTROL_PLANE_MACHINE_TYPE=Standard_D2s_v3 \ +AZURE_NODE_MACHINE_TYPE=Standard_NC6s_v3 \ +AZURE_LOCATION=southcentralus \ +clusterctl config cluster azure-gpu \ + --kubernetes-version=v1.19.3 \ + --worker-machine-count=1 \ + --flavor=nvidia-gpu > azure-gpu-cluster.yaml +``` + +### Create the cluster + +Apply the manifest from the previous step to your management cluster to have CAPZ create a +workload cluster: + +```bash +$ kubectl apply -f azure-gpu-cluster.yaml +cluster.cluster.x-k8s.io/azure-gpu created +azurecluster.infrastructure.cluster.x-k8s.io/azure-gpu created +kubeadmcontrolplane.controlplane.cluster.x-k8s.io/azure-gpu-control-plane created +azuremachinetemplate.infrastructure.cluster.x-k8s.io/azure-gpu-control-plane created +machinedeployment.cluster.x-k8s.io/azure-gpu-md-0 created +azuremachinetemplate.infrastructure.cluster.x-k8s.io/azure-gpu-md-0 created +kubeadmconfigtemplate.bootstrap.cluster.x-k8s.io/azure-gpu-md-0 created +``` + +Wait until the cluster and nodes are finished provisioning. The GPU nodes make take several minutes +to provision, since each one must install drivers and supporting software. + +```bash +$ kubectl get cluster azure-gpu +NAME PHASE +azure-gpu Provisioned +$ kubectl get machines +NAME PROVIDERID PHASE VERSION +azure-gpu-control-plane-t94nm azure:////subscriptions//resourceGroups/azure-gpu/providers/Microsoft.Compute/virtualMachines/azure-gpu-control-plane-nnb57 Running v1.19.2 +azure-gpu-md-0-f6b88dd78-vmkph azure:////subscriptions//resourceGroups/azure-gpu/providers/Microsoft.Compute/virtualMachines/azure-gpu-md-0-gcc8v Running v1.19.2 +``` + +You can run these commands against the workload cluster to verify that the +[NVIDIA device plugin](https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/master/nvidia-device-plugin.yml) +has initialized and the `nvidia.com/gpu` resource is available: + +```bash +$ clusterctl get kubeconfig azure-gpu > azure-gpu-cluster.conf +$ export KUBECONFIG=azure-gpu-cluster.conf +$ kubectl -n kube-system get po | grep nvidia +kube-system nvidia-device-plugin-daemonset-d5dn6 1/1 Running 0 16m +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +azure-gpu-control-plane-nnb57 Ready master 42m v1.19.2 +azure-gpu-md-0-gcc8v Ready 38m v1.19.2 +$ kubectl get node azure-gpu-md-0-gcc8v -o jsonpath={.status.allocatable} | jq +{ + "attachable-volumes-azure-disk": "12", + "cpu": "6", + "ephemeral-storage": "119716326407", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "memory": "115312060Ki", + "nvidia.com/gpu": "1", + "pods": "110" +} +``` + +### Run a test app + +Let's create a pod manifest for the `cuda-vector-add` example from the Kubernetes documentation and +deploy it: + +```shell +$ cat > cuda-vector-add.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: cuda-vector-add +spec: + restartPolicy: OnFailure + containers: + - name: cuda-vector-add + # https://github.com/kubernetes/kubernetes/blob/v1.7.11/test/images/nvidia-cuda/Dockerfile + image: "k8s.gcr.io/cuda-vector-add:v0.1" + resources: + limits: + nvidia.com/gpu: 1 # requesting 1 GPU +EOF +$ kubectl apply -f cuda-vector-add.yaml +``` + +The container will download, run, and perform a [CUDA](https://developer.nvidia.com/cuda-zone) +calculation with the GPU. + +```bash +$ kubectl get po cuda-vector-add +cuda-vector-add 0/1 Completed 0 91s +$ kubectl logs cuda-vector-add +[Vector addition of 50000 elements] +Copy input data from the host memory to the CUDA device +CUDA kernel launch with 196 blocks of 256 threads +Copy output data from the CUDA device to the host memory +Test PASSED +Done +``` + +If you see output like the above, your GPU cluster is working! diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh index bdfe5ca308d..e14ae70e9c6 100755 --- a/scripts/ci-entrypoint.sh +++ b/scripts/ci-entrypoint.sh @@ -121,7 +121,7 @@ run_upstream_e2e_tests() { fi fi - # setting this env prevents ginkg e2e from trying to run provider setup + # setting this env prevents ginkgo e2e from trying to run provider setup export KUBERNETES_CONFORMANCE_TEST="y" # run the tests (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \ diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml new file mode 100644 index 00000000000..75c7799cbd6 --- /dev/null +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -0,0 +1,341 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/master/nvidia-device-plugin.yml + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + plugin_dir = "" + disabled_plugins = [] + required_plugins = [] + oom_score = 0 + + [grpc] + address = "/run/containerd/containerd.sock" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + + [ttrpc] + address = "" + uid = 0 + gid = 0 + + [debug] + address = "" + uid = 0 + gid = 0 + level = "" + + [metrics] + address = "" + grpc_histogram = false + + [cgroup] + path = "" + + [timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + + [plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "overlayfs" + default_runtime_name = "nvidia-container-runtime" + no_pivot = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v1" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v1" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "nvidia-container-runtime" + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + [plugins."io.containerd.runtime.v1.linux"] + shim = "containerd-shim" + runtime = "nvidia-container-runtime" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + [plugins."io.containerd.snapshotter.v1.devmapper"] + root_path = "" + pool_name = "" + base_image_size = "" + owner: root:root + path: /etc/containerd/nvidia-config.toml + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - cp /etc/containerd/config.toml /etc/containerd/config.toml.old + - mv /etc/containerd/nvidia-config.toml /etc/containerd/config.toml + - systemctl restart containerd + preKubeadmCommands: + - curl -sL https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key + add - + - curl -sL https://nvidia.github.io/nvidia-container-runtime/$(. /etc/os-release;echo + $ID$VERSION_ID)/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list + - apt update + - apt install ubuntu-drivers-common -y + - ubuntu-drivers install + - apt install nvidia-container-runtime -y + useExperimentalRetryJoin: true diff --git a/templates/flavors/README.md b/templates/flavors/README.md index 0bf52a347fc..ea3d3fad6e1 100644 --- a/templates/flavors/README.md +++ b/templates/flavors/README.md @@ -25,7 +25,19 @@ run ```tilt up ${flavors}``` to spin up worker clusters in Azure represented by Add your desired flavors to tilt_config.json: ```json { - "worker-flavors": ["default", "aks", "ephemeral", "external-cloud-provider", "ipv6", "machinepool", "system-assigned-identity", "user-assigned-identity", "machinepool-system-assigned-identity", "machinepool-user-assigned-identity"] + "worker-flavors": [ + "default", + "aks", + "ephemeral", + "external-cloud-provider", + "ipv6", + "machinepool", + "machinepool-system-assigned-identity", + "machinepool-user-assigned-identity", + "nvidia-gpu", + "system-assigned-identity", + "user-assigned-identity" + ] } ``` @@ -83,3 +95,31 @@ If you wish to override the default variables for flavor workers, you can specif } } ``` + +Here is a practical example: creating a GPU-enabled cluster requires N-series nodes. You can set an +N-series node type just for the `nvidia-gpu` flavor in `tilt-settings.json` to override any default: + +```json +{ + "kustomize_substitutions": { + "AZURE_SUBSCRIPTION_ID_B64": "****", + "AZURE_TENANT_ID_B64": "****", + "AZURE_CLIENT_SECRET_B64": "****", + "AZURE_CLIENT_ID_B64": "****", + "AZURE_ENVIRONMENT": "AzurePublicCloud" + }, + "worker-templates": { + "flavors": { + "nvidia-gpu": { + "AZURE_NODE_MACHINE_TYPE": "Standard_NC6s_v3" + } + }, + "metadata": { + "AZURE_CONTROL_PLANE_MACHINE_TYPE": "Standard_D2s_v3", + "AZURE_LOCATION": "southcentralus", + "KUBERNETES_VERSION": "v1.19.3", + "WORKER_MACHINE_COUNT": "1" + } + } +} +``` diff --git a/templates/flavors/nvidia-gpu/kustomization.yaml b/templates/flavors/nvidia-gpu/kustomization.yaml new file mode 100644 index 00000000000..0557ff60e5a --- /dev/null +++ b/templates/flavors/nvidia-gpu/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: default +resources: + - ../base + - machine-deployment.yaml +patchesStrategicMerge: + - patches/kubeadm-controlplane.yaml diff --git a/templates/flavors/nvidia-gpu/machine-deployment.yaml b/templates/flavors/nvidia-gpu/machine-deployment.yaml new file mode 100644 index 00000000000..5a7e2b698e2 --- /dev/null +++ b/templates/flavors/nvidia-gpu/machine-deployment.yaml @@ -0,0 +1,204 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + location: ${AZURE_LOCATION} + vmSize: ${AZURE_NODE_MACHINE_TYPE} + osDisk: + osType: "Linux" + diskSizeGB: 128 + managedDisk: + storageAccountType: "Premium_LRS" + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + useExperimentalRetryJoin: true + preKubeadmCommands: + # Enable NVIDIA container runtime package repository + - curl -sL https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add - + - curl -sL https://nvidia.github.io/nvidia-container-runtime/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list + - apt update + # Install NVIDIA drivers + - apt install ubuntu-drivers-common -y + - ubuntu-drivers install + # Install NVIDIA container runtime + - apt install nvidia-container-runtime -y + postKubeadmCommands: + # Configure containerd for NVIDIA container runtime + - cp /etc/containerd/config.toml /etc/containerd/config.toml.old + - mv /etc/containerd/nvidia-config.toml /etc/containerd/config.toml + - systemctl restart containerd + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data["local_hostname"] }}' + kubeletExtraArgs: + cloud-provider: azure + cloud-config: /etc/kubernetes/azure.json + files: + - path: /etc/kubernetes/azure.json + owner: root:root + permissions: "0644" + contentFrom: + secret: + name: ${CLUSTER_NAME}-md-0-azure-json + key: worker-node-azure.json + - path: /etc/containerd/nvidia-config.toml + owner: root:root + permissions: "0644" + content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + plugin_dir = "" + disabled_plugins = [] + required_plugins = [] + oom_score = 0 + + [grpc] + address = "/run/containerd/containerd.sock" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + + [ttrpc] + address = "" + uid = 0 + gid = 0 + + [debug] + address = "" + uid = 0 + gid = 0 + level = "" + + [metrics] + address = "" + grpc_histogram = false + + [cgroup] + path = "" + + [timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + + [plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "overlayfs" + default_runtime_name = "nvidia-container-runtime" + no_pivot = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v1" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v1" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "nvidia-container-runtime" + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + [plugins."io.containerd.runtime.v1.linux"] + shim = "containerd-shim" + runtime = "nvidia-container-runtime" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + [plugins."io.containerd.snapshotter.v1.devmapper"] + root_path = "" + pool_name = "" + base_image_size = "" diff --git a/templates/flavors/nvidia-gpu/patches/kubeadm-controlplane.yaml b/templates/flavors/nvidia-gpu/patches/kubeadm-controlplane.yaml new file mode 100644 index 00000000000..6b745326987 --- /dev/null +++ b/templates/flavors/nvidia-gpu/patches/kubeadm-controlplane.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + useExperimentalRetryJoin: true + postKubeadmCommands: + # Install the NVIDIA device plugin for Kubernetes + - KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/master/nvidia-device-plugin.yml diff --git a/templates/test/cluster-template-prow-nvidia-gpu.yaml b/templates/test/cluster-template-prow-nvidia-gpu.yaml new file mode 100644 index 00000000000..fa0031eb6d2 --- /dev/null +++ b/templates/test/cluster-template-prow-nvidia-gpu.yaml @@ -0,0 +1,365 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/master/nvidia-device-plugin.yml + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: ${AZURE_GPU_NODE_STORAGE_TYPE:=Standard_LRS} + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_GPU_NODE_MACHINE_TYPE:=Standard_NV6} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + plugin_dir = "" + disabled_plugins = [] + required_plugins = [] + oom_score = 0 + + [grpc] + address = "/run/containerd/containerd.sock" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + + [ttrpc] + address = "" + uid = 0 + gid = 0 + + [debug] + address = "" + uid = 0 + gid = 0 + level = "" + + [metrics] + address = "" + grpc_histogram = false + + [cgroup] + path = "" + + [timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + + [plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "overlayfs" + default_runtime_name = "nvidia-container-runtime" + no_pivot = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v1" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v1" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "nvidia-container-runtime" + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + [plugins."io.containerd.runtime.v1.linux"] + shim = "containerd-shim" + runtime = "nvidia-container-runtime" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + [plugins."io.containerd.snapshotter.v1.devmapper"] + root_path = "" + pool_name = "" + base_image_size = "" + owner: root:root + path: /etc/containerd/nvidia-config.toml + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - cp /etc/containerd/config.toml /etc/containerd/config.toml.old + - mv /etc/containerd/nvidia-config.toml /etc/containerd/config.toml + - systemctl restart containerd + preKubeadmCommands: + - curl -sL https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key + add - + - curl -sL https://nvidia.github.io/nvidia-container-runtime/$(. /etc/os-release;echo + $ID$VERSION_ID)/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list + - apt update + - apt install ubuntu-drivers-common -y + - ubuntu-drivers install + - apt install nvidia-container-runtime -y + useExperimentalRetryJoin: true +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-ipv6/cni-resource-set.yaml b/templates/test/prow-ipv6/cni-resource-set.yaml index 23b9571b7a7..de760b6e79c 100644 --- a/templates/test/prow-ipv6/cni-resource-set.yaml +++ b/templates/test/prow-ipv6/cni-resource-set.yaml @@ -18,4 +18,4 @@ spec: resources: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 - strategy: ApplyOnce \ No newline at end of file + strategy: ApplyOnce diff --git a/templates/test/prow-machine-pool/cni-resource-set.yaml b/templates/test/prow-machine-pool/cni-resource-set.yaml index 4dbde1c6f63..c50d9fc791a 100644 --- a/templates/test/prow-machine-pool/cni-resource-set.yaml +++ b/templates/test/prow-machine-pool/cni-resource-set.yaml @@ -18,4 +18,4 @@ spec: resources: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 - strategy: ApplyOnce \ No newline at end of file + strategy: ApplyOnce diff --git a/templates/test/prow-nvidia-gpu/cni-resource-set.yaml b/templates/test/prow-nvidia-gpu/cni-resource-set.yaml new file mode 100644 index 00000000000..c50d9fc791a --- /dev/null +++ b/templates/test/prow-nvidia-gpu/cni-resource-set.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-nvidia-gpu/kustomization.yaml b/templates/test/prow-nvidia-gpu/kustomization.yaml new file mode 100644 index 00000000000..b3d26dab5c3 --- /dev/null +++ b/templates/test/prow-nvidia-gpu/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../flavors/nvidia-gpu + - cni-resource-set.yaml +patchesStrategicMerge: + - ../patches/tags.yaml + - ../patches/cni-resource-set.yaml +patchesJson6902: +- path: patches/node-storage-type.yaml + target: + group: infrastructure.cluster.x-k8s.io + version: v1alpha3 + kind: AzureMachineTemplate + name: "${CLUSTER_NAME}-md-0" diff --git a/templates/test/prow-nvidia-gpu/patches/node-storage-type.yaml b/templates/test/prow-nvidia-gpu/patches/node-storage-type.yaml new file mode 100644 index 00000000000..58d92da868c --- /dev/null +++ b/templates/test/prow-nvidia-gpu/patches/node-storage-type.yaml @@ -0,0 +1,6 @@ +- op: replace + path: /spec/template/spec/osDisk/managedDisk/storageAccountType + value: "${AZURE_GPU_NODE_STORAGE_TYPE:=Standard_LRS}" +- op: replace + path: /spec/template/spec/vmSize + value: "${AZURE_GPU_NODE_MACHINE_TYPE:=Standard_NV6}" diff --git a/templates/test/prow/cni-resource-set.yaml b/templates/test/prow/cni-resource-set.yaml index 4dbde1c6f63..c50d9fc791a 100644 --- a/templates/test/prow/cni-resource-set.yaml +++ b/templates/test/prow/cni-resource-set.yaml @@ -18,4 +18,4 @@ spec: resources: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 - strategy: ApplyOnce \ No newline at end of file + strategy: ApplyOnce diff --git a/test/e2e/azure_accelnet.go b/test/e2e/azure_accelnet.go index 30a401fa815..3e43a62a785 100644 --- a/test/e2e/azure_accelnet.go +++ b/test/e2e/azure_accelnet.go @@ -60,13 +60,15 @@ func AzureAcceleratedNetworkingSpec(ctx context.Context, inputGetter func() Azur By("verifying EnableAcceleratedNetworking for the primary NIC of each VM") // NOTE: add SKUs being tested to this lookup table. acceleratedNetworking := map[compute.VirtualMachineSizeTypes]bool{ - compute.VirtualMachineSizeTypesStandardB2ms: false, - compute.VirtualMachineSizeTypesStandardD2V2: true, - compute.VirtualMachineSizeTypesStandardD2V3: false, - compute.VirtualMachineSizeTypesStandardD2sV3: false, - compute.VirtualMachineSizeTypesStandardD4V2: true, - compute.VirtualMachineSizeTypesStandardD4V3: true, - compute.VirtualMachineSizeTypesStandardD8sV3: true, + compute.VirtualMachineSizeTypesStandardB2ms: false, + compute.VirtualMachineSizeTypesStandardD2V2: true, + compute.VirtualMachineSizeTypesStandardD2V3: false, + compute.VirtualMachineSizeTypesStandardD2sV3: false, + compute.VirtualMachineSizeTypesStandardD4V2: true, + compute.VirtualMachineSizeTypesStandardD4V3: true, + compute.VirtualMachineSizeTypesStandardD8sV3: true, + compute.VirtualMachineSizeTypesStandardNC6sV3: false, + compute.VirtualMachineSizeTypesStandardNV6: false, } rgName := input.ClusterName page, err := vmsClient.List(ctx, rgName) diff --git a/test/e2e/azure_gpu.go b/test/e2e/azure_gpu.go new file mode 100644 index 00000000000..61edf101d63 --- /dev/null +++ b/test/e2e/azure_gpu.go @@ -0,0 +1,99 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/test/framework" +) + +// AzureGPUSpecInput is the input for AzureGPUSpec. +type AzureGPUSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string + SkipCleanup bool +} + +// AzureGPUSpec implements a test that verifies a GPU-enabled application runs on an +// "nvidia-gpu"-flavored CAPZ cluster. +func AzureGPUSpec(ctx context.Context, inputGetter func() AzureGPUSpecInput) { + var ( + specName = "azure-gpu" + input AzureGPUSpecInput + machineType = os.Getenv("AZURE_GPU_NODE_MACHINE_TYPE") + ) + + input = inputGetter() + Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + if machineType != "" { + Expect(machineType).To(HavePrefix("Standard_N"), "AZURE_GPU_NODE_MACHINE_TYPE is \"%s\" which isn't a GPU SKU in %s spec", machineType, specName) + } + + By("creating a Kubernetes client to the workload cluster") + clusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName) + Expect(clusterProxy).NotTo(BeNil()) + clientset := clusterProxy.GetClientSet() + Expect(clientset).NotTo(BeNil()) + + By("running a CUDA vector calculation job") + jobsClient := clientset.BatchV1().Jobs(corev1.NamespaceDefault) + jobName := "cuda-vector-add" + gpuJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: corev1.NamespaceDefault, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: jobName, + Image: "k8s.gcr.io/cuda-vector-add:v0.1", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + }, + } + _, err := jobsClient.Create(gpuJob) + Expect(err).NotTo(HaveOccurred()) + gpuJobInput := WaitForJobCompleteInput{ + Getter: jobsClientAdapter{client: jobsClient}, + Job: gpuJob, + Clientset: clientset, + } + WaitForJobComplete(ctx, gpuJobInput, e2eConfig.GetIntervals(specName, "wait-job")...) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index ded33bbac47..94c9fe287a2 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -234,4 +234,53 @@ var _ = Describe("Workload cluster creation", func() { }) }) }) + + // ci-e2e.sh and Prow CI skip this test by default, since N-series GPUs are relatively expensive + // and may require specific quota limits on the subscription. + // To include this test, set `GINKGO_SKIP=""`. + // You can override the default SKU `Standard_NV6` and `Standard_LRS` storage by setting + // the `AZURE_GPU_NODE_MACHINE_TYPE` and `AZURE_GPU_NODE_STORAGE_TYPE` environment variables. + // See https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/ for pricing. + Context("Creating a GPU-enabled cluster", func() { + It("with a single control plane node and 1 node", func() { + result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "nvidia-gpu", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachinePools: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + }) + cluster = result.Cluster + + Context("Running a GPU-based calculation", func() { + AzureGPUSpec(ctx, func() AzureGPUSpecInput { + return AzureGPUSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + Context("Validating accelerated networking", func() { + AzureAcceleratedNetworkingSpec(ctx, func() AzureAcceleratedNetworkingSpecInput { + return AzureAcceleratedNetworkingSpecInput{ + ClusterName: clusterName, + } + }) + }) + }) + }) }) diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 5a7cdd11857..3fbf64b019a 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -54,6 +54,8 @@ providers: targetName: "cluster-template-mhc.yaml" - sourcePath: "${PWD}/templates/test/cluster-template-prow-machine-pool.yaml" targetName: "cluster-template-machine-pool.yaml" + - sourcePath: "${PWD}/templates/test/cluster-template-prow-nvidia-gpu.yaml" + targetName: "cluster-template-nvidia-gpu.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.18.8}"