Merge pull request #3206 from tpdownes/a3_docker

Adopt local SSD storage for A3 docker images
GoogleCloudPlatform · Nov 1, 2024 · f7dabee · f7dabee
2 parents 38f5344 + d35a85c
commit f7dabee
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 3 deletions.
diff --git a/examples/machine-learning/a3-highgpu-8g/ml-slurm-a3-2-cluster.yaml b/examples/machine-learning/a3-highgpu-8g/ml-slurm-a3-2-cluster.yaml
@@ -52,6 +52,7 @@ vars:
   enable_ops_agent: true
   # enable the NVIDIA DCGM daemon and integration into Cloud Ops Agent
   enable_nvidia_dcgm: true
+  localssd_mountpoint: /mnt/localssd
 
 deployment_groups:
 - group: cluster
@@ -114,8 +115,17 @@ deployment_groups:
       # Failure to do will result in VMs that lose data and do not automatically
       # mount local SSD filesystems
       local_ssd_filesystem:
-        mountpoint: /mnt/localssd
+        mountpoint: $(vars.localssd_mountpoint)
         permissions: "1777" # must quote numeric filesystem permissions!
+      # Docker was successfully installed in the image, this configures it
+      # to use the A3-specific local SSD volumes to store container images
+      docker:
+        enabled: true
+        world_writable: true
+        daemon_config: |
+          {
+            "data-root": "$(vars.localssd_mountpoint)/docker"
+          }
       runners:
       - type: ansible-local
         destination: enable_nvidia_dcgm.yml

diff --git a/examples/machine-learning/a3-megagpu-8g/slurm-a3mega-cluster.yaml b/examples/machine-learning/a3-megagpu-8g/slurm-a3mega-cluster.yaml
@@ -30,6 +30,7 @@ vars:
     project: $(vars.project_id)
   enable_login_public_ips: true
   enable_controller_public_ips: true
+  localssd_mountpoint: /mnt/localssd
 
 deployment_groups:
 - group: cluster
@@ -89,8 +90,17 @@ deployment_groups:
       # Failure to do will result in VMs that lose data and do not automatically
       # mount local SSD filesystems
       local_ssd_filesystem:
-        mountpoint: /mnt/localssd
+        mountpoint: $(vars.localssd_mountpoint)
         permissions: "1777" # must quote numeric filesystem permissions!
+      # Docker was successfully installed in the image, this configures it
+      # to use the A3-specific local SSD volumes to store container images
+      docker:
+        enabled: true
+        world_writable: true
+        daemon_config: |
+          {
+            "data-root": "$(vars.localssd_mountpoint)/docker"
+          }
       runners:
       - type: ansible-local
         destination: slurm_aperture.yml

diff --git a/modules/scripts/startup-script/files/install_docker.yml b/modules/scripts/startup-script/files/install_docker.yml
@@ -43,7 +43,10 @@
       dest: /etc/docker/daemon.json
       mode: '0644'
       content: '{{ docker_daemon_config }}'
-      validate: /usr/bin/dockerd --validate --config-file %s
+      # validate flag requires Docker server version 23.0 and above
+      # can add this back after private A3 DLVM image is deprecated
+      # this image comes with Docker version 20.10.17
+      # validate: /usr/bin/dockerd --validate --config-file %s
     when: docker_daemon_config
     notify:
     - Restart Docker