[Develop] Add support for an external shared /home

Users can now specify `/home` as a mount point in SharedStorage.  This was previously a reserved directory
When a user specifies this, the data in `/home` is transferred without replacement to the external filesystem.
This means that in order to share the `/home` directory across clusters, users must specify the same security
credentials when creating the clusters.

CHANGELOG.md

@@ -11,6 +11,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Install [Spack](https://spack.io) by default in cluster user's home directory.
 - Add support for `Scheduling/SlurmSettings/Database/DatabaseName` parameter to render `StorageLoc` in the slurmdbd configuration generated by ParallelCluster.
 - Add the option to use EFS storage instead of NFS exports from the head node root volume for intra-cluster shared ParallelCluster, Intel, Slurm, and login node data.
+- Allow for mounting `home` as an EFS or FSx external shared storage via the `SharedStorage` section of the config file.
 
 **CHANGES**
 

cookbooks/aws-parallelcluster-environment/recipes/config.rb

@@ -18,9 +18,9 @@
   action :configure
 end
 include_recipe 'aws-parallelcluster-environment::ephemeral_drives'
-# update_fs_mapping generates the shared storages mapping file so must be executed before shared storages recipes
+# update_fs_mapping generates the shared storage mapping file, so it must be executed before shared storage recipes
 include_recipe 'aws-parallelcluster-environment::update_fs_mapping'
-# Export home dir from the head node
+# Export the home dir from the head node when using ebs
 include_recipe 'aws-parallelcluster-environment::export_home'
 
 if node['cluster']['internal_shared_storage_type'] == 'ebs'
@@ -32,16 +32,5 @@
 
 include_recipe 'aws-parallelcluster-environment::ebs'
 include_recipe 'aws-parallelcluster-environment::raid'
-include_recipe "aws-parallelcluster-environment::efs"
-
-# Mount FSx directory with manage_fsx resource
-lustre "mount fsx" do
-  fsx_fs_id_array node['cluster']['fsx_fs_ids'].split(',')
-  fsx_fs_type_array node['cluster']['fsx_fs_types'].split(',')
-  fsx_shared_dir_array node['cluster']['fsx_shared_dirs'].split(',')
-  fsx_dns_name_array node['cluster']['fsx_dns_names'].split(',')
-  fsx_mount_name_array node['cluster']['fsx_mount_names'].split(',')
-  fsx_volume_junction_path_array node['cluster']['fsx_volume_junction_paths'].split(',')
-  action :mount
-  not_if { node['cluster']['fsx_fs_ids'].split(',').empty? }
-end
+include_recipe 'aws-parallelcluster-environment::efs'
+include_recipe 'aws-parallelcluster-environment::fsx'

cookbooks/aws-parallelcluster-environment/recipes/config/efs.rb

@@ -11,31 +11,26 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 return if on_docker?
-efs_shared_dir_array = node['cluster']['efs_shared_dirs'].split(',')
-efs_fs_id_array = node['cluster']['efs_fs_ids'].split(',')
-efs_encryption_in_transit_array = node['cluster']['efs_encryption_in_transits'].split(',')
-efs_iam_authorization_array = node['cluster']['efs_iam_authorizations'].split(',')
+shared_dir_array = node['cluster']['efs_shared_dirs'].split(',')
+id_array = node['cluster']['efs_fs_ids'].split(',')
+encryption_array = node['cluster']['efs_encryption_in_transits'].split(',')
+iam_array = node['cluster']['efs_iam_authorizations'].split(',')
 
-cx_shared_dir_array = []
-cx_efs_fs_id_array = []
-cx_efs_encryption_array = []
-cx_efs_iam_array = []
-
-# Identify the customer use filesystems and store their data in arrays for the EFS resource
-efs_shared_dir_array.each_with_index do |dir, index|
-  next if node['cluster']['internal_shared_dirs'].include?(dir)
-  cx_shared_dir_array.push(dir)
-  cx_efs_fs_id_array.push(efs_fs_id_array[index])
-  cx_efs_encryption_array.push(efs_encryption_in_transit_array[index])
-  cx_efs_iam_array.push(efs_iam_authorization_array[index])
+# Identify the previously mounted filesystems and remove them from the set of filesystems to mount
+shared_dir_array.each_with_index do |dir, index|
+  next unless node['cluster']['internal_shared_dirs'].include?(dir) || dir == "/home"
+  shared_dir_array.delete(dir)
+  id_array.delete_at(index)
+  encryption_array.delete_at(index)
+  iam_array.delete_at(index)
 end
 
 # Mount EFS directories with the efs resource
 efs "mount efs" do
-  shared_dir_array cx_shared_dir_array
-  efs_fs_id_array cx_efs_fs_id_array
-  efs_encryption_in_transit_array cx_efs_encryption_array
-  efs_iam_authorization_array cx_efs_iam_array
+  shared_dir_array shared_dir_array
+  efs_fs_id_array id_array
+  efs_encryption_in_transit_array encryption_array
+  efs_iam_authorization_array iam_array
   action :mount
-  not_if { cx_shared_dir_array.empty? }
+  not_if { shared_dir_array.empty? }
 end

cookbooks/aws-parallelcluster-environment/recipes/config/export_home.rb

@@ -14,6 +14,18 @@
 
 return if on_docker?
 
+return unless node['cluster']['internal_shared_storage_type'] == 'ebs'
+
+# Check if home is a shared filesystem and return if it is because
+# there is nothing to export
+is_home_shared = false
+efs_shared_dir_array = node['cluster']['efs_shared_dirs'].split(',')
+fsx_shared_dir_array = node['cluster']['fsx_shared_dirs'].split(',')
+(efs_shared_dir_array + fsx_shared_dir_array).each do |dir|
+  is_home_shared = true if dir == "/home"
+end
+return if is_home_shared
+
 case node['cluster']['node_type']
 when 'HeadNode'
   volume "export /home" do

cookbooks/aws-parallelcluster-environment/recipes/config/fsx.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+return if on_docker?
+id_array = node['cluster']['fsx_fs_ids'].split(',')
+type_array = node['cluster']['fsx_fs_types'].split(',')
+shared_dir_array = node['cluster']['fsx_shared_dirs'].split(',')
+dns_name_array = node['cluster']['fsx_dns_names'].split(',')
+mount_name_array = node['cluster']['fsx_mount_names'].split(',')
+volume_junction_path_array = node['cluster']['fsx_volume_junction_paths'].split(',')
+
+# Identify the previously mounted filesystems and remove them from the set of filesystems to mount
+shared_dir_array.each_with_index do |dir, index|
+  next unless dir == "/home"
+  id_array.delete_at(index)
+  type_array.delete_at(index)
+  shared_dir_array.delete(dir)
+  dns_name_array.delete_at(index)
+  mount_name_array.delete_at(index)
+  volume_junction_path_array.delete_at(index)
+end
+
+# Mount FSx shared directories with the lustre resource
+lustre "mount fsx" do
+  fsx_fs_id_array id_array
+  fsx_fs_type_array type_array
+  fsx_shared_dir_array shared_dir_array
+  fsx_dns_name_array dns_name_array
+  fsx_mount_name_array mount_name_array
+  fsx_volume_junction_path_array volume_junction_path_array
+  action :mount
+  not_if { id_array.empty? }
+end

cookbooks/aws-parallelcluster-environment/recipes/config/mount_home.rb

@@ -11,12 +11,96 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 
-volume "mount /home" do
-  action :mount
-  shared_dir '/home'
-  device(lazy { "#{node['cluster']['head_node_private_ip']}:#{node['cluster']['head_node_home_path']}" })
-  fstype 'nfs'
-  options node['cluster']['nfs']['hard_mount_options']
-  retries 10
-  retry_delay 6
+return if on_docker?
+
+shared_storage = { 'efs' => node['cluster']['efs_shared_dirs'].split(','),
+                   # Disambiguate from the lustre resource property below
+                   'fsx' => node['cluster']['fsx_shared_dirs'].split(','),
+                   'ebs' => node['cluster']['ebs_shared_dirs'].split(','),
+                   # Making sure they are all the same object type, even though raid is just a string
+                   'raid' => node['cluster']['raid_shared_dir'].split(','),
+}
+
+# Check if home is a shared filesystem
+shared_home = 'none'
+shared_storage.each do |type, dirs|
+  next unless dirs.include?('/home')
+  shared_home = type
+end
+
+if shared_home == 'none'
+  # Mount the NFS export to compute and login nodes, the head node will export /home later
+  case node['cluster']['node_type']
+  when 'ComputeFleet', 'LoginNode'
+    volume "mount /home" do
+      action :mount
+      shared_dir '/home'
+      device(lazy { "#{node['cluster']['head_node_private_ip']}:#{node['cluster']['head_node_home_path']}" })
+      fstype 'nfs'
+      options node['cluster']['nfs']['hard_mount_options']
+      retries 10
+      retry_delay 6
+    end
+  when 'HeadNode'
+    Chef::Log.info("Do not mount NFS shares on the HeadNode")
+  else
+    raise "node_type must be ComputeFleet, LoginNode or HeadNode"
+  end
+else
+  # Identify the filesystem that is shared and mount it
+  include_recipe "aws-parallelcluster-environment::update_fs_mapping"
+  include_recipe "aws-parallelcluster-environment::backup_home_shared_data"
+  case shared_home
+  when 'efs'
+    shared_storage['efs'].each_with_index do |dir, index|
+      next unless dir == "/home"
+      efs "mount shared efs home" do
+        shared_dir_array [dir]
+        efs_fs_id_array [node['cluster']['efs_fs_ids'].split(',')[index]]
+        efs_encryption_in_transit_array [node['cluster']['efs_encryption_in_transits'].split(',')[index]]
+        efs_iam_authorization_array [node['cluster']['efs_iam_authorizations'].split(',')[index]]
+        action :mount
+      end
+      break
+    end
+  when 'fsx'
+    shared_storage['fsx'].each_with_index do |dir, index|
+      next unless dir == "/home"
+      lustre "mount shared fsx home" do
+        fsx_fs_id_array [node['cluster']['fsx_fs_ids'].split(',')[index]]
+        fsx_fs_type_array [node['cluster']['fsx_fs_types'].split(',')[index]]
+        fsx_shared_dir_array [dir]
+        fsx_dns_name_array [node['cluster']['fsx_dns_names'].split(',')[index]]
+        fsx_mount_name_array [node['cluster']['fsx_mount_names'].split(',')[index]]
+        fsx_volume_junction_path_array [node['cluster']['fsx_volume_junction_paths'].split(',')[index]]
+        action :mount
+      end
+      break
+    end
+  when 'ebs'
+    case node['cluster']['node_type']
+    when 'HeadNode'
+      shared_storage['ebs'].each_with_index do |dir, index|
+        next unless dir == "/home"
+        manage_ebs "add ebs" do
+          shared_dir_array [dir]
+          vol_array [node['cluster']['volume'].split(',')[index]]
+          action %i(mount export)
+        end
+      end
+    when 'ComputeFleet', 'LoginNode'
+      volume "mount /home" do
+        action :mount
+        shared_dir '/home'
+        device(lazy { "#{node['cluster']['head_node_private_ip']}:#{format_directory('/home')}" })
+        fstype 'nfs'
+        options node['cluster']['nfs']['hard_mount_options']
+        retries 10
+        retry_delay 6
+      end
+    end
+  when 'raid'
+    Chef::Log.info("TODO add raid")
+  end
+  include_recipe "aws-parallelcluster-environment::restore_home_shared_data"
 end

cookbooks/aws-parallelcluster-environment/recipes/init.rb

@@ -26,7 +26,8 @@
   raise "internal_shared_storage_type must be ebs or efs"
 end
 
-include_recipe "aws-parallelcluster-environment::mount_home" if %w(ComputeFleet LoginNode).include? node['cluster']['node_type']
+# Mount the home directory to all nodes if it is shared, otherwise mount the NFS share to compute and login nodes
+include_recipe "aws-parallelcluster-environment::mount_home"
 
 include_recipe "aws-parallelcluster-environment::network_interfaces"
 include_recipe 'aws-parallelcluster-environment::imds'

cookbooks/aws-parallelcluster-environment/recipes/init/backup_home_shared_data.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+#
+# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+return if on_docker?
+
+if node['cluster']['node_type'] == 'HeadNode'
+  # For each, backup the data to a temp location
+  # This is necessary to preserve any data in these directories that was
+  # generated during the build of ParallelCluster AMIs after converting to
+  # shared storage
+  bash "Backup /home" do
+    user 'root'
+    group 'root'
+    code <<-EOH
+      mkdir -p /tmp/home
+      rsync -a /home/ /tmp/home
+    EOH
+  end
+end

cookbooks/aws-parallelcluster-environment/recipes/init/restore_home_shared_data.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+#
+# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+return if on_docker?
+
+if node['cluster']['node_type'] == 'HeadNode'
+  # Restore the shared storage home data if it doesn't already exist
+  # This is necessary to preserve any data in these directories that was
+  # generated during the build of ParallelCluster AMIs after converting to
+  # shared storage and backed up to a temporary location previously
+  # Remove the backup after the copy is done
+  bash "Restore /home" do
+    user 'root'
+    group 'root'
+    code <<-EOH
+      rsync -a --ignore-existing /tmp/home/ /home
+      rm -rf /tmp/home/
+    EOH
+  end
+end