Refactor environment recipes to be clearer in functional description …

…and remove unnecessary recipes
aws · Sep 19, 2023 · 861e670 · 861e670
1 parent 60d9074
commit 861e670
Show file tree

Hide file tree

Showing 13 changed files with 103 additions and 197 deletions.
diff --git a/cookbooks/aws-parallelcluster-entrypoints/recipes/update.rb b/cookbooks/aws-parallelcluster-entrypoints/recipes/update.rb
@@ -17,7 +17,7 @@
 end
 
 # generate the updated shared storages mapping file
-include_recipe 'aws-parallelcluster-environment::fs_update'
+include_recipe 'aws-parallelcluster-environment::update_fs_mapping'
 
 include_recipe 'aws-parallelcluster-environment::directory_service'
 include_recipe 'aws-parallelcluster-slurm::update' if node['cluster']['scheduler'] == 'slurm'

diff --git a/cookbooks/aws-parallelcluster-environment/kitchen.environment-config.yml b/cookbooks/aws-parallelcluster-environment/kitchen.environment-config.yml
@@ -414,10 +414,10 @@ suites:
         fsx_shared_dirs: ''
         raid_shared_dir: ''
         ephemeral_dir: test1
-  - name: fs_update
+  - name: update_fs_mapping
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::fs_update]
+      - recipe[aws-parallelcluster-environment::update_fs_mapping]
     verifier:
       controls:
         - fs_data_file_created_correctly
@@ -440,10 +440,10 @@ suites:
         fsx_dns_names: dns1,dns2
         fsx_mount_names: mount1,mount2
         fsx_volume_junction_paths: value1,value2
-  - name: fs_update_default_values
+  - name: update_fs_mapping_default_values
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::fs_update]
+      - recipe[aws-parallelcluster-environment::update_fs_mapping]
     verifier:
       controls:
         - fs_data_file_with_default_values
@@ -472,42 +472,43 @@ suites:
         scheduler: slurm
         head_node_imds_secured: 'true'
         head_node_imds_allowed_users: ['root', 'nobody']
-  - name: mount_shared_compute
-    run_list:
-      - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::mount_shared]
-    verifier:
-      controls:
-        - mount_home
-        - mount_shared_compute
-    attributes:
-      dependencies:
-        - recipe:aws-parallelcluster-platform::directories
-        - resource:nfs
-        - recipe:aws-parallelcluster-environment::mock_export_directories
-      cluster:
-        node_type: 'ComputeFleet'
-        head_node_private_ip: '127.0.0.1'
-        head_node_home_path: '/fake_headnode_home'
-        shared_dir_head: '/fake_headnode_shared'
-  - name: mount_shared_login
-    run_list:
-      - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::mount_shared]
-    verifier:
-      controls:
-        - mount_home
-        - mount_shared_login
-    attributes:
-      dependencies:
-        - recipe:aws-parallelcluster-platform::directories
-        - resource:nfs
-        - recipe:aws-parallelcluster-environment::mock_export_directories
-      cluster:
-        node_type: 'LoginNode'
-        head_node_private_ip: '127.0.0.1'
-        head_node_home_path: '/fake_headnode_home'
-        shared_dir_head: '/fake_headnode_shared'
+# TODO replacement for the mount_internal_use_fs recipe since it uses shared storage
+#  - name: mount_shared_compute
+#    run_list:
+#      - recipe[aws-parallelcluster-tests::setup]
+#      - recipe[aws-parallelcluster-environment::mount_shared]
+#    verifier:
+#      controls:
+#        - mount_home
+#        - mount_shared_compute
+#    attributes:
+#      dependencies:
+#        - recipe:aws-parallelcluster-platform::directories
+#        - resource:nfs
+#        - recipe:aws-parallelcluster-environment::mock_export_directories
+#      cluster:
+#        node_type: 'ComputeFleet'
+#        head_node_private_ip: '127.0.0.1'
+#        head_node_home_path: '/fake_headnode_home'
+#        shared_dir_head: '/fake_headnode_shared'
+#  - name: mount_shared_login
+#    run_list:
+#      - recipe[aws-parallelcluster-tests::setup]
+#      - recipe[aws-parallelcluster-environment::mount_shared]
+#    verifier:
+#      controls:
+#        - mount_home
+#        - mount_shared_login
+#    attributes:
+#      dependencies:
+#        - recipe:aws-parallelcluster-platform::directories
+#        - resource:nfs
+#        - recipe:aws-parallelcluster-environment::mock_export_directories
+#      cluster:
+#        node_type: 'LoginNode'
+#        head_node_private_ip: '127.0.0.1'
+#        head_node_home_path: '/fake_headnode_home'
+#        shared_dir_head: '/fake_headnode_shared'
   - name: raid_compute
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
@@ -541,7 +542,7 @@ suites:
   - name: shared_storages_compute
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::shared_storages]
+      - recipe[aws-parallelcluster-environment::mount_cx_fs]
     verifier:
       controls:
         - shared_storages_compute_and_login
@@ -555,7 +556,7 @@ suites:
   - name: shared_storages_login
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-environment::shared_storages]
+      - recipe[aws-parallelcluster-environment::mount_cx_fs]
     verifier:
       controls:
         - shared_storages_compute_and_login

diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config.rb b/cookbooks/aws-parallelcluster-environment/recipes/config.rb
@@ -18,9 +18,9 @@
   action :configure
 end
 include_recipe 'aws-parallelcluster-environment::ephemeral_drives'
-# fs_update generates the shared storages mapping file so must be executed before shared storages recipes
-include_recipe 'aws-parallelcluster-environment::fs_update'
-include_recipe 'aws-parallelcluster-environment::shared_storages'
+# update_fs_mapping generates the shared storages mapping file so must be executed before shared storages recipes
+include_recipe 'aws-parallelcluster-environment::update_fs_mapping'
+include_recipe 'aws-parallelcluster-environment::export_home'
 include_recipe 'aws-parallelcluster-environment::ebs'
 include_recipe 'aws-parallelcluster-environment::raid'
-include_recipe "aws-parallelcluster-environment::fs_mount"
+include_recipe "aws-parallelcluster-environment::mount_cx_fs"
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config/export_home.rb b/cookbooks/aws-parallelcluster-environment/recipes/config/export_home.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+#
+# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+return if on_docker?
+
+case node['cluster']['node_type']
+when 'HeadNode'
+  volume "export /home" do
+    shared_dir "/home"
+    action :export
+  end
+when 'ComputeFleet', 'LoginNode'
+  Chef::Log.info("Export only from the HeadNode")
+else
+  raise "node_type must be HeadNode, ComputeFleet, or LoginNode"
+end
diff --git a/...er-environment/recipes/config/fs_mount.rb → ...environment/recipes/config/mount_cx_fs.rb b/...er-environment/recipes/config/fs_mount.rb → ...environment/recipes/config/mount_cx_fs.rb
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config/shared_storages.rb b/cookbooks/aws-parallelcluster-environment/recipes/config/shared_storages.rb
diff --git a/...r-environment/recipes/config/fs_update.rb → ...nment/recipes/config/update_fs_mapping.rb b/...r-environment/recipes/config/fs_update.rb → ...nment/recipes/config/update_fs_mapping.rb
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/init.rb b/cookbooks/aws-parallelcluster-environment/recipes/init.rb
@@ -16,13 +16,11 @@
 cloudwatch "Configure CloudWatch" do
   action :configure
 end
-include_recipe "aws-parallelcluster-environment::fs_update"
+include_recipe "aws-parallelcluster-environment::update_fs_mapping"
 include_recipe "aws-parallelcluster-environment::backup_internal_use_shared_data"
 include_recipe "aws-parallelcluster-environment::mount_internal_use_fs"
 include_recipe "aws-parallelcluster-environment::restore_internal_use_shared_data"
 
-# include_recipe "aws-parallelcluster-environment::mount_shared"
-
 include_recipe "aws-parallelcluster-environment::network_interfaces"
 include_recipe 'aws-parallelcluster-environment::imds'
 

diff --git a/cookbooks/aws-parallelcluster-environment/recipes/init/mount_shared.rb b/cookbooks/aws-parallelcluster-environment/recipes/init/mount_shared.rb
diff --git a/...onment/spec/unit/recipes/fs_mount_spec.rb → ...ent/spec/unit/recipes/mount_cx_fs_spec.rb b/...onment/spec/unit/recipes/fs_mount_spec.rb → ...ent/spec/unit/recipes/mount_cx_fs_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe 'aws-parallelcluster-environment::fs_mount' do
+describe 'aws-parallelcluster-environment::mount_cx_fs' do
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
       cached(:chef_run) do
@@ -13,10 +13,10 @@
       end
       cached(:node) { chef_run.node }
 
-      describe 'call the efs for mounting' do
+      describe 'call efs for mounting' do
         it { is_expected.to mount_efs('mount efs') }
       end
-      describe 'call the lustre for mounting' do
+      describe 'call lustre for mounting' do
         it { is_expected.to mount_lustre("mount fsx") }
       end
     end

diff --git a/...nt/spec/unit/recipes/mount_shared_spec.rb → ...nit/recipes/mount_internal_use_fs_spec.rb b/...nt/spec/unit/recipes/mount_shared_spec.rb → ...nit/recipes/mount_internal_use_fs_spec.rb
@@ -1,12 +1,14 @@
 require 'spec_helper'
 
-describe 'aws-parallelcluster-environment::mount_shared' do
+describe 'aws-parallelcluster-environment::mount_internal_use_fs' do
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
       cached(:chef_run) do
         runner = runner(platform: platform, version: version) do |node|
           node.override['cluster']['head_node_private_ip'] = '0.0.0.0'
           node.override['cluster']['node_type'] = 'ComputeFleet'
+          node.override['cluster']['internal_shared_dirs'] = %w[/opt/slurm /opt/intel]
+          node.override['cluster']['efs_shared_dirs'] = "/opt/slurm,/opt/intel"
         end
         runner.converge(described_recipe)
       end
@@ -21,13 +23,8 @@
           .with(retry_delay: 6)
       end
 
-      it 'mounts /opt/parallelcluster/shared' do
-        is_expected.to mount_volume('mount /opt/parallelcluster/shared')
-          .with(device: "0.0.0.0:/opt/parallelcluster/shared")
-          .with(fstype: 'nfs')
-          .with(options: 'hard,_netdev,noatime')
-          .with(retries: 10)
-          .with(retry_delay: 6)
+      describe 'call efs for mounting' do
+        it { is_expected.to mount_efs('mount internal use efs') }
       end
     end
   end

diff --git a/...onment/test/controls/mount_shared_spec.rb → ...st/controls/mount_internal_use_fs_spec.rb b/...onment/test/controls/mount_shared_spec.rb → ...st/controls/mount_internal_use_fs_spec.rb
@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 control 'mount_home' do
-  title 'Check if the home and the shared directories are mounted'
+  title 'Check if the home directory in mounted'
 
   only_if { !os_properties.on_docker? && (instance.compute_node? or instance.login_node?) }
 
@@ -22,7 +22,7 @@
 end
 
 control 'mount_shared_compute' do
-  title 'Check if the home and the shared directories are mounted'
+  title 'Check if the shared directory is mounted'
 
   only_if { !os_properties.on_docker? && instance.compute_node? }
 
@@ -34,7 +34,7 @@
 end
 
 control 'mount_shared_login' do
-  title 'Check if the home and the shared directories are mounted'
+  title 'Check if the shared directory is mounted'
 
   only_if { !os_properties.on_docker? && instance.login_node? }
 
@@ -44,3 +44,19 @@
     its('options') { should include 'rw' }
   end
 end
+
+control 'shared_storages_compute_and_login' do
+  title 'Check the shared storages configuration for compute node'
+
+  only_if { !os_properties.on_docker? && (instance.compute_node? or instance.login_node?) }
+
+  describe 'Check that /opt/intel dir has been mounted'
+  describe mount("/opt/intel") do
+    it { should be_mounted }
+    its('device') { should eq "127.0.0.1:/opt/intel" }
+    its('type') { should eq 'nfs4' }
+    its('options') { should include 'hard' }
+    its('options') { should include '_netdev' }
+    its('options') { should include 'noatime' }
+  end
+end