Merge branch 'develop' into develop

hgreebe · Nov 17, 2023 · f65fba9 · f65fba9
2 parents d983a21 + 9b32ac7
commit f65fba9
Show file tree

Hide file tree

Showing 83 changed files with 466 additions and 5,130 deletions.
diff --git a/Berksfile b/Berksfile
@@ -10,7 +10,7 @@ metadata
 cookbook "apt", path: "./cookbooks/third-party/apt-7.5.14"
 cookbook "iptables", path: "./cookbooks/third-party/iptables-8.0.0"
 cookbook "line", path: "./cookbooks/third-party/line-4.5.13"
-cookbook "nfs", path: "./cookbooks/third-party/nfs-5.0.0"
+cookbook "nfs", path: "./cookbooks/third-party/nfs-5.1.2"
 cookbook "openssh", path: "./cookbooks/third-party/openssh-2.11.3"
 cookbook "pyenv", path: "./cookbooks/third-party/pyenv-4.2.3"
 cookbook "selinux", path: "./cookbooks/third-party/selinux-6.1.12"

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,23 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
+3.9.0
+------
+
+**ENHANCEMENTS**
+- Add support for installing Intel OneAPI Base Toolkit and HPC Toolkit, and Intel Python.
+  - Intel OneAPI Base Toolkits: 2023.2.0
+  - Intel OneAPI HPC Toolkits: 2023.2.0
+  - Intel Python: 2023.2.0
+  - Critical Update for Intel oneAPI DPC++/C++ Compiler: 2023.2.1
+  - Critical Update for Intel Fortran Compiler & Intel Fortran Compiler Classic: 2023.2.1
+
+**CHANGES**
+- Upgrade third-party cookbook dependencies:
+  - nfs-5.1.2 (from nfs-5.0.0)
+
+**BUG FIXES**
+
 3.8.0
 ------
 
@@ -28,6 +45,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **BUG FIXES**
 - Fix inconsistent scaling configuration after cluster update rollback when modifying the list of instance types declared in the Compute Resources.
+- Fix users SSH keys generation when switching users without root privilege in clusters integrated with an external LDAP server through cluster configuration files.
 
 3.7.2
 ------

diff --git a/cookbooks/aws-parallelcluster-awsbatch/Berksfile b/cookbooks/aws-parallelcluster-awsbatch/Berksfile
@@ -15,7 +15,7 @@ cookbook "aws-parallelcluster-slurm", path: "../aws-parallelcluster-slurm"
 cookbook "apt", path: "../third-party/apt-7.5.14"
 cookbook "iptables", path: "../third-party/iptables-8.0.0"
 cookbook "line", path: "../third-party/line-4.5.13"
-cookbook "nfs", path: "../third-party/nfs-5.0.0"
+cookbook "nfs", path: "../third-party/nfs-5.1.2"
 cookbook "openssh", path: "../third-party/openssh-2.11.3"
 cookbook "pyenv", path: "../third-party/pyenv-4.2.3"
 cookbook "selinux", path: "../third-party/selinux-6.1.12"

diff --git a/cookbooks/aws-parallelcluster-awsbatch/metadata.rb b/cookbooks/aws-parallelcluster-awsbatch/metadata.rb
@@ -15,7 +15,7 @@
 
 depends 'apt', '~> 7.5.14'
 depends 'iptables', '~> 8.0.0'
-depends 'nfs', '~> 5.0.0'
+depends 'nfs', '~> 5.1.2'
 depends 'line', '~> 4.5.13'
 depends 'openssh', '~> 2.11.3'
 depends 'pyenv', '~> 4.2.3'

diff --git a/cookbooks/aws-parallelcluster-entrypoints/Berksfile b/cookbooks/aws-parallelcluster-entrypoints/Berksfile
@@ -12,7 +12,7 @@ cookbook "aws-parallelcluster-slurm", path: "../aws-parallelcluster-slurm"
 cookbook "apt", path: "../third-party/apt-7.5.14"
 cookbook "iptables", path: "../third-party/iptables-8.0.0"
 cookbook "line", path: "../third-party/line-4.5.13"
-cookbook "nfs", path: "../third-party/nfs-5.0.0"
+cookbook "nfs", path: "../third-party/nfs-5.1.2"
 cookbook "openssh", path: "../third-party/openssh-2.11.3"
 cookbook "pyenv", path: "../third-party/pyenv-4.2.3"
 cookbook "selinux", path: "../third-party/selinux-6.1.12"

diff --git a/cookbooks/aws-parallelcluster-environment/Berksfile b/cookbooks/aws-parallelcluster-environment/Berksfile
@@ -4,7 +4,7 @@ metadata
 
 cookbook "aws-parallelcluster-shared", path: "../aws-parallelcluster-shared"
 
-cookbook "nfs", path: "../third-party/nfs-5.0.0"
+cookbook "nfs", path: "../third-party/nfs-5.1.2"
 
 # dependency od nfs
 cookbook "line", path: "../third-party/line-4.5.13"

diff --git a/cookbooks/aws-parallelcluster-environment/metadata.rb b/cookbooks/aws-parallelcluster-environment/metadata.rb
@@ -15,6 +15,6 @@
 supports 'redhat', '= 8.7'
 
 depends 'line', '~> 4.5.13'
-depends 'nfs', '~> 5.0.0'
+depends 'nfs', '~> 5.1.2'
 
 depends 'aws-parallelcluster-shared', '~> 3.9.0'
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config/configure_pam_ssh_keygen.rb b/cookbooks/aws-parallelcluster-environment/recipes/config/configure_pam_ssh_keygen.rb
@@ -18,7 +18,7 @@
 pam_services = %w(sudo su sshd)
 pam_config_dir = "/etc/pam.d"
 generate_ssh_key_path = "#{node['cluster']['scripts_dir']}/generate_ssh_key.sh"
-ssh_key_generator_pam_config_line = "session    optional     pam_exec.so log=/var/log/parallelcluster/pam_ssh_key_generator.log #{generate_ssh_key_path}"
+ssh_key_generator_pam_config_line = "session    optional     pam_exec.so seteuid log=/var/log/parallelcluster/pam_ssh_key_generator.log #{generate_ssh_key_path}"
 if node['cluster']["directory_service"]["generate_ssh_keys_for_users"] == 'true'
   template generate_ssh_key_path do
     source 'directory_service/generate_ssh_key.sh.erb'
@@ -44,7 +44,7 @@
     pam_config_file = "#{pam_config_dir}/#{pam_service}"
     delete_lines "Ensure PAM service #{pam_service} is not configured to call SSH key generation script" do
       path pam_config_file
-      pattern %r{session\s+optional\s+pam_exec\.so\s+log=/var/log/parallelcluster/pam_ssh_key_generator\.log}
+      pattern %r{session\s+optional\s+pam_exec\.so\s+seteuid\s+log=/var/log/parallelcluster/pam_ssh_key_generator\.log}
       ignore_missing true
     end
   end

diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/directory_service_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/directory_service_spec.rb
@@ -78,7 +78,7 @@
   pam_services.each do |pam_service|
     describe file("/etc/pam.d/#{pam_service}") do
       it { should exist }
-      its('content') { should match %r{session\s+optional\s+pam_exec\.so\s+log=/var/log/parallelcluster/pam_ssh_key_generator\.log} }
+      its('content') { should match %r{session\s+optional\s+pam_exec\.so\s+seteuid\s+log=/var/log/parallelcluster/pam_ssh_key_generator\.log} }
     end
   end
 

diff --git a/cookbooks/aws-parallelcluster-platform/kitchen.platform-config.yml b/cookbooks/aws-parallelcluster-platform/kitchen.platform-config.yml
@@ -93,13 +93,16 @@ suites:
     verifier:
       controls:
         - /tag:config_intel_hpc/
+        - /tag:intel_one_api/
     attributes:
       resource: intel_hpc:configure
       dependencies:
         - resource:package_repos:update
         - resource:intel_hpc:setup
       cluster:
         enable_intel_hpc_platform: 'true'
+        install_intel_base_toolkit: 'true'
+        install_intel_hpc_toolkit: 'true'
         node_type: HeadNode
   - name: sticky_bits
     run_list:

diff --git a/cookbooks/aws-parallelcluster-platform/resources/intel_hpc/install_intel_software.rb b/cookbooks/aws-parallelcluster-platform/resources/intel_hpc/install_intel_software.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :install_intel_software
+
+unified_mode true
+
+property :intel_offline_installer_dir, String, required: true
+property :software_name, String, required: true
+property :software_url, String, required: true
+
+default_action :install
+
+action :install do
+  remote_file "#{new_resource.intel_offline_installer_dir}/#{new_resource.software_name}" do
+    source new_resource.software_url
+    mode '0744'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+  bash "install Intel #{new_resource.software_name}" do
+    cwd new_resource.intel_offline_installer_dir
+    code <<-INTEL
+    set -e
+    sh #{new_resource.software_name} -a -s --eula accept
+    rm -f #{new_resource.software_name}
+    INTEL
+  end
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/intel_hpc/intel_hpc.rb b/cookbooks/aws-parallelcluster-platform/resources/intel_hpc/intel_hpc.rb
@@ -22,5 +22,71 @@
 end
 
 action :configure do
-  # do nothing
+  return unless intel_hpc_supported? && (node['cluster']['install_intel_base_toolkit'] == "true" || node['cluster']['install_intel_hpc_toolkit'] == "true" || node['cluster']['install_intel_python'] == "true")
+
+  base_toolkit_version = "2023.2.0.49397"
+  base_toolkit_name = "l_BaseKit_p_#{base_toolkit_version}_offline.sh"
+  base_toolkit_url = "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/992857b9-624c-45de-9701-f6445d845359/#{base_toolkit_name}"
+  hpc_toolkit_version = "2023.2.0.49440"
+  hpc_toolkit_name = "l_HPCKit_p_#{hpc_toolkit_version}_offline.sh"
+  hpc_toolkit_url = "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0722521a-34b5-4c41-af3f-d5d14e88248d/#{hpc_toolkit_name}"
+  intel_python_version = "2023.2.0.49422"
+  intel_python_name = "l_pythoni39_oneapi_p_#{intel_python_version}_offline.sh"
+  intel_python_url = "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/03aae3a8-623a-47cf-9655-5dd8fcf86430/#{intel_python_name}"
+  # Below are critical security updates not included in the tookits:
+  cpp_compiler_version = "2023.2.1.8"
+  cpp_compiler_name = "l_dpcpp-cpp-compiler_p_#{cpp_compiler_version}_offline.sh"
+  cpp_compiler_url = "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/ebf5d9aa-17a7-46a4-b5df-ace004227c0e/#{cpp_compiler_name}"
+  fortran_compiler_version = "2023.2.1.8"
+  fortran_compiler_name = "l_fortran-compiler_p_#{fortran_compiler_version}_offline.sh"
+  fortran_compiler_url = "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0d65c8d4-f245-4756-80c4-6712b43cf835/#{fortran_compiler_name}"
+
+  intel_offline_installer_dir = '/opt/intel/offlineInstaller'
+
+  directory intel_offline_installer_dir do
+    recursive true
+  end
+
+  if node['cluster']['install_intel_base_toolkit'] == "true"
+    install_intel_software "Install Intel Base Toolkit" do
+      software_name base_toolkit_name
+      software_url base_toolkit_url
+      intel_offline_installer_dir intel_offline_installer_dir
+    end
+    install_intel_software "Critical Update for Intel oneAPI DPC++/C++ Compiler" do
+      software_name cpp_compiler_name
+      software_url cpp_compiler_url
+      intel_offline_installer_dir intel_offline_installer_dir
+    end
+  end
+  if node['cluster']['install_intel_hpc_toolkit'] == "true"
+    install_intel_software "Intel OneAPI HPC Toolkits" do
+      software_name hpc_toolkit_name
+      software_url hpc_toolkit_url
+      intel_offline_installer_dir intel_offline_installer_dir
+    end
+    install_intel_software "Critical Update for Intel Fortran Compiler & Intel Fortran Compiler Classic" do
+      software_name fortran_compiler_name
+      software_url fortran_compiler_url
+      intel_offline_installer_dir intel_offline_installer_dir
+    end
+  end
+  if node['cluster']['install_intel_python'] == "true"
+    install_intel_software "Install Intel Python" do
+      software_name intel_python_name
+      software_url intel_python_url
+      intel_offline_installer_dir intel_offline_installer_dir
+    end
+  end
+  bash "copy Intel modulefiles to MODULEPATH" do
+    cwd "/opt/intel"
+    code <<-INTEL
+    set -e
+    ./modulefiles-setup.sh --output-dir="/usr/share/Modules/modulefiles/intel"
+    INTEL
+  end
+end
+
+def intel_hpc_supported?
+  !arm_instance?
 end
diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/intel_hpc_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/intel_hpc_spec.rb
@@ -137,3 +137,31 @@
     its('mode') { should cmp '0755' }
   end
 end
+
+control 'tag:intel_one_api_toolkits_configured' do
+  # TODO: Enable this test in daily run. This test requires larger root volume size.
+  # TODO: After increasing the root volume size, config_intel_hpc_enough_space_on_root_volume needs to be ajusted.
+  title 'Checks Intel OneApi Toolkits have been installed'
+
+  only_if { !os_properties.on_docker? }
+  only_if { !os_properties.centos7? && !os_properties.arm? }
+
+  intel_directory = "/opt/intel"
+
+  if node['cluster']['install_intel_base_toolkit'] == 'true'
+    %w(advisor ccl compiler dal dnnl dpl ipp ippcp mkl vtune).each do |software|
+      describe directory("#{intel_directory}/#{software}") do
+        it { should exist }
+      end
+    end
+  end
+
+  modulefile_dir = "/usr/share/Modules/modulefiles"
+  # Intel PSXE module file
+  describe file("#{modulefile_dir}/intel") do
+    it { should exist }
+    its('owner') { should eq 'root' }
+    its('group') { should eq 'root' }
+    its('mode') { should cmp '0755' }
+  end
+end
diff --git a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb
@@ -49,10 +49,7 @@ def for_all_oses
 end
 
 def runner(platform:, version:, step_into: [])
-  path = '../aws-parallelcluster-shared/spec/ubuntu.22.json' if "#{platform}#{version}" == 'ubuntu22.04'
-  ChefSpec::SoloRunner.new(
-    platform: platform, version: version, step_into: step_into, path: path
-  ) do |node|
+  ChefSpec::SoloRunner.new(platform: platform, version: version, step_into: step_into) do |node|
     yield node if block_given?
   end
 end