[ADC Build Image] Include differing paths for iso regions

hgreebe · Jul 24, 2024 · 5f21850 · 5f21850
1 parent 6b278d2
commit 5f21850
Show file tree

Hide file tree

Showing 15 changed files with 134 additions and 118 deletions.
diff --git a/cookbooks/aws-parallelcluster-awsbatch/recipes/install.rb b/cookbooks/aws-parallelcluster-awsbatch/recipes/install.rb
@@ -28,6 +28,9 @@
 
 # Check whether install a custom aws-parallelcluster-awsbatch-cli package or the standard one
 # Install awsbatch cli into awsbatch virtual env
+if aws_region.start_with?("us-iso") && !node['cluster']['custom_awsbatchcli_package'].empty?
+  node['cluster']['custom_awsbatchcli_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/awsbatch/aws-parallelcluster.tgz"
+end
 if !node['cluster']['custom_awsbatchcli_package'].nil? && !node['cluster']['custom_awsbatchcli_package'].empty?
   # Install custom aws-parallelcluster package
   bash "install aws-parallelcluster-awsbatch-cli" do
@@ -49,23 +52,7 @@
   end
 else
   # Install aws-parallelcluster-awsbatch-cli package
-  remote_file "#{Chef::Config[:file_cache_path]}/aws-parallelcluster.tgz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/awsbatch/aws-parallelcluster.tgz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
-
-  bash "install aws-parallelcluster-awsbatch-cli" do
-    cwd Chef::Config[:file_cache_path]
-    code <<-CLI
-      set -e
-      mkdir aws-parallelcluster-awsbatch-cli
-      tar -xzf aws-parallelcluster.tgz --directory aws-parallelcluster-awsbatch-cli
-
-      cd aws-parallelcluster-awsbatch-cli/*aws-parallelcluster-*
-      #{node['cluster']['awsbatch_virtualenv_path']}/bin/pip install awsbatch-cli/
-    CLI
+  execute "pip_install_parallelcluster_awsbatch_cli" do
+    command "#{node['cluster']['awsbatch_virtualenv_path']}/bin/pip install aws-parallelcluster-awsbatch-cli==#{node['cluster']['parallelcluster-awsbatch-cli-version']}"
   end
 end
diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb
@@ -19,6 +19,25 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
+remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf node-dependencies.tgz
+    cd node
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]

diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb
@@ -33,51 +33,14 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf node-dependencies.tgz
-    cd node
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
+if aws_region.start_with?("us-iso") && !is_custom_node?
+  node['cluster']['custom_node_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/node/aws-parallelcluster-node.tgz"
 end
 
 if is_custom_node?
   include_recipe 'aws-parallelcluster-computefleet::custom_parallelcluster_node'
 else
-  remote_file "#{Chef::Config[:file_cache_path]}/aws-parallelcluster-node.tgz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/node/aws-parallelcluster-node.tgz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
-
-  bash "install official aws-parallelcluster-node" do
-    cwd Chef::Config[:file_cache_path]
-    code <<-NODE
-    set -e
-    [[ ":$PATH:" != *":/usr/local/bin:"* ]] && PATH="/usr/local/bin:${PATH}"
-    echo "PATH is $PATH"
-    source #{node_virtualenv_path}/bin/activate
-    pip uninstall --yes aws-parallelcluster-node
-    rm -fr aws-parallelcluster-node
-    mkdir aws-parallelcluster-node
-    tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-node
-    cd aws-parallelcluster-node/*aws-parallelcluster-node-*
-    pip install .
-    deactivate
-  NODE
+  execute "install official aws-parallelcluster-node" do
+    command "#{virtualenv_path}/bin/pip install aws-parallelcluster-node==#{node['cluster']['parallelcluster-node-version']}"
   end
 end
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install.rb b/cookbooks/aws-parallelcluster-environment/recipes/install.rb
@@ -21,7 +21,7 @@
 raid 'Install RAID prerequisite packages'
 lustre 'Install FSx options'
 efs 'Install efs-utils'
-# stunnel 'Install stunnel'
+stunnel 'Install stunnel'
 system_authentication "Install packages required for directory service integration"
 # TODO: Disable spack until the feature is complete
 # spack 'Install Spack'
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efa/efa_alinux2.rb b/cookbooks/aws-parallelcluster-environment/resources/efa/efa_alinux2.rb
@@ -24,6 +24,10 @@ def conflicting_packages
   end
 
   def prerequisites
-    %w(libibverbs-utils librdmacm-utils)
+    if aws_region.start_with?("us-iso")
+      %w(libibverbs-utils librdmacm-utils)
+    else
+      %w(environment-modules libibverbs-utils librdmacm-utils)
+    end
   end
 end
diff --git a/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb
@@ -44,8 +44,18 @@
 action_class do
   def base_url
     # https://docs.aws.amazon.com/fsx/latest/LustreGuide/install-lustre-client.html#lustre-client-rhel
-    "https://fsx-lustre-client-repo.s3.#{aws_region}.#{aws_domain}/el/#{node['platform_version']}/$basearch"  end
+    if aws_region.start_with?("us-iso")
+      "https://fsx-lustre-client-repo.s3.#{aws_region}.#{aws_domain}/el/#{node['platform_version']}/$basearch"
+    else
+      "https://fsx-lustre-client-repo.s3.amazonaws.com/el/#{node['platform_version']}/$basearch"
+    end
+  end
 
   def public_key
-    "https://fsx-lustre-client-repo-public-keys.s3.#{aws_region}.#{aws_domain}/fsx-rpm-public-key.asc"  end
+    if aws_region.start_with?("us-iso")
+      "https://fsx-lustre-client-repo-public-keys.s3.#{aws_region}.#{aws_domain}/fsx-rpm-public-key.asc"
+    else
+      "https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc"
+    end
+  end
 end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_amazon2.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_amazon2.rb
@@ -25,6 +25,14 @@ def gdrcopy_platform
   'amzn-2'
 end
 
+def gdrcopy_build_dependencies
+  if aws_region.start_with?("us-iso")
+    %w(dkms rpm-build make check check-devel)
+  else
+    %w(dkms rpm-build make check check-devel subunit subunit-devel)
+  end
+end
+
 def gdrcopy_arch
   arm_instance? ? 'aarch64' : 'x86_64'
 end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb
@@ -24,7 +24,11 @@ def gdrcopy_enabled?
 end
 
 def gdrcopy_build_dependencies
-  %w(rpm-build make check check-devel)
+  if aws_region.start_with?("us-iso")
+    %w(rpm-build make check check-devel)
+  else
+    %w(dkms rpm-build make check check-devel subunit subunit-devel)
+  end
 end
 
 def gdrcopy_platform

diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/partial/_gdrcopy_common_rhel.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/partial/_gdrcopy_common_rhel.rb
@@ -16,10 +16,6 @@ def gdrcopy_service
   'gdrcopy'
 end
 
-def gdrcopy_build_dependencies
-  %w(dkms rpm-build make check check-devel)
-end
-
 def installation_code
   <<~COMMAND
   CUDA=/usr/local/cuda ./build-rpm-packages.sh

diff --git a/...books/aws-parallelcluster-platform/resources/install_packages/install_packages_amazon2.rb b/...books/aws-parallelcluster-platform/resources/install_packages/install_packages_amazon2.rb
@@ -22,7 +22,7 @@
 def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
   # iptables needed for IMDS setup
-  %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
+  packages = %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
      libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool
      httpd boost-devel system-lsb mlocate atlas-devel glibc-static iproute
      libffi-devel dkms libedit-devel sendmail cmake byacc libglvnd-devel libgcrypt-devel libevent-devel
@@ -32,6 +32,12 @@ def default_packages
      jq wget python-pip NetworkManager-config-routing-rules
      python3 python3-pip iptables libcurl-devel yum-plugin-versionlock
      coreutils environment-modules bzip2)
+
+  if aws_region.start_with?("us-iso")
+    packages -= %w(moreutils)
+  end
+
+  packages
 end
 
 action :install_extras do
@@ -41,22 +47,24 @@ def default_packages
     alinux_extras_topic topic
   end
 
-  remote_file "epel_deps.tar.gz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel7/#{node['kernel']['machine']}/epel_deps.tar.gz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
+  if aws_region.start_with?("us-iso")
+    remote_file "epel_deps.tar.gz" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel7/#{node['kernel']['machine']}/epel_deps.tar.gz"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
 
-  bash 'yum install missing deps' do
-    user 'root'
-    group 'root'
-    code <<-REQ
-    set -e
-    tar xzf epel_deps.tar.gz
-    cd epel
-    yum install -y * 2>/dev/null
-    REQ
+    bash 'yum install missing deps' do
+      user 'root'
+      group 'root'
+      code <<-REQ
+      set -e
+      tar xzf epel_deps.tar.gz
+      cd epel
+      yum install -y * 2>/dev/null
+      REQ
+    end
   end
 end
diff --git a/...books/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb b/...books/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb
@@ -23,41 +23,42 @@ def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
   # Removed libssh2-devel from base_packages since is not shipped by RedHat 8 and in conflict with package libssh-0.9.6-3.el8.x86_64
   # iptables needed for IMDS setup
-  # %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
-  #    libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf libtool
-  #    httpd boost-devel mlocate R atlas-devel
-  #    blas-devel libffi-devel dkms libedit-devel jq
-  #    libical-devel sendmail libxml2-devel libglvnd-devel
-  #    libgcrypt-devel libevent-devel glibc-static bind-utils
-  #    iproute NetworkManager-config-routing-rules python3 python3-pip iptables libcurl-devel yum-plugin-versionlock
-  #    coreutils moreutils curl environment-modules gcc gcc-c++ bzip2) R
-  %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools
-     libXmu-devel libdb-devel tcl-devel automake autoconf libtool
-     httpd boost-devel mlocate atlas-devel
-     libffi-devel jq
+  packages = %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
+     libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf libtool
+     httpd boost-devel mlocate R atlas-devel
+     blas-devel libffi-devel dkms libedit-devel jq
      libical-devel sendmail libxml2-devel libglvnd-devel
-     libgcrypt-devel libevent-devel bind-utils
-     iproute python3 python3-pip iptables libcurl-devel
-     coreutils curl environment-modules gcc gcc-c++ bzip2)
+     libgcrypt-devel libevent-devel glibc-static bind-utils
+     iproute NetworkManager-config-routing-rules python3 python3-pip iptables libcurl-devel yum-plugin-versionlock
+     coreutils moreutils curl environment-modules gcc gcc-c++ bzip2)
+
+  if aws_region.start_with?("us-iso")
+    packages -= %w(openmotif-devel hwloc-devel R blas-devel dkms libedit-devel glibc-static
+       NetworkManager-config-routing-rules yum-plugin-versionlock moreutils)
+  end
+
+  packages
 end
 
 action :install_extras do
-  remote_file "epel_deps.tar.gz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel8/x86_64/epel_deps.tar.gz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
+  if aws_region.start_with?("us-iso")
+    remote_file "epel_deps.tar.gz" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel8/x86_64/epel_deps.tar.gz"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
 
-  bash 'yum install missing deps' do
-    user 'root'
-    group 'root'
-    code <<-REQ
-    set -e
-    tar xzf epel_deps.tar.gz
-    cd epel
-    yum install -y * 2>/dev/null
-    REQ
+    bash 'yum install missing deps' do
+      user 'root'
+      group 'root'
+      code <<-REQ
+      set -e
+      tar xzf epel_deps.tar.gz
+      cd epel
+      yum install -y * 2>/dev/null
+      REQ
+    end
   end
 end
diff --git a/...oks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb b/...oks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb
@@ -66,6 +66,9 @@
       owner 'root'
       group 'root'
       mode '0644'
+      variables(
+        compiler_path: compiler_path
+      )
     end
   end
 

diff --git a/cookbooks/aws-parallelcluster-shared/resources/install_pyenv.rb b/cookbooks/aws-parallelcluster-shared/resources/install_pyenv.rb
@@ -16,6 +16,10 @@
   python_version = new_resource.python_version || node['cluster']['python-version']
   python_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/python/Python-#{python_version}.tgz"
 
+  if !aws_region.start_with?("us-iso") && new_resource.python_version
+    python_url = "https://www.python.org/ftp/python/#{python_version}/Python-#{python_version}.tgz"
+  end
+
   if new_resource.user_only
     raise "user property is required for resource install_pyenv when user_only is set to true" unless new_resource.user
     prefix = new_resource.prefix || "#{::File.expand_path("~#{user}")}/.pyenv"

diff --git a/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb b/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb
@@ -23,7 +23,10 @@
 
 action :setup do
   include_recipe 'yum'
-  # include_recipe "yum-epel"
+
+  unless aws_region.start_with?("us-iso")
+    include_recipe "yum-epel"
+  end
 
   package 'yum-utils' do
     retries 3

diff --git a/...ooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb b/...ooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb
@@ -15,5 +15,11 @@
 use 'partial/_slurm_dependencies_common'
 
 def dependencies
-  %w(json-c-devel perl dbus-devel)
+  packages = %w(json-c-devel http-parser-devel lua-devel perl dbus-devel)
+
+  if aws_region.start_with?("us-iso")
+    packages -= %w(http-parser-devel lua-devel)
+  end
+
+  packages
 end