Avoid to fail when building RHEL8.7 AMI with kernel not supported by …

…Lustre

The latest RHEL8.7 kernel 4.18.0-425.19.2.el8_7 introduced a kABI regression,
causing Lustre to re-release the client compiled for this latest version.

This change means that this same client will not modprobe for kernel 4.18.0-425.13.1.el8_7.

With this patch we're skipping Lustre installation and printing a warning in the log,
like we did for 4.18.0-425.3.1.el8 that has a similar issue.

### Tests

EC2:
```
# RHEL-8.7.0_HVM-20230330-x86_64-56-Hourly2-GP2 in eu-west-1
# with kernel 4.18.0-425.13.1.el8_7
export KITCHEN_RHEL8_AMI=ami-0fa2f7b35eeb82b7a
bash kitchen.ec2.sh environment-install test lustre-rhel8
```
Before the patch: this test was failing on converge phase,
After the patch: converge is passing and printing the following WARN message:
```
  WARN: FSx for Lustre is not supported in kernel version 4.18.0-425.13.1.el8_7.x86_64 of RHEL 8.7, please update the kernel version
```

Docker:
```
bash kitchen.docker.sh environment-install test lustre-rhel8
```
This test passes because in docker we have RHEL 8.8 with a fake value for kernel version and we're skipping modprobe command.
Use 477 version in the kernel release test variable to match 8.8 kernel version available on docker.

### References
* 4.18.0-425.3.1 issue: https://access.redhat.com/solutions/6985596
* 4.18.0-425.19.2 issue: openzfs/zfs#14724

Signed-off-by: Enrico Usai <[email protected]>

cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb

@@ -25,13 +25,15 @@
 
 action :setup do
   version = node['platform_version']
+  log "Installing FSx for Lustre. Platform version: #{version}, kernel version: #{node['cluster']['kernel_release']}"
   if version.to_f < 8.2
     log "FSx for Lustre is not supported in this RHEL version #{version}, supported versions are >= 8.2" do
       level :warn
     end
-    # rhel8 kernel 4.18.0-425.3.1.el8 has broken kABI compat https://github.com/openzfs/zfs/issues/14724
-  elsif node['cluster']['kernel_release'].include? "4.18.0-425.3.1.el8"
-    log "FSx for Lustre is not supported in kernel version 4.18.0-425.3.1.el8 of RHEL, please update the kernel version" do
+  elsif version.to_f == 8.7 && (node['cluster']['kernel_release'].include?("4.18.0-425.3.1.el8") || node['cluster']['kernel_release'].include?("4.18.0-425.13.1.el8_7"))
+    # Rhel8.7 kernel 4.18.0-425.3.1.el8 and 4.18.0-425.13.1.el8_7 has broken kABI compat
+    # See https://access.redhat.com/solutions/6985596 and https://github.com/openzfs/zfs/issues/14724
+    log "FSx for Lustre is not supported in kernel version #{node['cluster']['kernel_release']} of RHEL #{version}, please update the kernel version" do
       level :warn
     end
   else

cookbooks/aws-parallelcluster-environment/spec/unit/resources/lustre_setup_spec.rb

@@ -171,21 +171,55 @@ def self.setup(chef_run)
     end
   end
 
-  context "on redhat with kernel 4.18.0-425.3.1.el8" do
-    cached(:chef_run) do
-      runner = runner(
-        platform: 'redhat', version: '8',
-        step_into: ['lustre']
-      ) do |node|
-        node.automatic['platform_version'] = "8.2"
-        node.override['cluster']['kernel_release'] = "anything 4.18.0-425.3.1.el8 something"
+  [%w(8.7 4.18.0-425.3.1.el8.x86_64), %w(8.7 4.18.0-425.13.1.el8_7.x86_64)].each do |platform_version, kernel_version|
+    context "on redhat #{platform_version} with kernel #{kernel_version}" do
+      cached(:chef_run) do
+        runner = runner(
+          platform: 'redhat', version: '8',
+          step_into: ['lustre']
+        ) do |node|
+          node.automatic['platform_version'] = platform_version
+          node.override['cluster']['kernel_release'] = kernel_version
+        end
+        Lustre.setup(runner)
+      end
+
+      it 'can not install lustre' do
+        is_expected.to write_log("FSx for Lustre is not supported in kernel version #{kernel_version} of RHEL #{platform_version}, please update the kernel version")
+          .with(level: :warn)
       end
-      Lustre.setup(runner)
     end
+  end
 
-    it 'can not install lustre' do
-      is_expected.to write_log("FSx for Lustre is not supported in kernel version 4.18.0-425.3.1.el8 of RHEL, please update the kernel version")
-        .with(level: :warn)
+  [%w(8.8 4.18.0-425.3.1.el8.x86_64), %w(8.8 4.18.0-425.13.1.el8_7.x86_64)].each do |platform_version, kernel_version|
+    context "on redhat #{platform_version} with kernel #{kernel_version}" do
+      cached(:chef_run) do
+        runner = runner(
+          platform: 'redhat', version: '8',
+          step_into: ['lustre']
+        ) do |node|
+          node.automatic['platform_version'] = platform_version
+          node.override['cluster']['kernel_release'] = kernel_version
+        end
+        Lustre.setup(runner)
+      end
+
+      it 'installs lustre packages from repository and installs kernel module lnet' do
+        is_expected.to create_yum_repository("aws-fsx")
+          .with(baseurl: "https://fsx-lustre-client-repo.s3.amazonaws.com/el/8.8/$basearch")
+          .with(gpgkey: 'https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc')
+          .with(retries: 3)
+          .with(retry_delay: 5)
+
+        is_expected.to run_execute('yum-config-manager_skip_if_unavail')
+          .with(command: "yum-config-manager --setopt=\*.skip_if_unavailable=1 --save")
+
+        is_expected.to install_package(%w(kmod-lustre-client lustre-client dracut))
+          .with(retries: 3)
+          .with(retry_delay: 5)
+
+        is_expected.to install_kernel_module("lnet")
+      end
     end
   end
 

cookbooks/aws-parallelcluster-environment/test/controls/lustre_spec.rb

@@ -1,7 +1,7 @@
 control 'tag:install_lustre_client_installed' do
   title "Verify that lustre client is installed"
   minimal_lustre_client_version = '2.12'
-  if (os_properties.centos? && inspec.os.release.to_f >= 7.5) || os_properties.redhat?
+  if os_properties.centos? && inspec.os.release.to_f >= 7.5
     describe package('kmod-lustre-client') do
       it { should be_installed }
     end
@@ -10,7 +10,7 @@
       it { should be_installed }
     end
 
-    if (os_properties.centos? && inspec.os.release.to_f >= 7.7) || os_properties.redhat?
+    if os_properties.centos? && inspec.os.release.to_f >= 7.7
       describe package('kmod-lustre-client') do
         its('version') { should cmp >= minimal_lustre_client_version }
       end
@@ -27,6 +27,34 @@
     end
   end
 
+  if os_properties.redhat? && inspec.os.release.to_f >= 8.2
+    unless inspec.os.release.to_f == 8.7 && (node['cluster']['kernel_release'].include?("4.18.0-425.3.1.el8") || node['cluster']['kernel_release'].include?("4.18.0-425.13.1.el8_7"))
+      describe package('kmod-lustre-client') do
+        it { should be_installed }
+      end
+
+      describe package('lustre-client') do
+        it { should be_installed }
+      end
+
+      if (os_properties.centos? && inspec.os.release.to_f >= 7.7) || os_properties.redhat?
+        describe package('kmod-lustre-client') do
+          its('version') { should cmp >= minimal_lustre_client_version }
+        end
+
+        describe package('lustre-client') do
+          its('version') { should cmp >= minimal_lustre_client_version }
+        end
+
+        describe yum.repo('aws-fsx') do
+          it { should exist }
+          it { should be_enabled }
+          its('baseurl') { should include 'fsx-lustre-client-repo.s3.amazonaws.com' }
+        end
+      end
+    end
+  end
+
   if os_properties.debian_family?
     describe apt('https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu') do
       it { should exist }
@@ -60,10 +88,12 @@
 control 'tag:install_lustre_lnet_kernel_module_enabled' do
   title "Verify that lnet kernel module is enabled"
   only_if { !os_properties.on_docker? && !os_properties.alinux2? }
-  describe kernel_module("lnet") do
-    it { should be_loaded }
-    it { should_not be_disabled }
-    it { should_not be_blacklisted }
+  unless os_properties.redhat? && inspec.os.release.to_f == 8.7 && (node['cluster']['kernel_release'].include?("4.18.0-425.3.1.el8") || node['cluster']['kernel_release'].include?("4.18.0-425.13.1.el8_7"))
+    describe kernel_module("lnet") do
+      it { should be_loaded }
+      it { should_not be_disabled }
+      it { should_not be_blacklisted }
+    end
   end
 end
 

kitchen.docker.yml

@@ -68,4 +68,4 @@ platforms:
     attributes:
       cluster:
         base_os: rhel8
-        kernel_release: '4.18.0-425.13.1.el8_7.x86_64'
+        kernel_release: '4.18.0-477.13.1.el8_7.x86_64' # Use 477 version to match 8.8 kernel version available on docker