From d0998e82c3e0fed8476ce799765dfd0152df1176 Mon Sep 17 00:00:00 2001
From: Matus Marhefka <mmarhefk@redhat.com>
Date: Fri, 18 Oct 2024 10:52:37 +0200
Subject: [PATCH] Update nist-validation test to provide more granular results

Scapval tool is failing when we build SCE content by default
in RHEL 9 and RHEL 10 data streams because it doesn't expect
content to use checking systems other than the OVAL and OCIL
(base requirement `SRC-118`). For more details see
https://github.com/ComplianceAsCode/content/pull/12488

We can waive this fail, it shouldn't cause any problems for
3rd party scanners as our content still contains also OVAL
checks. To do so the test has been updated to parse XML results
file generated by the scapval tool.

The test is also updated to work on RHEL 10 where `java-21-openjdk`
is the default as scapval tool has no problem running with this
newer version of java.
---
 conf/waivers/30-permanent              |  7 +++++++
 static-checks/nist-validation/main.fmf |  8 ++------
 static-checks/nist-validation/test.py  | 22 +++++++++++++++-------
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/conf/waivers/30-permanent b/conf/waivers/30-permanent
index def653fb..14ccddf9 100644
--- a/conf/waivers/30-permanent
+++ b/conf/waivers/30-permanent
@@ -84,4 +84,11 @@
 /hardening/.*/ospp/configure_crypto_policy
     rhel.is_centos() and rhel == 9
 
+# scapval waivers
+#
+# Caused by SCE content being built by default, enabled
+# in https://github.com/ComplianceAsCode/content/pull/12488
+/static-checks/nist-validation/ssg-rhel9-ds/SRC-118
+    rhel >= 9
+
 # vim: syntax=python
diff --git a/static-checks/nist-validation/main.fmf b/static-checks/nist-validation/main.fmf
index 2db0c428..e484bede 100644
--- a/static-checks/nist-validation/main.fmf
+++ b/static-checks/nist-validation/main.fmf
@@ -4,14 +4,10 @@ result: custom
 environment+:
     PYTHONPATH: ../..
 duration: 15m
-require+:
-  # we use java-17 specifically here because the NIST tool needs it and does not
-  # work with any newer version
+recommend+:
   - java-17-openjdk
+  - java-21-openjdk
 adjust:
   - when: arch != x86_64
     enabled: false
     because: the test is not architecture-specific, one is enough
-  - when: distro == rhel-10
-    enabled: false
-    because: TODO - RHEL-10 doesn't have Java 17, see requires above
diff --git a/static-checks/nist-validation/test.py b/static-checks/nist-validation/test.py
index 1a958e48..25bc6e5b 100755
--- a/static-checks/nist-validation/test.py
+++ b/static-checks/nist-validation/test.py
@@ -5,6 +5,7 @@
 import zipfile
 import requests
 import subprocess
+import xml.etree.ElementTree as ET
 
 from lib import util, results
 
@@ -16,6 +17,8 @@
 zip.extractall()
 os.chmod('scapval.sh', 0o755)
 
+ns = {'nist': 'http://csrc.nist.gov/ns/decima/results/1.0'}
+
 for datastream in util.iter_datastreams():
     ds_name = datastream.stem
     report_file = f'{ds_name}.report.html'
@@ -27,12 +30,17 @@
         '-valresultfile', result_file,
         '-file', datastream,
     ]
-    proc = util.subprocess_run(cmd, stdout=subprocess.PIPE, check=True, universal_newlines=True)
-    if 'The target is valid' in proc.stdout:
-        results.report('pass', ds_name)
-    elif 'The target is invalid' in proc.stdout:
-        results.report('fail', ds_name, logs=[report_file, result_file])
-    else:
-        raise RuntimeError("SCAPval out has not been correctly parsed")
+    util.subprocess_run(cmd, stdout=subprocess.DEVNULL, check=True)
+    tree = ET.parse(result_file)
+    root = tree.getroot()
+    for elem in root.findall('./nist:results/nist:base-requirement', ns):
+        name = f'{ds_name}/{elem.attrib["id"]}'
+        status = elem.find('./nist:status', ns).text
+        if status in ['NOT_TESTED', 'NOT_APPLICABLE']:
+            continue
+        elif status in ['PASS', 'WARNING', 'INFORMATIONAL']:
+            results.report('pass', name)
+        else:
+            results.report('fail', name, logs=[report_file, result_file])
 
 results.report_and_exit()