From b19149da41e6908b7541f2f74f3f39fcba00d509 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Tue, 2 Aug 2022 09:09:20 -0400 Subject: [PATCH 1/2] cice.setup: allow command line to override suite options Options chosen on the 'cice.setup' command line (using the '-s' flag) are added to the options defined for each test in the test suite definition file, when running a test suite. This is implemented by appending the options from the test suite (in variable 'sets_tmp') to the options from the command line ('sets_base') in the variable 'sets', which is ultimately (via the variable 'setsx') looped through to apply each option. Since 'sets_tmp' is appended to 'sets_base', options on the command line can't override options from the test suite file, which means one can't e.g. run a test suite with 'kdyn=3' and expect all tests to use this option if any option specified in the test suite set 'kdyn' to some other value. To allow options from the command line to override options from the test suite, reverse the order in which 'sets_tmp' and 'sets_base' are used to define 'sets'. This is in line with the common behaviour of the command line taking precedence. Adjust the documentation accordingly, fixing a typo along the way. --- cice.setup | 2 +- doc/source/user_guide/ug_testing.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cice.setup b/cice.setup index 4994c7ee1..b3d63de0e 100755 --- a/cice.setup +++ b/cice.setup @@ -638,7 +638,7 @@ EOF set bfbcomp_tmp = `echo $line | cut -d' ' -f5` # Append sets from .ts file to the $sets variable - set sets = "$sets_base,$sets_tmp" + set sets = "$sets_tmp,$sets_base" # Create a new bfbcomp_base variable to store bfbcomp passed to cice.setup # Use bfbcomp_base or bfbcomp_tmp diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index b8d42ad6d..a200d0843 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -325,7 +325,7 @@ If a user adds ``--set`` to the suite, all tests in that suite will add that opt ./cice.setup --suite base_suite,decomp_suite --mach wolf --env gnu --testid myid -s debug -The option settings defined in the suite have precendence over the command line +The option settings defined at the command line have precedence over the test suite values if there are conflicts. The predefined test suites are defined under **configuration/scripts/tests** and @@ -474,7 +474,7 @@ Test Suite Examples ./results.csh If there are conflicts between the ``--set`` options in the suite and on the command line, - the suite will take precedence. + the command line options will take precedence. 5) **Multiple test suites from a single command line** From 66ec2d17895a967f29d7449ad908d2c01e641372 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Tue, 2 Aug 2022 09:36:22 -0400 Subject: [PATCH 2/2] cice.setup: name test suite cases less ambiguously In the previous commit, we allowed options from the command line to override options from the test suite definition file. However, test case directories are still named using a sorted list of all active options, both from command line and the suite definition file (variable 'setsarray'). This is nonoptimal since it is not clear from looking at the test directory name which options have precedence in case of conflict. Change the naming logic so that options from the command line are last in the test directory name, in a "last-one-wins" fashion. To do that, let 'setsarray' be defined from the test suite options ('sets_tmp') and add a second loop for the command line options ('sets_base'). Note that we do not check if the same option is mentioned both in the test suite and the command line, in order not to complicate the code further. This also allows greatly simplifying the logic used to adjust 'bfbcomp' test names to include command line options. This part of the code is checking if the options for the test aginst which to compare contain duplicates and 'none', which is unnecessary since these options come directly from the test suite definition file. --- cice.setup | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/cice.setup b/cice.setup index b3d63de0e..6c3fc864b 100755 --- a/cice.setup +++ b/cice.setup @@ -761,7 +761,7 @@ EOF if (${docase} == 0) then set soptions = "" # Create sorted array and remove duplicates and "none" - set setsarray = `echo ${sets} | sed 's/,/ /g' | fmt -1 | sort -u` + set setsarray = `echo ${sets_tmp} | sed 's/,/ /g' | fmt -1 | sort -u` if ("${setsarray}" != "") then foreach field (${setsarray}) if (${field} != "none") then @@ -769,6 +769,15 @@ EOF endif end endif + # Add options from command line, sort and remove duplicates + set soptions_base = "" + set setsarray_base = `echo ${sets_base} | sed 's/,/ /g' | fmt -1 | sort -u` + if ("${setsarray_base}" != "") then + foreach field (${setsarray_base}) + set soptions = ${soptions}"_"${field} + set soptions_base = ${soptions_base}"_"${field} + end + endif # soptions starts with _ set testname_noid = "${machcomp}_${test}_${grid}_${pesx}${soptions}" set testname_base = "${machcomp}_${test}_${grid}_${pesx}${soptions}.${testid}" @@ -777,26 +786,8 @@ EOF if (${dosuite} == 1) then # Add -s flags in cice.setup to bfbcomp name - # Parse bfbcomp test_grid_pes and sets - # Add sets_base and sort unique - # Create fbfbcomp string that should be consistent with base casename - set bfbcomp_regex="\(.*_[0-9x]*\)_\(.*\)" - set bfbcomp_test_grid_pes=`echo ${bfbcomp} | sed "s/${bfbcomp_regex}/\1/"` - set bfbcomp_sets=`echo ${bfbcomp} | sed "s/${bfbcomp_regex}/\2/" | sed 's/_/,/g' ` - set bfbcomp_sets="${bfbcomp_sets},${sets_base}" - set bfbcomp_soptions = "" - # Create sorted array and remove duplicates and "none" - set bfbcomp_setsarray = `echo ${bfbcomp_sets} | sed 's/,/ /g' | fmt -1 | sort -u` - if ("${bfbcomp_setsarray}" != "") then - foreach field (${bfbcomp_setsarray}) - if (${field} != "none") then - set bfbcomp_soptions = ${bfbcomp_soptions}"_"${field} - endif - end - endif - set fbfbcomp = ${spval} if ($bfbcomp != ${spval}) then - set fbfbcomp = ${machcomp}_${bfbcomp_test_grid_pes}${bfbcomp_soptions} + set fbfbcomp = ${machcomp}_${bfbcomp}${soptions_base} endif endif endif