testsuite: Add flux job list annotation tests

Add flux job list annotation tests to t2203-job-manager-dummysched-single
and t2204-job-manager-dummysched-unlimited.

Add list attribute tests to t/t2230-job-info-list.

t/job-manager/sched-helper.sh

@@ -88,3 +88,58 @@ jmgr_check_no_annotations() {
         test -z "$(${JMGR_JOB_LIST} | grep ${id} | cut -f 6-)" && return 0
         return 1
 }
+
+# internal function to get job annotation key value via flux job list
+#
+# arg1 - jobid
+# arg2 - key
+_jinfo_get_annotation() {
+        local id=$1
+        local key=$2
+        local note="$(flux job list -A | grep ${id} | jq .annotations | jq .\"${key}\")"
+        echo $note
+}
+
+# verify if annotation published to job-info
+#
+# function will loop for up to 5 seconds in case annotation update
+# arrives slowly
+#
+# arg1 - jobid
+# arg2 - key in annotation
+# arg3 - value of key in annotation
+#
+# callers should set HAVE_JQ requirement
+jinfo_check_annotation() {
+        local id=$1
+        local key=$2
+        local value="$3"
+        for try in $(seq 1 10); do
+                test "$(_jinfo_get_annotation $id $key)" = "${value}" && return 0
+                sleep 0.5
+        done
+        return 1
+}
+
+# verify that job contains no annotations via job-info
+#
+# arg1 - jobid
+#
+# callers should set HAVE_JQ requirement
+jinfo_check_no_annotations() {
+        local id=$1
+        flux job list -A | grep ${id} | jq -e .annotations > /dev/null && return 1
+        return 0
+}
+
+# verify if job contains specific annotation key through job-info
+#
+# arg1 - jobid
+# arg2 - key in annotation
+#
+# callers should set HAVE_JQ requirement
+jinfo_check_annotation_exists() {
+        local id=$1
+        local key=$2
+        flux job list -A | grep ${id} | jq .annotations | jq -e .\"${key}\" > /dev/null
+}

t/t2203-job-manager-dummysched-single.t

@@ -67,6 +67,14 @@ test_expect_success HAVE_JQ 'job-manager: annotate job id 3 (RRSSS)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+test_expect_success HAVE_JQ 'job-manager: annotate job id 3 in job-info (RRSSS)' '
+        jinfo_check_no_annotations $(cat job1.id) &&
+        jinfo_check_no_annotations $(cat job2.id) &&
+        jinfo_check_annotation $(cat job3.id) "sched.reason_pending" "\"no cores available\""&&
+        jinfo_check_no_annotations $(cat job4.id) &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
 test_expect_success 'job-manager: running job has alloc event' '
         flux job wait-event --timeout=5.0 $(cat job1.id) alloc
 '
@@ -91,6 +99,14 @@ test_expect_success HAVE_JQ 'job-manager: annotate job id 4 (RIRSS)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+test_expect_success HAVE_JQ 'job-manager: annotate job id 4 in job-info (RRSSS)' '
+        jinfo_check_no_annotations $(cat job1.id) &&
+        jinfo_check_no_annotations $(cat job2.id) &&
+        jinfo_check_no_annotations $(cat job3.id) &&
+        jinfo_check_annotation $(cat job4.id) "sched.reason_pending" "\"no cores available\"" &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
 test_expect_success 'job-manager: first S job sent alloc, second S did not' '
         flux job wait-event --timeout=5.0 $(cat job4.id) debug.alloc-request &&
         ! flux job wait-event --timeout=0.1 $(cat job5.id) debug.alloc-request
@@ -136,6 +152,14 @@ test_expect_success HAVE_JQ 'job-manager: no annotations (RIRRR)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+test_expect_success HAVE_JQ 'job-manager: no annotations in job-info (RIRRR)' '
+        jinfo_check_no_annotations $(cat job1.id) &&
+        jinfo_check_no_annotations $(cat job2.id) &&
+        jinfo_check_no_annotations $(cat job3.id) &&
+        jinfo_check_no_annotations $(cat job4.id) &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
 test_expect_success 'job-manager: cancel 1' '
         flux job cancel $(cat job1.id)
 '
@@ -170,6 +194,14 @@ test_expect_success HAVE_JQ 'job-manager: no annotations (IIIII)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+test_expect_success HAVE_JQ 'job-manager: no annotations in job-info (IIIII)' '
+        jinfo_check_no_annotations $(cat job1.id) &&
+        jinfo_check_no_annotations $(cat job2.id) &&
+        jinfo_check_no_annotations $(cat job3.id) &&
+        jinfo_check_no_annotations $(cat job4.id) &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
 test_expect_success 'job-manager: simulate alloc failure' '
         flux module debug --setbit 0x1 sched-dummy &&
         flux job submit --flags=debug basic.json >job6.id &&

t/t2204-job-manager-dummysched-unlimited.t

@@ -70,6 +70,17 @@ test_expect_success HAVE_JQ 'job-manager: annotate job id 3-5 (RRSSS)' '
         jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
 '
 
+test_expect_success HAVE_JQ 'job-manager: annotate job id 3-5 in job-info (RRSSS)' '
+        jinfo_check_annotation $(cat job1.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job2.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job3.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job3.id) "sched.jobs_ahead" "0" &&
+        jinfo_check_annotation $(cat job4.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job4.id) "sched.jobs_ahead" "1" &&
+        jinfo_check_annotation $(cat job5.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job5.id) "sched.jobs_ahead" "2"
+'
+
 test_expect_success 'job-manager: cancel 2' '
         flux job cancel $(cat job2.id)
 '
@@ -94,6 +105,20 @@ test_expect_success HAVE_JQ 'job-manager: annotate job id 4-5 (RIRSS)' '
         jmgr_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
 '
 
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-info
+test_expect_success HAVE_JQ 'job-manager: annotate job id 4-5 in job-info (RIRSS)' '
+        jinfo_check_annotation $(cat job1.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job2.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job3.id) "sched.resource_summary" "\"1core\"" &&
+        test_must_fail jinfo_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jinfo_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jinfo_check_annotation $(cat job4.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jinfo_check_annotation $(cat job5.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job5.id) "sched.jobs_ahead" "1"
+'
+
 test_expect_success 'job-manager: cancel 5' '
         flux job cancel $(cat job5.id)
 '
@@ -117,10 +142,25 @@ test_expect_success HAVE_JQ 'job-manager: annotate job id 4 (RIRSI)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+# compared to above, note that job id #2 retains annotations, it is
+# cached in job-info
+test_expect_success HAVE_JQ 'job-manager: annotate job id 4-5 in job-info (RIRSS)' '
+        jinfo_check_annotation $(cat job1.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job2.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job3.id) "sched.resource_summary" "\"1core\"" &&
+        test_must_fail jinfo_check_annotation_exists $(cat job3.id) "sched.reason_pending" &&
+        test_must_fail jinfo_check_annotation_exists $(cat job3.id) "sched.jobs_ahead" &&
+        jinfo_check_annotation $(cat job4.id) "sched.reason_pending" "\"no cores\"" &&
+        jinfo_check_annotation $(cat job4.id) "sched.jobs_ahead" "0" &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
+# cancel non-running jobs first, to ensure they are not accidentally run when
+# running jobs free resources.
 test_expect_success 'job-manager: cancel all jobs' '
-        flux job cancel $(cat job1.id) &&
+        flux job cancel $(cat job4.id) &&
         flux job cancel $(cat job3.id) &&
-        flux job cancel $(cat job4.id)
+        flux job cancel $(cat job1.id)
 '
 
 test_expect_success 'job-manager: job state IIIII' '
@@ -139,6 +179,15 @@ test_expect_success HAVE_JQ 'job-manager: no annotations (IIIII)' '
         jmgr_check_no_annotations $(cat job5.id)
 '
 
+# compared to above, note that job ids that ran retain annotations
+test_expect_success HAVE_JQ 'job-manager: no annotations in job-info (IIIII)' '
+        jinfo_check_annotation $(cat job1.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job2.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_annotation $(cat job3.id) "sched.resource_summary" "\"1core\"" &&
+        jinfo_check_no_annotations $(cat job4.id) &&
+        jinfo_check_no_annotations $(cat job5.id)
+'
+
 test_expect_success 'job-manager: remove sched-dummy' '
         flux module remove sched-dummy
 '

t/t2230-job-info-list.t

@@ -817,7 +817,8 @@ exception_type \
 exception_severity \
 exception_note \
 result \
-expiration
+expiration \
+annotations
 "
 
 test_expect_success HAVE_JQ 'list request with empty attrs works' '