From 4b194242181d4c8332197adbd3c58e2b2606fe7c Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Mon, 16 Dec 2024 16:48:44 -0800 Subject: [PATCH] testsuite: cover sched reload with partial alloc Problem: there is no test coverage for reloading the scheduler with partially allocated jobs. Add a test that runs if the scheduler was able to send the hello request with the partial-ok flag. Use the convenience scripts from sharness.d to load/reload the scheduler modules, and load qmanager with synchronization so that tests are not racing with the hello handshake that happens after module loading completes. --- t/t1026-rv1-partial-release.t | 125 ++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 15 deletions(-) diff --git a/t/t1026-rv1-partial-release.t b/t/t1026-rv1-partial-release.t index da315d633..bc5e58fac 100755 --- a/t/t1026-rv1-partial-release.t +++ b/t/t1026-rv1-partial-release.t @@ -30,24 +30,44 @@ hk_wait_for_running () { sleep 0.1 done } - -fluxion_free_cores() { +# Usage: hk_wait_for_allocated_nnodes count +hk_wait_for_allocated_nnodes () { + count=0 + while test $(flux housekeeping list -no {allocated.nnodes}) -ne $1; do + count=$(($count+1)); + test $count -eq 300 && return 1 # max 300 * 0.1s sleep = 30s + sleep 0.1 + done +} +# Usage: fluxion_free ncores|nnodes +fluxion_free () { FLUX_RESOURCE_LIST_RPC=sched.resource-status \ - flux resource list -s free -no {ncores} + flux resource list -s free -no {$1} +} +# Usage: fluxion_allocated ncores|nnodes +fluxion_allocated () { + FLUX_RESOURCE_LIST_RPC=sched.resource-status \ + flux resource list -s allocated -no {$1} } test_expect_success 'load fluxion modules' ' flux module remove -f sched-simple && - flux module load sched-fluxion-resource && - flux module load sched-fluxion-qmanager && + load_resource match-format=rv1_nosched && + load_qmanager_sync && flux resource list && FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list ' + +# Check job manager hello debug message for +partial-ok flag +if flux dmesg | grep +partial-ok; then + test_set_prereq HAVE_PARTIAL_OK +fi + test_expect_success 'run a normal job, resources are free' ' flux run -vvv -xN4 /bin/true && - test_debug "echo free=\$(fluxion_free_cores)" && - test $(fluxion_free_cores) -eq $TOTAL_NCORES + test_debug "echo free=\$(fluxion_free ncores)" && + test $(fluxion_free ncores) -eq $TOTAL_NCORES ' test_expect_success 'run 4 single node jobs, resources are free' ' flux submit -v --cc=1-4 -xN1 --wait /bin/true && @@ -56,8 +76,8 @@ test_expect_success 'run 4 single node jobs, resources are free' ' ' test_expect_success 'run 16 single core jobs, resources are free' ' flux submit -v --cc=1-16 -n1 --wait /bin/true && - test_debug "echo free=\$(fluxion_free_cores)" && - test $(fluxion_free_cores) -eq $TOTAL_NCORES + test_debug "echo free=\$(fluxion_free ncores)" && + test $(fluxion_free ncores) -eq $TOTAL_NCORES ' test_expect_success 'clear dmesg buffer' ' flux dmesg -C @@ -65,8 +85,8 @@ test_expect_success 'clear dmesg buffer' ' test_expect_success 'run a job with unequal core distribution, resources are free' ' flux run -vvv -n7 -l flux getattr rank && test_debug "flux job info $(flux job last) R | jq" && - test_debug "echo free=\$(fluxion_free_cores)" && - test $(fluxion_free_cores) -eq $TOTAL_NCORES + test_debug "echo free=\$(fluxion_free ncores)" && + test $(fluxion_free ncores) -eq $TOTAL_NCORES ' test_expect_success 'attempt to ensure dmesg buffer synchronized' ' flux logger test-sentinel && @@ -96,16 +116,91 @@ test_expect_success 'attempt to ensure dmesg buffer synchronized' ' dmesg_wait test-sentinel ' test_expect_success 'all resources free' ' - test_debug "echo free=\$(fluxion_free_cores)" && - test $(fluxion_free_cores) -eq $TOTAL_NCORES + test_debug "echo free=\$(fluxion_free ncores)" && + test $(fluxion_free ncores) -eq $TOTAL_NCORES ' test_expect_success 'no errors from fluxion' ' flux dmesg -H >log2.out && test_must_fail grep "free RPC failed to remove all resources" log.out ' +test_expect_success HAVE_PARTIAL_OK 'reconfigure housekeeping with sticky node' ' + flux config load <<-EOF + [job-manager.housekeeping] + command = [ + "sh", + "-c", + "test \$(flux getattr rank) -eq 0 && sleep inf; exit 0" + ] + release-after = "0s" + EOF +' +test_expect_success HAVE_PARTIAL_OK 'run a job and wait for node to get stuck' ' + flux run -N4 true && + hk_wait_for_running 1 && + hk_wait_for_allocated_nnodes 1 +' +test_expect_success HAVE_PARTIAL_OK 'fluxion shows 1 node allocated' ' + test $(fluxion_allocated nnodes) -eq 1 +' +test_expect_success HAVE_PARTIAL_OK 'reload fluxion modules' ' + remove_qmanager && + reload_resource match-format=rv1_nosched && + load_qmanager_sync && + flux resource list && + FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list +' +test_expect_success HAVE_PARTIAL_OK 'fluxion still shows 1 node allocated' ' + test $(fluxion_allocated nnodes) -eq 1 +' +test_expect_success HAVE_PARTIAL_OK 'kill housekeeping' ' + flux housekeeping kill --all +' +test_expect_success HAVE_PARTIAL_OK 'fluxion shows 0 nodes allocated' ' + hk_wait_for_running 0 && + test $(fluxion_allocated nnodes) -eq 0 +' +test_expect_success HAVE_PARTIAL_OK 'reload fluxion modules with match-format=rv1' ' + remove_qmanager && + reload_resource match-format=rv1 && + load_qmanager_sync && + flux resource list && + FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list +' +test_expect_success HAVE_PARTIAL_OK 'run a job and wait for node to get stuck' ' + flux run -N4 true + hk_wait_for_running 1 && + hk_wait_for_allocated_nnodes 1 +' +test_expect_success HAVE_PARTIAL_OK 'fluxion shows 1 nodes allocated' ' + test $(fluxion_allocated nnodes) -eq 1 +' +test_expect_success HAVE_PARTIAL_OK 'run a sleep job on one node' ' + flux submit --wait-event=alloc -N1 sleep 3600 +' +test_expect_success HAVE_PARTIAL_OK 'fluxion shows 2 nodes allocated' ' + test $(fluxion_allocated nnodes) -eq 2 +' +test_expect_success HAVE_PARTIAL_OK 'reload fluxion modules with match-format=rv1' ' + remove_qmanager && + reload_resource match-format=rv1 && + load_qmanager_sync && + flux resource list && + FLUX_RESOURCE_LIST_RPC=sched.resource-status flux resource list +' +test_expect_success HAVE_PARTIAL_OK 'fluxion still shows 2 nodes allocated' ' + test $(fluxion_allocated nnodes) -eq 2 +' +test_expect_success HAVE_PARTIAL_OK 'kill housekeeping' ' + flux housekeeping kill --all +' +test_expect_success HAVE_PARTIAL_OK 'fluxion shows 1 node allocated' ' + hk_wait_for_running 0 && + test $(fluxion_allocated nnodes) -eq 1 +' + test_expect_success 'unload fluxion modules' ' - flux module remove sched-fluxion-qmanager && - flux module remove sched-fluxion-resource && + remove_qmanager && + remove_resource && flux module load sched-simple ' test_done