diff --git a/t/t1001-mf-priority-basic.t b/t/t1001-mf-priority-basic.t index 5bddbd74..ada5d6c9 100755 --- a/t/t1001-mf-priority-basic.t +++ b/t/t1001-mf-priority-basic.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' @@ -96,6 +97,10 @@ test_expect_success 'update plugin with sample test data' ' flux python fake_payload.py ' +# The following set of tests will check the integer priority calculated by the +# priority plugin for two associations. The fair-share of the first association +# is 0.45321 and the fair-share of the second is 0.11345. In addition, job +# priorities get affected if the user sets a custom urgency on their job. test_expect_success 'submit a job with default urgency' ' jobid=$(flux submit --setattr=system.bank=account3 -n1 hostname) && flux job wait-event -f json ${jobid} priority | jq '.context.priority' > job1.test && @@ -156,6 +161,9 @@ test_expect_success 'submit a job using default bank' ' flux cancel ${jobid} ' +# The following two tests ensure job submissions are rejected when a user +# tries to submit a job under a bank they do not belong to or when the format +# of the bank is not valid. test_expect_success 'submit a job using a bank the user does not belong to' ' test_must_fail flux submit --setattr=system.bank=account1 -n1 hostname > bad_bank.out 2>&1 && test_debug "cat bad_bank.out" && @@ -168,6 +176,10 @@ test_expect_success 'reject job when invalid bank format is passed in' ' grep "unable to unpack bank arg" invalid_fmt.out ' +# This set of tests simulates a special case where a portion of the Association +# object that is kept with every job is missing, and therefore, an exception on +# the job is raised with a message explaining that it cannot load the +# information for the job. test_expect_success 'pass special key to user/bank struct to nullify information' ' cat <<-EOF >null_struct.json { diff --git a/t/t1002-mf-priority-small-no-tie.t b/t/t1002-mf-priority-small-no-tie.t index e0fbc977..a2ad88df 100755 --- a/t/t1002-mf-priority-small-no-tie.t +++ b/t/t1002-mf-priority-small-no-tie.t @@ -20,6 +20,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' diff --git a/t/t1003-mf-priority-small-tie.t b/t/t1003-mf-priority-small-tie.t index 0b4baa80..3d06a081 100755 --- a/t/t1003-mf-priority-small-tie.t +++ b/t/t1003-mf-priority-small-tie.t @@ -20,6 +20,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' diff --git a/t/t1004-mf-priority-small-tie-all.t b/t/t1004-mf-priority-small-tie-all.t index 7b5a032a..522957b5 100755 --- a/t/t1004-mf-priority-small-tie-all.t +++ b/t/t1004-mf-priority-small-tie-all.t @@ -20,6 +20,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' diff --git a/t/t1005-max-jobs-limits.t b/t/t1005-max-jobs-limits.t index a4746d9a..f57f95bb 100755 --- a/t/t1005-max-jobs-limits.t +++ b/t/t1005-max-jobs-limits.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' @@ -85,6 +86,13 @@ test_expect_success 'add a default queue and send it to the plugin' ' flux python fake_payload.py ' +# The following set of tests will check that the priority plugin enforces +# accounting limits defined per-association in the flux-accounting database, +# specifically an association's max running jobs and max active jobs limit. +# When the association hits their max running jobs limit, subsequently +# submitted jobs will be held by having an accounting-specific dependency added +# to it. Once they hit their max active jobs limit, subsequently submitted jobs +# will be rejected with an accounting-specific rejection message. test_expect_success 'submit max number of jobs' ' jobid1=$(flux python ${SUBMIT_AS} 5011 sleep 60) && jobid2=$(flux python ${SUBMIT_AS} 5011 sleep 60) @@ -117,7 +125,7 @@ test_expect_success 'a submitted job while at max-running-jobs limit will have a flux cancel ${jobid2} ' -test_expect_success 'submit max number of jobs with a mix of default bank and explicitly set bank' ' +test_expect_success 'submit max number of jobs to the same bank (explicitly and by default)' ' jobid1=$(flux python ${SUBMIT_AS} 5011 sleep 60) && jobid2=$(flux python ${SUBMIT_AS} 5011 --setattr=system.bank=account3 -n 1 sleep 60) ' diff --git a/t/t1008-mf-priority-update.t b/t/t1008-mf-priority-update.t index f45566e8..6be94824 100755 --- a/t/t1008-mf-priority-update.t +++ b/t/t1008-mf-priority-update.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1011-job-archive-interface.t b/t/t1011-job-archive-interface.t index 0cec115b..69710e64 100755 --- a/t/t1011-job-archive-interface.t +++ b/t/t1011-job-archive-interface.t @@ -139,7 +139,7 @@ test_expect_success 'check that job usage and fairshare values get updated' ' ' # if update-usage is called in the same half-life period when no jobs are found -# for a user, their job usage factor should be affected; this test is taken +# for a user, their job usage factor should not be affected; this test is taken # from the set of job-archive interface Python unit tests test_expect_success 'call update-usage in the same half-life period where no jobs are run' ' flux account -p ${DB_PATH} update-usage && diff --git a/t/t1012-mf-priority-load.t b/t/t1012-mf-priority-load.t index 04e307c5..58f5bfbc 100755 --- a/t/t1012-mf-priority-load.t +++ b/t/t1012-mf-priority-load.t @@ -75,6 +75,14 @@ test_expect_success 'create fake_payload.py' ' EOF ' +# The following test simulates the following scenario: a user submits a job to +# a bank they do not have access to with the priority plugin loaded BEFORE it +# is updated with flux-accounting information (i.e the plugin knows nothing +# about which users belong to which bank). The plugin is then updated with the +# flux-accounting information, and while looping through the job.state.priority +# callback for each job, discovers that this submitted job comes from a user +# who does not have access to the bank they submitted this job under. +# Therefore, an exception is raised on the job. test_expect_success 'submitting a job specifying an incorrect bank with no user data results in a job exception' ' jobid0=$(flux submit --setattr=system.bank=account4 -n1 sleep 60) && flux python fake_payload.py && @@ -88,6 +96,11 @@ test_expect_success 'unload and reload mf_priority.so' ' flux jobtap list | grep mf_priority ' +# The following set of tests simulates an association submitting a job while +# the priority plugin is loaded BEFORE it is updated with flux-accounting +# information. The plugin holds the job while it waits to receive accounting +# information, and once it is updated, will annotate the job with the bank name +# and allow it to proceed to run. test_expect_success 'submit sleep 60 jobs with no data update' ' jobid1=$(flux submit -n1 sleep 60) ' @@ -131,6 +144,9 @@ test_expect_success 'unload mf_priority.so' ' flux jobtap remove mf_priority.so ' +# The following set of tests makes sure that if a job is held before the plugin +# is successfully loaded and is updated with flux-accounting information, the +# job can still successfully transition to RUN state. test_expect_success 'submit a job with no plugin loaded' ' jobid4=$(flux submit -n 1 sleep 60) && flux job wait-event -vt 60 ${jobid4} depend diff --git a/t/t1013-mf-priority-queues.t b/t/t1013-mf-priority-queues.t index b00fafc2..37393103 100755 --- a/t/t1013-mf-priority-queues.t +++ b/t/t1013-mf-priority-queues.t @@ -22,6 +22,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1015-mf-priority-urgency.t b/t/t1015-mf-priority-urgency.t index 30108c35..8388d73e 100755 --- a/t/t1015-mf-priority-urgency.t +++ b/t/t1015-mf-priority-urgency.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' diff --git a/t/t1019-mf-priority-info-fetch.t b/t/t1019-mf-priority-info-fetch.t index 67eddc42..6f6dbf37 100755 --- a/t/t1019-mf-priority-info-fetch.t +++ b/t/t1019-mf-priority-info-fetch.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1020-mf-priority-issue262.t b/t/t1020-mf-priority-issue262.t index 767641d3..af311063 100755 --- a/t/t1020-mf-priority-issue262.t +++ b/t/t1020-mf-priority-issue262.t @@ -20,6 +20,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1021-mf-priority-issue332.t b/t/t1021-mf-priority-issue332.t index 0cb6f61e..451ab67e 100755 --- a/t/t1021-mf-priority-issue332.t +++ b/t/t1021-mf-priority-issue332.t @@ -23,6 +23,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1022-mf-priority-issue346.t b/t/t1022-mf-priority-issue346.t index 18bc21cc..886a00dc 100755 --- a/t/t1022-mf-priority-issue346.t +++ b/t/t1022-mf-priority-issue346.t @@ -22,6 +22,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1027-mf-priority-issue376.t b/t/t1027-mf-priority-issue376.t index 28ba34c9..f35ec019 100755 --- a/t/t1027-mf-priority-issue376.t +++ b/t/t1027-mf-priority-issue376.t @@ -21,6 +21,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1028-mf-priority-issue385.t b/t/t1028-mf-priority-issue385.t index e0287e84..d79c0827 100755 --- a/t/t1028-mf-priority-issue385.t +++ b/t/t1028-mf-priority-issue385.t @@ -21,6 +21,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB, start flux-accounting service' ' flux account -p $(pwd)/FluxAccountingTest.db create-db && flux account-service -p ${DB_PATH} -t diff --git a/t/t1029-mf-priority-default-bank.t b/t/t1029-mf-priority-default-bank.t index b10e9b59..7115be87 100755 --- a/t/t1029-mf-priority-default-bank.t +++ b/t/t1029-mf-priority-default-bank.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'load multi-factor priority plugin' ' flux jobtap load -r .priority-default ${MULTI_FACTOR_PRIORITY} ' diff --git a/t/t1030-mf-priority-update-queue.t b/t/t1030-mf-priority-update-queue.t index d0c4d1c1..6049a420 100755 --- a/t/t1030-mf-priority-update-queue.t +++ b/t/t1030-mf-priority-update-queue.t @@ -22,6 +22,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1031-mf-priority-issue406.t b/t/t1031-mf-priority-issue406.t index e96a45ce..2443e616 100755 --- a/t/t1031-mf-priority-issue406.t +++ b/t/t1031-mf-priority-issue406.t @@ -19,6 +19,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1032-mf-priority-update-bank.t b/t/t1032-mf-priority-update-bank.t index 3c00b8fe..33946b5f 100755 --- a/t/t1032-mf-priority-update-bank.t +++ b/t/t1032-mf-priority-update-bank.t @@ -21,6 +21,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db ' diff --git a/t/t1033-mf-priority-update-job.t b/t/t1033-mf-priority-update-job.t index 9687bff2..68d9a1ca 100755 --- a/t/t1033-mf-priority-update-job.t +++ b/t/t1033-mf-priority-update-job.t @@ -22,6 +22,7 @@ test_expect_success 'allow guest access to testexec' ' allow-guests = true EOF ' + test_expect_success 'create flux-accounting DB' ' flux account -p $(pwd)/FluxAccountingTest.db create-db '