From 816288fa7b432b37154e35323132bfeb111c781c Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 29 Mar 2019 00:54:42 -0700 Subject: [PATCH 1/2] scheduler: check daily job limit on each job, not just when choosing AV --- sched/sched_send.cpp | 4 ++++ sched/sched_version.cpp | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/sched/sched_send.cpp b/sched/sched_send.cpp index 97730b8317d..d629c5d1a10 100644 --- a/sched/sched_send.cpp +++ b/sched/sched_send.cpp @@ -192,6 +192,10 @@ const char* find_user_friendly_name(int appid) { return "deprecated application"; } +// Called at start of request handling. +// 1) if there's a global jobs/day limit, enforce it using HAV limit +// 2) if last RPC was yesterday or earlier, clear n_jobs_today for HAV +// static void update_quota(DB_HOST_APP_VERSION& hav) { if (config.daily_result_quota) { if (hav.max_jobs_per_day == 0) { diff --git a/sched/sched_version.cpp b/sched/sched_version.cpp index af975e2309c..e3aab9dee7f 100644 --- a/sched/sched_version.cpp +++ b/sched/sched_version.cpp @@ -130,6 +130,9 @@ inline int scaled_max_jobs_per_day(DB_HOST_APP_VERSION& hav, HOST_USAGE& hu) { return n; } +// are we at the jobs/day limit for this (host, app version)? +// (if so don't use the app version) +// inline bool daily_quota_exceeded(DB_ID_TYPE gavid, HOST_USAGE& hu) { DB_HOST_APP_VERSION* havp = lookup_host_app_version(gavid); if (!havp) return false; @@ -668,6 +671,12 @@ BEST_APP_VERSION* get_app_version( } } + // do this check again since we might have sent a job w/ this AV + // + if (daily_quota_exceeded(bavp->avp->id, bavp->host_usage)) { + break; + } + if (config.debug_version_select) { app_version_desc(*bavp, buf); log_messages.printf(MSG_NORMAL, From 58673969567911ed20380a39d73373115556e257 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 9 Apr 2019 22:58:18 -0700 Subject: [PATCH 2/2] scheduler: fix bug in daily (host, app version) job limit. I was doing the check on each call to get_app_version(). But it turns out this gets called during the job scan, before calling add_result_to_reply(), which is where n_jobs_today gets incremented. Solution:move the check to the loop (in sched_score.cpp) where add_result_reply() is called. Also: fix bad logic in work_needed() where a global check (on job limit in user project prefs) was being done inside a loop over resources. --- sched/sched_keyword.cpp | 6 +++--- sched/sched_score.cpp | 11 +++++++++++ sched/sched_send.cpp | 30 ++++++++++++++++-------------- sched/sched_version.cpp | 12 +++++------- sched/sched_version.h | 1 + 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/sched/sched_keyword.cpp b/sched/sched_keyword.cpp index beb022ff0fd..16cb026f044 100644 --- a/sched/sched_keyword.cpp +++ b/sched/sched_keyword.cpp @@ -54,7 +54,7 @@ double keyword_score(int i) { USER_KEYWORDS& uk = g_request->user_keywords; if (uk.empty()) { if (config.debug_keyword) { - log_messages.printf(MSG_NORMAL, "user has no keywords; returning 0\n"); + log_messages.printf(MSG_NORMAL, "[keyword] user has no keywords; returning 0\n"); } return 0; } @@ -66,7 +66,7 @@ double keyword_score(int i) { WU_RESULT& wr = ssp->wu_results[i]; if (empty(wr.workunit.keywords)) { if (config.debug_keyword) { - log_messages.printf(MSG_NORMAL, "job has no keywords; returning 0\n"); + log_messages.printf(MSG_NORMAL, "[keyword] job has no keywords; returning 0\n"); } return 0; } @@ -74,7 +74,7 @@ double keyword_score(int i) { } double s = keyword_score_aux(uk, jk); if (config.debug_keyword) { - log_messages.printf(MSG_NORMAL, "keyword score: %f\n", s); + log_messages.printf(MSG_NORMAL, "[keyword] keyword score: %f\n", s); } return s; } diff --git a/sched/sched_score.cpp b/sched/sched_score.cpp index 68596a87a96..fc738f407d3 100644 --- a/sched/sched_score.cpp +++ b/sched/sched_score.cpp @@ -272,6 +272,17 @@ void send_work_score_type(int rt) { continue; } + // check daily limit for (host, app version) + // + if (daily_quota_exceeded(job.bavp)) { + if (config.debug_quota) { + log_messages.printf(MSG_NORMAL, + "[quota] daily host/app version quota now exceeded\n" + ); + } + break; + } + if (!sema_locked) { lock_sema(); sema_locked = true; diff --git a/sched/sched_send.cpp b/sched/sched_send.cpp index d629c5d1a10..3573ab337fb 100644 --- a/sched/sched_send.cpp +++ b/sched/sched_send.cpp @@ -780,24 +780,26 @@ bool work_needed(bool locality_sched) { } } - // see if we've reached limits on in-progress jobs + // check user-specified project prefs limit on # of jobs in progress + // + int mj = g_wreq->project_prefs.max_jobs_in_progress; + if (mj && config.max_jobs_in_progress.project_limits.total.njobs >= mj) { + if (config.debug_send) { + log_messages.printf(MSG_NORMAL, + "[send] user project preferences job limit exceeded\n" + ); + } + g_wreq->max_jobs_on_host_exceeded = true; + return false; + } + + // check config.xml limits on in-progress jobs // bool some_type_allowed = false; for (int i=0; iproject_prefs.max_jobs_in_progress; - if (mj) { - if (config.max_jobs_in_progress.project_limits.total.njobs >= mj) { - proj_pref_exceeded = true; - } - } - - if (proj_pref_exceeded || config.max_jobs_in_progress.exceeded(NULL, i)) { + if (config.max_jobs_in_progress.exceeded(NULL, i)) { if (config.debug_quota) { log_messages.printf(MSG_NORMAL, "[quota] reached limit on %s jobs in progress\n", @@ -815,7 +817,7 @@ bool work_needed(bool locality_sched) { if (!some_type_allowed) { if (config.debug_send) { log_messages.printf(MSG_NORMAL, - "[send] in-progress job limit exceeded\n" + "[send] config.xml max_jobs_in_progress limit exceeded\n" ); } g_wreq->max_jobs_on_host_exceeded = true; diff --git a/sched/sched_version.cpp b/sched/sched_version.cpp index e3aab9dee7f..931f343cf7a 100644 --- a/sched/sched_version.cpp +++ b/sched/sched_version.cpp @@ -150,6 +150,10 @@ inline bool daily_quota_exceeded(DB_ID_TYPE gavid, HOST_USAGE& hu) { return false; } +bool daily_quota_exceeded(BEST_APP_VERSION* bavp) { + return daily_quota_exceeded(bavp->avp->id, bavp->host_usage); +} + // scan through client's anonymous apps and pick the best one // CLIENT_APP_VERSION* get_app_version_anonymous( @@ -671,12 +675,6 @@ BEST_APP_VERSION* get_app_version( } } - // do this check again since we might have sent a job w/ this AV - // - if (daily_quota_exceeded(bavp->avp->id, bavp->host_usage)) { - break; - } - if (config.debug_version_select) { app_version_desc(*bavp, buf); log_messages.printf(MSG_NORMAL, @@ -813,7 +811,7 @@ BEST_APP_VERSION* get_app_version( if (daily_quota_exceeded(av.id, host_usage)) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, - "[version] [AV#%lu] daily quota exceeded\n", av.id + "[version] [AV#%lu] daily HAV quota exceeded\n", av.id ); } continue; diff --git a/sched/sched_version.h b/sched/sched_version.h index c8387b1100c..ba9521e79dc 100644 --- a/sched/sched_version.h +++ b/sched/sched_version.h @@ -19,3 +19,4 @@ extern BEST_APP_VERSION* get_app_version( const WORKUNIT&, bool check_req, bool reliable_only ); extern void estimate_flops_anon_platform(); +extern bool daily_quota_exceeded(BEST_APP_VERSION*);