Merge cockroachdb#57587

57587: jobs: invert status predicate to improve query plan r=ajwerner a=ajwerner

The jobs code regularly scans the set of adoptable jobs (generally small) to
find if there are abandoned jobs in need of adoption. In cockroachdb#56864 we added a
predicate to improve these loops to stop touching already terminal jobs. This
was a big improvement in what data is written but not as big of an improvement
in what data was read. When the jobs table is large, the queries, prior to this
patch, performed expensive full table scans. With this patch the query plan is
improved to utilize the index. Were there a check constraint indicating the
allowable values of status, then the optimizer could have created the same
plan. However, there isn't.

Release note (bug fix): Remove system.jobs full table scan which is
expensive in the face of many completed jobs.

Co-authored-by: Andrew Werner <[email protected]>

pkg/jobs/adopt.go

@@ -25,6 +25,14 @@ import (
 	"github.com/cockroachdb/errors"
 )
 
+const claimableStatusTupleString = `(` +
+	`'` + string(StatusRunning) + `', ` +
+	`'` + string(StatusPending) + `', ` +
+	`'` + string(StatusCancelRequested) + `', ` +
+	`'` + string(StatusPauseRequested) + `', ` +
+	`'` + string(StatusReverting) + `'` +
+	`)`
+
 // claimJobs places a claim with the given SessionID to job rows that are
 // available.
 func (r *Registry) claimJobs(ctx context.Context, s sqlliveness.Session) error {
@@ -34,14 +42,11 @@ func (r *Registry) claimJobs(ctx context.Context, s sqlliveness.Session) error {
    UPDATE system.jobs
       SET claim_session_id = $1, claim_instance_id = $2
     WHERE claim_session_id IS NULL
-      AND status NOT IN ($3, $4, $5)
+      AND status IN `+claimableStatusTupleString+`
  ORDER BY created DESC
-    LIMIT $6
+    LIMIT $3
 RETURNING id;`,
-			s.ID().UnsafeBytes(), r.ID(),
-			// Don't touch terminal jobs.
-			StatusSucceeded, StatusCanceled, StatusFailed,
-			maxAdoptionsPerLoop,
+			s.ID().UnsafeBytes(), r.ID(), maxAdoptionsPerLoop,
 		)
 		if err != nil {
 			return errors.Wrap(err, "could not query jobs table")

pkg/jobs/registry.go

@@ -630,11 +630,9 @@ func (r *Registry) Start(
 UPDATE system.jobs
    SET claim_session_id = NULL
  WHERE claim_session_id <> $1
-   AND status NOT IN ($2, $3, $4)
+   AND status IN `+claimableStatusTupleString+`
    AND NOT crdb_internal.sql_liveness_is_alive(claim_session_id)`,
 			s.ID().UnsafeBytes(),
-			// Don't touch terminal jobs.
-			StatusSucceeded, StatusCanceled, StatusFailed,
 		); err != nil {
 			log.Errorf(ctx, "error expiring job sessions: %s", err)
 		}