From 9b2229ba966cab78514a5bc8ad2aece58723914a Mon Sep 17 00:00:00 2001 From: Alex Sarkesian Date: Fri, 24 Mar 2023 21:08:40 -0400 Subject: [PATCH 1/5] server: fix flaky drain test under race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While previously `TestDrain` would issue a drain request twice, and expect that after the second drain request there would be no remaining leases, we have seen in some race builds that a lease extension can occur before that second drain, leaving one lease remaining after the second drain request. This can be seen in the following log example: ``` I230325 00:39:18.151604 14728 1@server/drain.go:145 ⋮ [T1,n1] 383 drain request received with doDrain = true, shutdown = false ... I230325 00:39:18.155547 986 kv/kvserver/replica_proposal.go:272 ⋮ [T1,n1,s1,r51/1:‹/Table/5{0-1}›,raft] 385 new range lease repl=(n1,s1):1 seq=1 start=0,0 exp=1679704764.152223164,0 pro=1679704758.152223164,0 following repl=(n1,s1):1 seq=1 start=0,0 exp=1679704746.135729956,0 pro=1679704740.135729956,0 I230325 00:39:18.172450 14728 1@server/drain.go:399 ⋮ [T1,n1] 386 (DEBUG) initiating kvserver node drain I230325 00:39:18.172613 14728 1@kv/kvserver/store.go:1559 ⋮ [T1,drain,n1,s1] 387 (DEBUG) store marked as draining I230325 00:39:18.182123 14728 1@server/drain.go:293 ⋮ [T1,n1] 388 drain remaining: 1 I230325 00:39:18.182249 14728 1@server/drain.go:295 ⋮ [T1,n1] 389 drain details: range lease iterations: 1 I230325 00:39:18.182404 14728 1@server/drain.go:175 ⋮ [T1,n1] 390 drain request completed without server shutdown ``` This change modifies the test to repeatedly issue drain requests until there is no remaining work, allowing the drain to complete upon subsequent requests. Fixes: #86974 Release note: None --- pkg/server/drain_test.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/server/drain_test.go b/pkg/server/drain_test.go index d9fced71ed94..c91833638e9a 100644 --- a/pkg/server/drain_test.go +++ b/pkg/server/drain_test.go @@ -23,7 +23,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/appstatspb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" - "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/grpcutil" @@ -38,7 +37,6 @@ import ( // TestDrain tests the Drain RPC. func TestDrain(t *testing.T) { defer leaktest.AfterTest(t)() - skip.UnderRaceWithIssue(t, 86974, "flaky test") defer log.Scope(t).Close(t) doTestDrain(t) } @@ -64,18 +62,27 @@ func doTestDrain(tt *testing.T) { // Issue another probe. This checks that the server is still running // (i.e. Shutdown: false was effective), the draining status is - // still properly reported, and the server slept once. + // still properly reported, and that the server only slept once (which only + // should occur on the first drain). resp = t.sendProbe() t.assertDraining(resp, true) // probe-only has no remaining. t.assertRemaining(resp, false) t.assertEqual(1, drainSleepCallCount) - // Issue another drain. Verify that the remaining is zero (i.e. complete) - // and that the server did not sleep again. - resp = t.sendDrainNoShutdown() - t.assertDraining(resp, true) - t.assertRemaining(resp, false) + // Repeat drain commands until we verify that there are zero remaining leases + // (i.e. complete). Also validate that the server did not sleep again. + testutils.SucceedsSoon(t, func() error { + resp = t.sendDrainNoShutdown() + if !resp.IsDraining { + return errors.Newf("expected draining") + } + if resp.DrainRemainingIndicator > 0 { + return errors.Newf("still %d remaining, desc: %s", resp.DrainRemainingIndicator, + resp.DrainRemainingDescription) + } + return nil + }) t.assertEqual(1, drainSleepCallCount) // Now issue a drain request without drain but with shutdown. From 5d9b97a67c6bade8bb5a921b7fb881ac44f11579 Mon Sep 17 00:00:00 2001 From: adityamaru Date: Fri, 24 Mar 2023 02:18:43 -0400 Subject: [PATCH 2/5] jobs,*: stop writing payload and progress to system.jobs This change introduces a cluster version after which the payload and progress of a job will not be written to the system.jobs table. This will ensure that the system.job_info table is the single, source of truth for these two pieces of information. This cluster version has an associated upgrade that schema changes the `payload` column of the `system.jobs` table to be nullable, thereby allowing us to stop writing to it. This upgrade step is necessary for a future patch where we will drop the payload and progress columns. Without this intermediate upgrade step the `ALTER TABLE ... DROP COLUMN` upgrade job will attempt to write to dropped columns as part of its execution thereby failing to run the upgrade. Informs: #97762 Release note: None --- .../settings/settings-for-tenants.txt | 2 +- docs/generated/settings/settings.html | 2 +- pkg/ccl/backupccl/backup_test.go | 4 +- .../replication_random_client_test.go | 2 +- .../streamingest/testdata/alter_tenant | 2 +- pkg/cli/testdata/declarative-rules/deprules | 2 +- pkg/cli/testdata/declarative-rules/oprules | 2 +- pkg/cli/zip_test.go | 7 +-- pkg/clusterversion/cockroach_versions.go | 8 ++++ pkg/cmd/roachtest/tests/backup.go | 5 +- .../roachtest/tests/mixed_version_backup.go | 4 +- pkg/jobs/delegate_control_test.go | 1 + pkg/jobs/jobs_test.go | 16 ++++--- pkg/jobs/lease_test.go | 4 +- pkg/jobs/registry.go | 48 +++++++++++++++---- pkg/jobs/registry_external_test.go | 25 ++-------- pkg/jobs/registry_test.go | 35 +++++++++----- pkg/jobs/update.go | 44 ++++++++++------- pkg/jobs/update_test.go | 29 ++++++----- pkg/jobs/utils.go | 2 +- pkg/kv/kvserver/BUILD.bazel | 1 + pkg/kv/kvserver/client_tenant_test.go | 9 +++- pkg/server/admin_test.go | 8 ++-- .../spanconfigmanager/manager_test.go | 12 ++++- pkg/sql/catalog/bootstrap/testdata/testdata | 8 ++-- pkg/sql/catalog/systemschema/system.go | 4 +- .../systemschema_test/testdata/bootstrap | 4 +- pkg/sql/crdb_internal_test.go | 34 ++++++------- pkg/sql/delegate/job_control.go | 2 +- .../testdata/logic_test/crdb_internal_catalog | 2 +- .../testdata/logic_test/create_index | 14 +++--- .../testdata/logic_test/information_schema | 2 - pkg/sql/logictest/testdata/logic_test/system | 2 +- .../tests/local-mixed-22.2-23.1/BUILD.bazel | 2 +- .../local-mixed-22.2-23.1/generated_test.go | 7 --- pkg/sql/row/expr_walker_test.go | 2 +- pkg/sql/schema_changer_test.go | 2 +- pkg/sql/schemachanger/schemachanger_test.go | 2 +- pkg/sql/ttl/ttljob/ttljob_test.go | 10 ++-- pkg/testutils/jobutils/jobs_verification.go | 2 +- pkg/upgrade/upgrademanager/manager.go | 4 +- pkg/upgrade/upgrades/system_job_info.go | 15 ++++++ pkg/upgrade/upgrades/upgrades.go | 6 +++ 43 files changed, 233 insertions(+), 165 deletions(-) diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index c54730fd8deb..84015734387e 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -289,4 +289,4 @@ trace.opentelemetry.collector string address of an OpenTelemetry trace collecto trace.snapshot.rate duration 0s if non-zero, interval at which background trace snapshots are captured trace.span_registry.enabled boolean true if set, ongoing traces can be seen at https:///#/debug/tracez trace.zipkin.collector string the address of a Zipkin instance to receive traces, as :. If no port is specified, 9411 will be used. -version version 1000023.1-2 set the active cluster version in the format '.' +version version 1000023.1-4 set the active cluster version in the format '.' diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index b2d6a68d2e64..8447398b2ee5 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -241,6 +241,6 @@
trace.snapshot.rate
duration0sif non-zero, interval at which background trace snapshots are captured
trace.span_registry.enabled
booleantrueif set, ongoing traces can be seen at https://<ui>/#/debug/tracez
trace.zipkin.collector
stringthe address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used. -
version
version1000023.1-2set the active cluster version in the format '<major>.<minor>' +
version
version1000023.1-4set the active cluster version in the format '<major>.<minor>' diff --git a/pkg/ccl/backupccl/backup_test.go b/pkg/ccl/backupccl/backup_test.go index 996bc434da1d..081e3e5599ee 100644 --- a/pkg/ccl/backupccl/backup_test.go +++ b/pkg/ccl/backupccl/backup_test.go @@ -1755,8 +1755,8 @@ func createAndWaitForJob( var jobID jobspb.JobID db.QueryRow( - t, `INSERT INTO system.jobs (created, status, payload, progress) VALUES ($1, $2, $3, $4) RETURNING id`, - timeutil.FromUnixMicros(now), jobs.StatusRunning, payload, progressBytes, + t, `INSERT INTO system.jobs (created, status) VALUES ($1, $2) RETURNING id`, + timeutil.FromUnixMicros(now), jobs.StatusRunning, ).Scan(&jobID) db.Exec( t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, jobID, jobs.GetLegacyPayloadKey(), payload, diff --git a/pkg/ccl/streamingccl/streamingest/replication_random_client_test.go b/pkg/ccl/streamingccl/streamingest/replication_random_client_test.go index bb89758bdf0a..39ecb91d7cc4 100644 --- a/pkg/ccl/streamingccl/streamingest/replication_random_client_test.go +++ b/pkg/ccl/streamingccl/streamingest/replication_random_client_test.go @@ -50,7 +50,7 @@ import ( func getHighWaterMark(ingestionJobID int, sqlDB *gosql.DB) (*hlc.Timestamp, error) { var progressBytes []byte if err := sqlDB.QueryRow( - `SELECT progress FROM system.jobs WHERE id = $1`, ingestionJobID, + `SELECT progress FROM crdb_internal.system_jobs WHERE id = $1`, ingestionJobID, ).Scan(&progressBytes); err != nil { return nil, err } diff --git a/pkg/ccl/streamingccl/streamingest/testdata/alter_tenant b/pkg/ccl/streamingccl/streamingest/testdata/alter_tenant index 0ed9ec656963..5eb7e95e7b81 100644 --- a/pkg/ccl/streamingccl/streamingest/testdata/alter_tenant +++ b/pkg/ccl/streamingccl/streamingest/testdata/alter_tenant @@ -10,7 +10,7 @@ ALTER TENANT "destination" SET REPLICATION RETENTION = '42s' query-sql as=destination-system SELECT crdb_internal.pb_to_json('payload', payload)->'streamIngestion'->'replicationTtlSeconds' as retention_ttl_seconds -FROM system.jobs +FROM crdb_internal.system_jobs WHERE id = (SELECT replication_job_id FROM [SHOW TENANT "destination" WITH REPLICATION STATUS]) ---- 42 diff --git a/pkg/cli/testdata/declarative-rules/deprules b/pkg/cli/testdata/declarative-rules/deprules index d8965be3a265..08f0d741899a 100644 --- a/pkg/cli/testdata/declarative-rules/deprules +++ b/pkg/cli/testdata/declarative-rules/deprules @@ -1,6 +1,6 @@ dep ---- -debug declarative-print-rules 1000023.1-2 dep +debug declarative-print-rules 1000023.1-4 dep deprules ---- - name: 'CheckConstraint transitions to ABSENT uphold 2-version invariant: PUBLIC->VALIDATED' diff --git a/pkg/cli/testdata/declarative-rules/oprules b/pkg/cli/testdata/declarative-rules/oprules index 3e188f0d5916..33829ab8affa 100644 --- a/pkg/cli/testdata/declarative-rules/oprules +++ b/pkg/cli/testdata/declarative-rules/oprules @@ -1,6 +1,6 @@ op ---- -debug declarative-print-rules 1000023.1-2 op +debug declarative-print-rules 1000023.1-4 op rules ---- [] diff --git a/pkg/cli/zip_test.go b/pkg/cli/zip_test.go index 01fb31ff024f..d2e2b1a50247 100644 --- a/pkg/cli/zip_test.go +++ b/pkg/cli/zip_test.go @@ -28,7 +28,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/kv/kvpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/storepool" @@ -591,13 +590,9 @@ func TestToHex(t *testing.T) { } // Stores index and type of marshaled messages in the table row. // Negative indices work from the end - this is needed because parsing the - // fields is not alway s precise as there can be spaces in the fields but the + // fields is not always precise as there can be spaces in the fields but the // hex fields are always in the end of the row and they don't contain spaces. hexFiles := map[string][]hexField{ - "debug/system.jobs.txt": { - {idx: -2, msg: &jobspb.Payload{}}, - {idx: -1, msg: &jobspb.Progress{}}, - }, "debug/system.descriptor.txt": { {idx: 2, msg: &descpb.Descriptor{}}, }, diff --git a/pkg/clusterversion/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go index 28cf4526c7f7..c71285fa82c0 100644 --- a/pkg/clusterversion/cockroach_versions.go +++ b/pkg/clusterversion/cockroach_versions.go @@ -517,6 +517,10 @@ const ( // the process of upgrading from previous supported releases to 23.2. V23_2Start + // V23_2StopWritingPayloadAndProgressToSystemJobs is the version where the + // payload and progress columns are no longer written to system.jobs. + V23_2StopWritingPayloadAndProgressToSystemJobs + // ************************************************* // Step (1): Add new versions here. // Do not add new versions to a patch release. @@ -898,6 +902,10 @@ var rawVersionsSingleton = keyedVersions{ Key: V23_2Start, Version: roachpb.Version{Major: 23, Minor: 1, Internal: 2}, }, + { + Key: V23_2StopWritingPayloadAndProgressToSystemJobs, + Version: roachpb.Version{Major: 23, Minor: 1, Internal: 4}, + }, // ************************************************* // Step (2): Add new versions here. diff --git a/pkg/cmd/roachtest/tests/backup.go b/pkg/cmd/roachtest/tests/backup.go index 228a07fa8a06..73dadf31b9a5 100644 --- a/pkg/cmd/roachtest/tests/backup.go +++ b/pkg/cmd/roachtest/tests/backup.go @@ -37,6 +37,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/testutils" + "github.com/cockroachdb/cockroach/pkg/testutils/jobutils" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/retry" @@ -942,8 +943,8 @@ revert=2'`) var status string var payloadBytes, progressBytes []byte require.NoError(t, conn.QueryRowContext( - ctx, `SELECT status, progress, payload FROM system.jobs WHERE id = $1`, jobID). - Scan(&status, &progressBytes, &payloadBytes)) + ctx, jobutils.InternalSystemJobsBaseQuery, jobID). + Scan(&status, &payloadBytes, &progressBytes)) if jobs.Status(status) == jobs.StatusFailed { var payload jobspb.Payload require.NoError(t, protoutil.Unmarshal(payloadBytes, &payload)) diff --git a/pkg/cmd/roachtest/tests/mixed_version_backup.go b/pkg/cmd/roachtest/tests/mixed_version_backup.go index 1b3823b0bdc0..bf5395179fe2 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_backup.go +++ b/pkg/cmd/roachtest/tests/mixed_version_backup.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/testutils" + "github.com/cockroachdb/cockroach/pkg/testutils/jobutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/retry" @@ -300,7 +301,8 @@ func (mvb *mixedVersionBackup) waitForJobSuccess( for r := retry.StartWithCtx(ctx, backupCompletionRetryOptions); r.Next(); { var status string var payloadBytes []byte - err := db.QueryRow(`SELECT status, payload FROM system.jobs WHERE id = $1`, jobID).Scan(&status, &payloadBytes) + err := db.QueryRow(fmt.Sprintf(`SELECT status, payload FROM (%s)`, + jobutils.InternalSystemJobsBaseQuery), jobID).Scan(&status, &payloadBytes) if err != nil { lastErr = fmt.Errorf("error reading (status, payload) for job %d: %w", jobID, err) l.Printf("%v", lastErr) diff --git a/pkg/jobs/delegate_control_test.go b/pkg/jobs/delegate_control_test.go index 7f5021c71545..b06cff11a92a 100644 --- a/pkg/jobs/delegate_control_test.go +++ b/pkg/jobs/delegate_control_test.go @@ -463,6 +463,7 @@ func TestJobControlByType(t *testing.T) { // Clear the system.jobs table for the next test run. th.sqlDB.Exec(t, fmt.Sprintf("DELETE FROM system.jobs WHERE id IN (%s)", jobIdsClause)) + th.sqlDB.Exec(t, fmt.Sprintf("DELETE FROM system.job_info WHERE job_id IN (%s)", jobIdsClause)) }) } } diff --git a/pkg/jobs/jobs_test.go b/pkg/jobs/jobs_test.go index 24e30063092a..7f4067d35700 100644 --- a/pkg/jobs/jobs_test.go +++ b/pkg/jobs/jobs_test.go @@ -50,6 +50,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sqlliveness" "github.com/cockroachdb/cockroach/pkg/sql/tests" "github.com/cockroachdb/cockroach/pkg/testutils" + "github.com/cockroachdb/cockroach/pkg/testutils/jobutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -91,7 +92,7 @@ func (expected *expectation) verify(id jobspb.JobID, expectedStatus jobs.Status) var payloadBytes []byte var progressBytes []byte if err := expected.DB.QueryRow( - `SELECT status, created, payload, progress FROM system.jobs WHERE id = $1`, id, + `SELECT status, created, payload, progress FROM crdb_internal.system_jobs WHERE id = $1`, id, ).Scan( &statusString, &created, &payloadBytes, &progressBytes, ); err != nil { @@ -1970,8 +1971,8 @@ func TestShowJobs(t *testing.T) { t.Fatal(err) } sqlDB.Exec(t, - `INSERT INTO system.jobs (id, status, created, payload, progress, claim_session_id, claim_instance_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - in.id, in.status, in.created, inPayload, inProgress, session.ID().UnsafeBytes(), instanceID, + `INSERT INTO system.jobs (id, status, created, claim_session_id, claim_instance_id) VALUES ($1, $2, $3, $4, $5)`, + in.id, in.status, in.created, session.ID().UnsafeBytes(), instanceID, ) sqlDB.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, in.id, jobs.GetLegacyPayloadKey(), inPayload) sqlDB.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, in.id, jobs.GetLegacyProgressKey(), inProgress) @@ -2181,7 +2182,7 @@ SELECT id, payload, progress FROM "".crdb_internal.system_jobs ORDER BY id DESC // Create the second row with a corrupted progress field. if _, err := sqlDB.Exec(` - INSERT INTO system.jobs(id, status, payload, progress) SELECT id+2, status, payload, '\xaaaa'::BYTES FROM system.jobs WHERE id = $1; + INSERT INTO system.jobs(id, status) SELECT id+2, status FROM system.jobs WHERE id = $1; `, jobID); err != nil { t.Fatal(err) } @@ -2200,7 +2201,7 @@ SELECT id, payload, progress FROM "".crdb_internal.system_jobs ORDER BY id DESC // Test what happens with a NULL progress field (which is a valid value). if _, err := sqlDB.Exec(` - INSERT INTO system.jobs(id, status, payload, progress) SELECT id+4, status, payload, NULL::BYTES FROM system.jobs WHERE id = $1; + INSERT INTO system.jobs(id, status) SELECT id+4, status FROM system.jobs WHERE id = $1; `, jobID); err != nil { t.Fatal(err) } @@ -3059,7 +3060,8 @@ func TestMetrics(t *testing.T) { var payloadBytes []byte var payload jobspb.Payload var status string - tdb.QueryRow(t, fmt.Sprintf("SELECT status, payload FROM system.jobs where id = %d", jobID)).Scan( + tdb.QueryRow(t, fmt.Sprintf("SELECT status, payload FROM (%s)", + jobutils.InternalSystemJobsBaseQuery), jobID).Scan( &status, &payloadBytes) require.Equal(t, "paused", status) require.NoError(t, protoutil.Unmarshal(payloadBytes, &payload)) @@ -3344,7 +3346,7 @@ func TestPauseReason(t *testing.T) { var payloadBytes []byte var payload jobspb.Payload var status string - tdb.QueryRow(t, "SELECT status, payload FROM system.jobs where id = $1", jobID).Scan( + tdb.QueryRow(t, "SELECT status, payload FROM crdb_internal.system_jobs where id = $1", jobID).Scan( &status, &payloadBytes) require.NoError(t, protoutil.Unmarshal(payloadBytes, &payload)) diff --git a/pkg/jobs/lease_test.go b/pkg/jobs/lease_test.go index ef1ab62d6188..7a1929c51d22 100644 --- a/pkg/jobs/lease_test.go +++ b/pkg/jobs/lease_test.go @@ -50,8 +50,8 @@ func TestJobsTableClaimFamily(t *testing.T) { now := timeutil.Now() _ = sqlDB.Query(t, ` -INSERT INTO system.jobs (id, status, payload, claim_session_id, claim_instance_id, num_runs, last_run) -VALUES (1, 'running', '@!%$%45', 'foo', 101, 100, $1)`, now) +INSERT INTO system.jobs (id, status, claim_session_id, claim_instance_id, num_runs, last_run) +VALUES (1, 'running', 'foo', 101, 100, $1)`, now) var status, sessionID string var instanceID, numRuns int64 var lastRun time.Time diff --git a/pkg/jobs/registry.go b/pkg/jobs/registry.go index bec1610a476a..5fb9e4fa30b1 100644 --- a/pkg/jobs/registry.go +++ b/pkg/jobs/registry.go @@ -464,9 +464,6 @@ func batchJobInsertStmt( jobs []*Job, modifiedMicros int64, ) (string, []interface{}, []jobspb.JobID, error) { - instanceID := r.ID() - columns := []string{`id`, `created`, `status`, `payload`, `progress`, `claim_session_id`, `claim_instance_id`, `job_type`} - numColumns := len(columns) marshalPanic := func(m protoutil.Message) []byte { data, err := protoutil.Marshal(m) if err != nil { @@ -479,7 +476,8 @@ func batchJobInsertStmt( if err != nil { return "", nil, nil, errors.NewAssertionErrorWithWrappedErrf(err, "failed to make timestamp for creation of job") } - + instanceID := r.ID() + columns := []string{`id`, `created`, `status`, `payload`, `progress`, `claim_session_id`, `claim_instance_id`, `job_type`} valueFns := map[string]func(*Job) (interface{}, error){ `id`: func(job *Job) (interface{}, error) { return job.ID(), nil }, `created`: func(job *Job) (interface{}, error) { return created, nil }, @@ -501,6 +499,24 @@ func batchJobInsertStmt( }, } + // TODO(adityamaru: Remove this once we are outside the compatability + // window for 22.2. + if r.settings.Version.IsActive(ctx, clusterversion.V23_2StopWritingPayloadAndProgressToSystemJobs) { + columns = []string{`id`, `created`, `status`, `claim_session_id`, `claim_instance_id`, `job_type`} + valueFns = map[string]func(*Job) (interface{}, error){ + `id`: func(job *Job) (interface{}, error) { return job.ID(), nil }, + `created`: func(job *Job) (interface{}, error) { return created, nil }, + `status`: func(job *Job) (interface{}, error) { return StatusRunning, nil }, + `claim_session_id`: func(job *Job) (interface{}, error) { return sessionID.UnsafeBytes(), nil }, + `claim_instance_id`: func(job *Job) (interface{}, error) { return instanceID, nil }, + `job_type`: func(job *Job) (interface{}, error) { + payload := job.Payload() + return payload.Type().String(), nil + }, + } + } + numColumns := len(columns) + // TODO(jayant): remove this version gate in 24.1 // To run the upgrade below, migration and schema change jobs will need to be // created using the old schema, which does not have the job_type column. @@ -595,9 +611,15 @@ func (r *Registry) CreateJobWithTxn( if err != nil { return errors.NewAssertionErrorWithWrappedErrf(err, "failed to construct job created timestamp") } - cols := [...]string{"id", "created", "status", "payload", "progress", "claim_session_id", "claim_instance_id", "job_type"} - const totalNumCols = len(cols) - vals := [totalNumCols]interface{}{jobID, created, StatusRunning, payloadBytes, progressBytes, s.ID().UnsafeBytes(), r.ID(), jobType.String()} + + cols := []string{"id", "created", "status", "payload", "progress", "claim_session_id", "claim_instance_id", "job_type"} + vals := []interface{}{jobID, created, StatusRunning, payloadBytes, progressBytes, s.ID().UnsafeBytes(), r.ID(), jobType.String()} + log.Infof(ctx, "active version is %s", r.settings.Version.ActiveVersion(ctx)) + if r.settings.Version.IsActive(ctx, clusterversion.V23_2StopWritingPayloadAndProgressToSystemJobs) { + cols = []string{"id", "created", "status", "claim_session_id", "claim_instance_id", "job_type"} + vals = []interface{}{jobID, created, StatusRunning, s.ID().UnsafeBytes(), r.ID(), jobType.String()} + } + totalNumCols := len(cols) numCols := totalNumCols placeholders := func() string { var p strings.Builder @@ -711,12 +733,18 @@ func (r *Registry) CreateAdoptableJobWithTxn( typ := j.mu.payload.Type().String() nCols := 7 - cols := [7]string{"id", "status", "payload", "progress", "created_by_type", "created_by_id", "job_type"} - placeholders := [7]string{"$1", "$2", "$3", "$4", "$5", "$6", "$7"} - values := [7]interface{}{jobID, StatusRunning, payloadBytes, progressBytes, createdByType, createdByID, typ} + cols := []string{"id", "status", "payload", "progress", "created_by_type", "created_by_id", "job_type"} + placeholders := []string{"$1", "$2", "$3", "$4", "$5", "$6", "$7"} + values := []interface{}{jobID, StatusRunning, payloadBytes, progressBytes, createdByType, createdByID, typ} if !r.settings.Version.IsActive(ctx, clusterversion.V23_1AddTypeColumnToJobsTable) { nCols -= 1 } + if r.settings.Version.IsActive(ctx, clusterversion.V23_2StopWritingPayloadAndProgressToSystemJobs) { + cols = []string{"id", "status", "created_by_type", "created_by_id", "job_type"} + placeholders = []string{"$1", "$2", "$3", "$4", "$5"} + values = []interface{}{jobID, StatusRunning, createdByType, createdByID, typ} + nCols = 5 + } // Insert the job row, but do not set a `claim_session_id`. By not // setting the claim, the job can be adopted by any node and will // be adopted by the node which next runs the adoption loop. diff --git a/pkg/jobs/registry_external_test.go b/pkg/jobs/registry_external_test.go index bbc4d630527d..8bf561d6a09e 100644 --- a/pkg/jobs/registry_external_test.go +++ b/pkg/jobs/registry_external_test.go @@ -40,7 +40,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" - "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" @@ -94,31 +93,15 @@ func TestExpiringSessionsAndClaimJobsDoesNotTouchTerminalJobs(t *testing.T) { s, sqlDB, _ := serverutils.StartServer(t, args) defer s.Stopper().Stop(ctx) - payload, err := protoutil.Marshal(&jobspb.Payload{ - Details: jobspb.WrapPayloadDetails(jobspb.BackupDetails{}), - }) - if err != nil { - t.Fatal(err) - } - - progress, err := protoutil.Marshal(&jobspb.Progress{ - Details: jobspb.WrapProgressDetails(jobspb.BackupProgress{}), - }) - if err != nil { - t.Fatal(err) - } - tdb := sqlutils.MakeSQLRunner(sqlDB) const insertQuery = ` INSERT INTO system.jobs ( status, - payload, - progress, claim_session_id, claim_instance_id ) - VALUES ($1, $2, $3, $4, $5) + VALUES ($1, $2, $3) RETURNING id; ` // Disallow clean up of claimed jobs @@ -128,12 +111,10 @@ RETURNING id; terminalClaims := make([][]byte, len(terminalStatuses)) for i, s := range terminalStatuses { terminalClaims[i] = uuid.MakeV4().GetBytes() // bogus claim - tdb.QueryRow(t, insertQuery, s, payload, progress, terminalClaims[i], 42). - Scan(&terminalIDs[i]) + tdb.QueryRow(t, insertQuery, s, terminalClaims[i], 42).Scan(&terminalIDs[i]) } var nonTerminalID jobspb.JobID - tdb.QueryRow(t, insertQuery, jobs.StatusRunning, payload, progress, uuid.MakeV4().GetBytes(), 42). - Scan(&nonTerminalID) + tdb.QueryRow(t, insertQuery, jobs.StatusRunning, uuid.MakeV4().GetBytes(), 42).Scan(&nonTerminalID) checkClaimEqual := func(id jobspb.JobID, exp []byte) error { const getClaimQuery = `SELECT claim_session_id FROM system.jobs WHERE id = $1` diff --git a/pkg/jobs/registry_test.go b/pkg/jobs/registry_test.go index 2782cfb79f25..847bdcff7b6c 100644 --- a/pkg/jobs/registry_test.go +++ b/pkg/jobs/registry_test.go @@ -202,8 +202,7 @@ INSERT INTO t."%s" VALUES('a', 'foo'); var id jobspb.JobID db.QueryRow(t, - `INSERT INTO system.jobs (status, payload, progress, created) VALUES ($1, $2, $3, $4) RETURNING id`, - status, payload, progress, created).Scan(&id) + `INSERT INTO system.jobs (status, created) VALUES ($1, $2) RETURNING id`, status, created).Scan(&id) db.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, id, GetLegacyPayloadKey(), payload) db.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, id, GetLegacyProgressKey(), progress) return strconv.Itoa(int(id)) @@ -296,8 +295,8 @@ func TestRegistryGCPagination(t *testing.T) { require.NoError(t, err) var jobID jobspb.JobID db.QueryRow(t, - `INSERT INTO system.jobs (status, created, payload) VALUES ($1, $2, $3) RETURNING id`, - StatusCanceled, timeutil.Now().Add(-time.Hour), payload).Scan(&jobID) + `INSERT INTO system.jobs (status, created) VALUES ($1, $2) RETURNING id`, + StatusCanceled, timeutil.Now().Add(-time.Hour)).Scan(&jobID) db.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, jobID, GetLegacyPayloadKey(), payload) } @@ -385,23 +384,33 @@ func TestCreateJobWritesToJobInfo(t *testing.T) { runTests := func(t *testing.T, createdJob *Job) { t.Run("verify against system.jobs", func(t *testing.T) { require.NoError(t, ief.Txn(ctx, func(ctx context.Context, txn isql.Txn) error { - progressQuery := `SELECT count(*) FROM system.jobs AS a LEFT JOIN system.job_info AS b ON a.progress = b.value WHERE b.job_id IS NULL;` + countSystemJobs := `SELECT count(*) FROM system.jobs` row, err := txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), - sessiondata.NodeUserSessionDataOverride, progressQuery) + sessiondata.NodeUserSessionDataOverride, countSystemJobs) if err != nil { return err } - count := tree.MustBeDInt(row[0]) - require.Equal(t, 0, int(count)) + jobsCount := tree.MustBeDInt(row[0]) - payloadQuery := `SELECT count(*) FROM system.jobs AS a LEFT JOIN system.job_info AS b ON a.payload = b.value WHERE b.job_id IS NULL;` + countSystemJobInfo := `SELECT count(*) FROM system.job_info;` row, err = txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), - sessiondata.NodeUserSessionDataOverride, payloadQuery) + sessiondata.NodeUserSessionDataOverride, countSystemJobInfo) if err != nil { return err } - count = tree.MustBeDInt(row[0]) - require.Equal(t, 0, int(count)) + jobInfoCount := tree.MustBeDInt(row[0]) + require.Equal(t, jobsCount*2, jobInfoCount) + + // Ensure no progress and payload is written to system.jobs. + nullPayloadAndProgress := `SELECT count(*) FROM system.jobs WHERE progress IS NOT NULL OR payload IS NOT NULL;` + row, err = txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), + sessiondata.NodeUserSessionDataOverride, nullPayloadAndProgress) + if err != nil { + return err + } + nullProgressAndPayload := tree.MustBeDInt(row[0]) + require.Equal(t, 0, int(nullProgressAndPayload)) + return nil })) }) @@ -1346,7 +1355,7 @@ func TestDisablingJobAdoptionClearsClaimSessionID(t *testing.T) { // Insert a running job with a `claim_session_id` equal to our overridden test // session. tdb.Exec(t, - "INSERT INTO system.jobs (id, status, created, payload, claim_session_id) values ($1, $2, $3, 'test'::bytes, $4)", + "INSERT INTO system.jobs (id, status, created, claim_session_id) values ($1, $2, $3, $4)", 1, StatusRunning, timeutil.Now(), session.ID(), ) diff --git a/pkg/jobs/update.go b/pkg/jobs/update.go index 352e906d4539..768089e1a5a5 100644 --- a/pkg/jobs/update.go +++ b/pkg/jobs/update.go @@ -207,7 +207,6 @@ func (u Updater) update(ctx context.Context, useReadLock bool, updateFn UpdateFn if err != nil { return err } - addSetter("payload", payloadBytes) } var progressBytes []byte @@ -219,7 +218,16 @@ func (u Updater) update(ctx context.Context, useReadLock bool, updateFn UpdateFn if err != nil { return err } - addSetter("progress", progressBytes) + } + + if !u.j.registry.settings.Version.IsActive(ctx, clusterversion.V23_2StopWritingPayloadAndProgressToSystemJobs) { + if payloadBytes != nil { + addSetter("payload", payloadBytes) + } + + if progressBytes != nil { + addSetter("progress", progressBytes) + } } if ju.md.RunStats != nil { @@ -228,22 +236,24 @@ func (u Updater) update(ctx context.Context, useReadLock bool, updateFn UpdateFn addSetter("num_runs", ju.md.RunStats.NumRuns) } - updateStmt := fmt.Sprintf( - "UPDATE system.jobs SET %s WHERE id = $1", - strings.Join(setters, ", "), - ) - n, err := u.txn.ExecEx( - ctx, "job-update", u.txn.KV(), - sessiondata.InternalExecutorOverride{User: username.NodeUserName()}, - updateStmt, params..., - ) - if err != nil { - return err - } - if n != 1 { - return errors.Errorf( - "expected exactly one row affected, but %d rows affected by job update", n, + if len(setters) != 0 { + updateStmt := fmt.Sprintf( + "UPDATE system.jobs SET %s WHERE id = $1", + strings.Join(setters, ", "), ) + n, err := u.txn.ExecEx( + ctx, "job-update", u.txn.KV(), + sessiondata.InternalExecutorOverride{User: username.NodeUserName()}, + updateStmt, params..., + ) + if err != nil { + return err + } + if n != 1 { + return errors.Errorf( + "expected exactly one row affected, but %d rows affected by job update", n, + ) + } } // Insert the job payload and details into the system.jobs_info table if the diff --git a/pkg/jobs/update_test.go b/pkg/jobs/update_test.go index 1a955273832a..c9021336555c 100644 --- a/pkg/jobs/update_test.go +++ b/pkg/jobs/update_test.go @@ -82,8 +82,6 @@ func TestUpdaterUpdatesJobInfo(t *testing.T) { expectedProgress jobspb.Progress) { infoStorage := createdJob.InfoStorage(txn) - // Verify the payload in the system.job_info is the same as what we read - // from system.jobs. payload, exists, err := infoStorage.GetLegacyPayload(ctx) require.NoError(t, err) require.True(t, exists) @@ -93,8 +91,6 @@ func TestUpdaterUpdatesJobInfo(t *testing.T) { } require.Equal(t, data, payload) - // Verify the progress in the system.job_info is the same as what we read - // from system.jobs. progress, exists, err := infoStorage.GetLegacyProgress(ctx) require.NoError(t, err) require.True(t, exists) @@ -108,23 +104,32 @@ func TestUpdaterUpdatesJobInfo(t *testing.T) { runTests := func(t *testing.T, createdJob *jobs.Job) { t.Run("verify against system.jobs", func(t *testing.T) { require.NoError(t, ief.Txn(ctx, func(ctx context.Context, txn isql.Txn) error { - progressQuery := `SELECT count(*) FROM system.jobs AS a LEFT JOIN system.job_info AS b ON a.progress = b.value WHERE b.job_id IS NULL;` + countSystemJobs := `SELECT count(*) FROM system.jobs` row, err := txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), - sessiondata.NodeUserSessionDataOverride, progressQuery) + sessiondata.NodeUserSessionDataOverride, countSystemJobs) if err != nil { return err } - count := tree.MustBeDInt(row[0]) - require.Equal(t, 0, int(count)) + jobsCount := tree.MustBeDInt(row[0]) - payloadQuery := `SELECT count(*) FROM system.jobs AS a LEFT JOIN system.job_info AS b ON a.payload = b.value WHERE b.job_id IS NULL;` + countSystemJobInfo := `SELECT count(*) FROM system.job_info;` row, err = txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), - sessiondata.NodeUserSessionDataOverride, payloadQuery) + sessiondata.NodeUserSessionDataOverride, countSystemJobInfo) if err != nil { return err } - count = tree.MustBeDInt(row[0]) - require.Equal(t, 0, int(count)) + jobInfoCount := tree.MustBeDInt(row[0]) + require.Equal(t, jobsCount*2, jobInfoCount) + + // Ensure no progress and payload is written to system.jobs. + nullPayloadAndProgress := `SELECT count(*) FROM system.jobs WHERE progress IS NOT NULL OR payload IS NOT NULL;` + row, err = txn.QueryRowEx(ctx, "verify-job-query", txn.KV(), + sessiondata.NodeUserSessionDataOverride, nullPayloadAndProgress) + if err != nil { + return err + } + nullProgressAndPayload := tree.MustBeDInt(row[0]) + require.Equal(t, 0, int(nullProgressAndPayload)) return nil })) }) diff --git a/pkg/jobs/utils.go b/pkg/jobs/utils.go index ad9935ead054..077d3217ca6d 100644 --- a/pkg/jobs/utils.go +++ b/pkg/jobs/utils.go @@ -34,7 +34,7 @@ func RunningJobExists( SELECT id, payload FROM - system.jobs + crdb_internal.system_jobs WHERE status IN ` + NonTerminalStatusTupleString + ` ORDER BY created` diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index b74e584e61e7..000c74e69221 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -429,6 +429,7 @@ go_test( "//pkg/testutils/testcluster", "//pkg/ts", "//pkg/ts/tspb", + "//pkg/upgrade/upgradebase", "//pkg/util", "//pkg/util/admission/admissionpb", "//pkg/util/caller", diff --git a/pkg/kv/kvserver/client_tenant_test.go b/pkg/kv/kvserver/client_tenant_test.go index 3ac53831ad0f..594a893dc7ef 100644 --- a/pkg/kv/kvserver/client_tenant_test.go +++ b/pkg/kv/kvserver/client_tenant_test.go @@ -26,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" _ "github.com/cockroachdb/cockroach/pkg/ccl" // for tenant functionality "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/keyvisualizer" "github.com/cockroachdb/cockroach/pkg/kv/kvpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/tenantrate" @@ -35,6 +36,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" + "github.com/cockroachdb/cockroach/pkg/upgrade/upgradebase" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -169,6 +171,11 @@ func TestTenantRateLimiter(t *testing.T) { TimeSource: timeSource, }, }, + KeyVisualizer: &keyvisualizer.TestingKnobs{SkipJobBootstrap: true}, + UpgradeManager: &upgradebase.TestingKnobs{ + SkipJobMetricsPollingJobBootstrap: true, + SkipAutoConfigRunnerJobBootstrap: true, + }, }, }) ctx := context.Background() @@ -211,7 +218,7 @@ func TestTenantRateLimiter(t *testing.T) { // bounds. writeCostLower := cfg.WriteBatchUnits + cfg.WriteRequestUnits writeCostUpper := cfg.WriteBatchUnits + cfg.WriteRequestUnits + float64(32)*cfg.WriteUnitsPerByte - tolerance := 10.0 // Leave space for a couple of other background requests. + tolerance := 30.0 // Leave space for a couple of other background requests. // burstWrites is a number of writes that don't exceed the burst limit. burstWrites := int((cfg.Burst - tolerance) / writeCostUpper) // tooManyWrites is a number of writes which definitely exceed the burst diff --git a/pkg/server/admin_test.go b/pkg/server/admin_test.go index 8bcd19b75580..2bb9f88783b0 100644 --- a/pkg/server/admin_test.go +++ b/pkg/server/admin_test.go @@ -1727,8 +1727,8 @@ func TestAdminAPIJobs(t *testing.T) { t.Fatal(err) } sqlDB.Exec(t, - `INSERT INTO system.jobs (id, status, payload, progress, num_runs, last_run, job_type) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - job.id, job.status, payloadBytes, progressBytes, job.numRuns, job.lastRun, payload.Type().String(), + `INSERT INTO system.jobs (id, status, num_runs, last_run, job_type) VALUES ($1, $2, $3, $4, $5)`, + job.id, job.status, job.numRuns, job.lastRun, payload.Type().String(), ) sqlDB.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, @@ -1918,8 +1918,8 @@ func TestAdminAPIJobsDetails(t *testing.T) { t.Fatal(err) } sqlDB.Exec(t, - `INSERT INTO system.jobs (id, status, payload, progress, num_runs, last_run) VALUES ($1, $2, $3, $4, $5, $6)`, - job.id, job.status, payloadBytes, progressBytes, job.numRuns, job.lastRun, + `INSERT INTO system.jobs (id, status, num_runs, last_run) VALUES ($1, $2, $3, $4)`, + job.id, job.status, job.numRuns, job.lastRun, ) sqlDB.Exec(t, `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, diff --git a/pkg/spanconfig/spanconfigmanager/manager_test.go b/pkg/spanconfig/spanconfigmanager/manager_test.go index 1727f4f7c0c5..6c50c6ea6b5c 100644 --- a/pkg/spanconfig/spanconfigmanager/manager_test.go +++ b/pkg/spanconfig/spanconfigmanager/manager_test.go @@ -172,9 +172,17 @@ func TestManagerStartsJobIfFailed(t *testing.T) { }) require.NoError(t, err) + id := ts.JobRegistry().(*jobs.Registry).MakeJobID() _, err = tc.ServerConn(0).Exec( - `INSERT INTO system.jobs (status, payload) VALUES ($1, $2)`, + `INSERT INTO system.jobs (id, status) VALUES ($1, $2)`, + id, jobs.StatusFailed, + ) + require.NoError(t, err) + _, err = tc.ServerConn(0).Exec( + `INSERT INTO system.job_info (job_id, info_key, value) VALUES ($1, $2, $3)`, + id, + jobs.GetLegacyPayloadKey(), payload, ) require.NoError(t, err) @@ -470,7 +478,7 @@ func waitForJobCheckpoint(t *testing.T, tdb *sqlutils.SQLRunner) { testutils.SucceedsSoon(t, func() error { var progressBytes []byte tdb.QueryRow(t, ` -SELECT progress FROM system.jobs +SELECT progress FROM crdb_internal.system_jobs WHERE id = (SELECT job_id FROM [SHOW AUTOMATIC JOBS] WHERE job_type = 'AUTO SPAN CONFIG RECONCILIATION') `).Scan(&progressBytes) diff --git a/pkg/sql/catalog/bootstrap/testdata/testdata b/pkg/sql/catalog/bootstrap/testdata/testdata index 270934b6b655..316860100cd4 100644 --- a/pkg/sql/catalog/bootstrap/testdata/testdata +++ b/pkg/sql/catalog/bootstrap/testdata/testdata @@ -1,4 +1,4 @@ -system hash=53d96661ba72bfc04ab9365c988d3ef71e25de490c00ff43eb267b91422c2717 +system hash=463a3ab3201f211297b8ff6b8782e2ebbe0cd3fbbc77e8159ae81a5f41a5a88f ---- [{"key":"04646573632d696467656e","value":"01c801"} ,{"key":"8b"} @@ -13,7 +13,7 @@ system hash=53d96661ba72bfc04ab9365c988d3ef71e25de490c00ff43eb267b91422c2717 ,{"key":"8b89948a89","value":"030afa050a086576656e746c6f67180c200128013a00422f0a0974696d657374616d7010011a0d080510001800300050da08600020003000680070007800800100880100980100422e0a096576656e745479706510021a0c08071000180030005019600020003000680070007800800100880100980100422d0a08746172676574494410031a0c0801104018003000501460002000300068007000780080010088010098010042300a0b7265706f7274696e67494410041a0c0801104018003000501460002000300068007000780080010088010098010042290a04696e666f10051a0c0807100018003000501960002001300068007000780080010088010098010042380a08756e69717565494410061a0c08081000180030005011600020002a09757569645f7634282930006800700078008001008801009801004807529e010a077072696d61727910011801220974696d657374616d702208756e6971756549442a096576656e74547970652a0874617267657449442a0b7265706f7274696e6749442a04696e666f30013006400040004a10080010001a00200028003000380040005a0070027003700470057a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201260a077072696d61727910001a0974696d657374616d701a08756e697175654944200120062800b201220a0f66616d5f325f6576656e745479706510021a096576656e745479706520022802b201200a0e66616d5f335f746172676574494410031a08746172676574494420032803b201260a1166616d5f345f7265706f7274696e67494410041a0b7265706f7274696e67494420042804b201180a0a66616d5f355f696e666f10051a04696e666f20052805b80106c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b89958a89","value":"030ad9060a0872616e67656c6f67180d200128013a00422f0a0974696d657374616d7010011a0d080510001800300050da08600020003000680070007800800100880100980100422c0a0772616e6765494410021a0c08011040180030005014600020003000680070007800800100880100980100422c0a0773746f7265494410031a0c08011040180030005014600020003000680070007800800100880100980100422e0a096576656e745479706510041a0c0807100018003000501960002000300068007000780080010088010098010042310a0c6f7468657252616e6765494410051a0c0801104018003000501460002001300068007000780080010088010098010042290a04696e666f10061a0c08071000180030005019600020013000680070007800800100880100980100423d0a08756e69717565494410071a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100480852a9010a077072696d61727910011801220974696d657374616d702208756e6971756549442a0772616e676549442a0773746f726549442a096576656e74547970652a0c6f7468657252616e676549442a04696e666f30013007400040004a10080010001a00200028003000380040005a00700270037004700570067a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201260a077072696d61727910001a0974696d657374616d701a08756e697175654944200120072800b2011e0a0d66616d5f325f72616e6765494410021a0772616e6765494420022802b2011e0a0d66616d5f335f73746f7265494410031a0773746f7265494420032803b201220a0f66616d5f345f6576656e745479706510041a096576656e745479706520042804b201280a1266616d5f355f6f7468657252616e6765494410051a0c6f7468657252616e6765494420052805b201180a0a66616d5f365f696e666f10061a04696e666f20062806b80107c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b89968a89","value":"030ad1030a027569180e200128013a0042280a036b657910011a0c08071000180030005019600020003000680070007800800100880100980100422a0a0576616c756510021a0c0808100018003000501160002001300068007000780080010088010098010042310a0b6c6173745570646174656410031a0d080510001800300050da08600020003000680070007800800100880100980100480452720a077072696d6172791001180122036b65792a0576616c75652a0b6c61737455706461746564300140004a10080010001a00200028003000380040005a00700270037a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201140a077072696d61727910001a036b657920012800b2011a0a0b66616d5f325f76616c756510021a0576616c756520022802b201260a1166616d5f335f6c6173745570646174656410031a0b6c6173745570646174656420032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} -,{"key":"8b89978a89","value":"030a990f0a046a6f6273180f200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100422b0a0673746174757310021a0c0807100018003000501960002000300068007000780080010088010098010042400a076372656174656410031a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422c0a077061796c6f616410041a0c08081000180030005011600020003000680070007800800100880100980100422d0a0870726f677265737310051a0c0808100018003000501160002001300068007000780080010088010098010042340a0f637265617465645f62795f7479706510061a0c0807100018003000501960002001300068007000780080010088010098010042320a0d637265617465645f62795f696410071a0c0801104018003000501460002001300068007000780080010088010098010042350a10636c61696d5f73657373696f6e5f696410081a0c0808100018003000501160002001300068007000780080010088010098010042360a11636c61696d5f696e7374616e63655f696410091a0c08011040180030005014600020013000680070007800800100880100980100422d0a086e756d5f72756e73100a1a0c08011040180030005014600020013000680070007800800100880100980100422e0a086c6173745f72756e100b1a0d080510001800300050da08600020013000680070007800800100880100980100422d0a086a6f625f74797065100c1a0c08071000180030005019600020013000680070007800800100880100980100480d52f6010a077072696d61727910011801220269642a067374617475732a07637265617465642a077061796c6f61642a0870726f67726573732a0f637265617465645f62795f747970652a0d637265617465645f62795f69642a10636c61696d5f73657373696f6e5f69642a11636c61696d5f696e7374616e63655f69642a086e756d5f72756e732a086c6173745f72756e2a086a6f625f74797065300140004a10080010001a00200028003000380040005a0070027003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a7c0a176a6f62735f7374617475735f637265617465645f696478100218002206737461747573220763726561746564300230033801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005aa4010a266a6f62735f637265617465645f62795f747970655f637265617465645f62795f69645f69647810031800220f637265617465645f62795f74797065220d637265617465645f62795f69642a06737461747573300630073801400040004a10080010001a00200028003000380040005a0070027a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005abc020a126a6f62735f72756e5f73746174735f696478100418002210636c61696d5f73657373696f6e5f696422067374617475732207637265617465642a086c6173745f72756e2a086e756d5f72756e732a11636c61696d5f696e7374616e63655f696430083002300338014000400040004a10080010001a00200028003000380040005a00700b700a70097a0408002000800100880100900103980100a20106080012001800a80100b20100ba01810173746174757320494e20282772756e6e696e67273a3a3a535452494e472c2027726576657274696e67273a3a3a535452494e472c202770656e64696e67273a3a3a535452494e472c202770617573652d726571756573746564273a3a3a535452494e472c202763616e63656c2d726571756573746564273a3a3a535452494e4729c00100c80100d00100e001005a6b0a116a6f62735f6a6f625f747970655f6964781005180022086a6f625f74797065300c380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b2017b0a1f66616d5f305f69645f7374617475735f637265617465645f7061796c6f616410001a0269641a067374617475731a07637265617465641a077061796c6f61641a0f637265617465645f62795f747970651a0d637265617465645f62795f69641a086a6f625f74797065200120022003200420062007200c2800b2011a0a0870726f677265737310011a0870726f677265737320052805b2014c0a05636c61696d10021a10636c61696d5f73657373696f6e5f69641a11636c61696d5f696e7374616e63655f69641a086e756d5f72756e731a086c6173745f72756e20082009200a200b2800b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} +,{"key":"8b89978a89","value":"030a990f0a046a6f6273180f200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100422b0a0673746174757310021a0c0807100018003000501960002000300068007000780080010088010098010042400a076372656174656410031a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422c0a077061796c6f616410041a0c08081000180030005011600020013000680070007800800100880100980100422d0a0870726f677265737310051a0c0808100018003000501160002001300068007000780080010088010098010042340a0f637265617465645f62795f7479706510061a0c0807100018003000501960002001300068007000780080010088010098010042320a0d637265617465645f62795f696410071a0c0801104018003000501460002001300068007000780080010088010098010042350a10636c61696d5f73657373696f6e5f696410081a0c0808100018003000501160002001300068007000780080010088010098010042360a11636c61696d5f696e7374616e63655f696410091a0c08011040180030005014600020013000680070007800800100880100980100422d0a086e756d5f72756e73100a1a0c08011040180030005014600020013000680070007800800100880100980100422e0a086c6173745f72756e100b1a0d080510001800300050da08600020013000680070007800800100880100980100422d0a086a6f625f74797065100c1a0c08071000180030005019600020013000680070007800800100880100980100480d52f6010a077072696d61727910011801220269642a067374617475732a07637265617465642a077061796c6f61642a0870726f67726573732a0f637265617465645f62795f747970652a0d637265617465645f62795f69642a10636c61696d5f73657373696f6e5f69642a11636c61696d5f696e7374616e63655f69642a086e756d5f72756e732a086c6173745f72756e2a086a6f625f74797065300140004a10080010001a00200028003000380040005a0070027003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a7c0a176a6f62735f7374617475735f637265617465645f696478100218002206737461747573220763726561746564300230033801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005aa4010a266a6f62735f637265617465645f62795f747970655f637265617465645f62795f69645f69647810031800220f637265617465645f62795f74797065220d637265617465645f62795f69642a06737461747573300630073801400040004a10080010001a00200028003000380040005a0070027a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005abc020a126a6f62735f72756e5f73746174735f696478100418002210636c61696d5f73657373696f6e5f696422067374617475732207637265617465642a086c6173745f72756e2a086e756d5f72756e732a11636c61696d5f696e7374616e63655f696430083002300338014000400040004a10080010001a00200028003000380040005a00700b700a70097a0408002000800100880100900103980100a20106080012001800a80100b20100ba01810173746174757320494e20282772756e6e696e67273a3a3a535452494e472c2027726576657274696e67273a3a3a535452494e472c202770656e64696e67273a3a3a535452494e472c202770617573652d726571756573746564273a3a3a535452494e472c202763616e63656c2d726571756573746564273a3a3a535452494e4729c00100c80100d00100e001005a6b0a116a6f62735f6a6f625f747970655f6964781005180022086a6f625f74797065300c380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b2017b0a1f66616d5f305f69645f7374617475735f637265617465645f7061796c6f616410001a0269641a067374617475731a07637265617465641a077061796c6f61641a0f637265617465645f62795f747970651a0d637265617465645f62795f69641a086a6f625f74797065200120022003200420062007200c2800b2011a0a0870726f677265737310011a0870726f677265737320052805b2014c0a05636c61696d10021a10636c61696d5f73657373696f6e5f69641a11636c61696d5f696e7374616e63655f69641a086e756d5f72756e731a086c6173745f72756e20082009200a200b2800b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899b8a89","value":"030ad20b0a0c7765625f73657373696f6e731813200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f7769642829300068007000780080010088010098010042310a0c68617368656453656372657410021a0c08081000180030005011600020003000680070007800800100880100980100422d0a08757365726e616d6510031a0c0807100018003000501960002000300068007000780080010088010098010042420a0963726561746564417410041a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422f0a0965787069726573417410051a0d080510001800300050da08600020003000680070007800800100880100980100422f0a097265766f6b6564417410061a0d080510001800300050da0860002001300068007000780080010088010098010042430a0a6c61737455736564417410071a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422e0a096175646974496e666f10081a0c08071000180030005019600020013000680070007800800100880100980100422c0a07757365725f696410091a0c080c100018003000501a600020003000680070007800800100880100980100480a52c2010a077072696d61727910011801220269642a0c6861736865645365637265742a08757365726e616d652a096372656174656441742a096578706972657341742a097265766f6b656441742a0a6c6173745573656441742a096175646974496e666f2a07757365725f6964300140004a10080010001a00200028003000380040005a00700270037004700570067007700870097a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a750a1a7765625f73657373696f6e735f6578706972657341745f6964781002180022096578706972657341743005380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a750a1a7765625f73657373696f6e735f6372656174656441745f6964781003180022096372656174656441743004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a750a1a7765625f73657373696f6e735f7265766f6b656441745f6964781004180022097265766f6b656441743006380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a770a1b7765625f73657373696f6e735f6c6173745573656441745f69647810051800220a6c6173745573656441743007380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201c6010a5166616d5f305f69645f6861736865645365637265745f757365726e616d655f6372656174656441745f6578706972657341745f7265766f6b656441745f6c6173745573656441745f6175646974496e666f10001a0269641a0c6861736865645365637265741a08757365726e616d651a096372656174656441741a096578706972657341741a097265766f6b656441741a0a6c6173745573656441741a096175646974496e666f1a07757365725f69642001200220032004200520062007200820092800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899c8a89","value":"030a960a0a107461626c655f737461746973746963731814200128013a00422c0a077461626c65494410011a0c0801104018003000501460002000300068007000780080010088010098010042400a0b737461746973746963494410021a0c08011040180030005014600020002a0e756e697175655f726f7769642829300068007000780080010088010098010042290a046e616d6510031a0c08071000180030005019600020013000680070007800800100880100980100423f0a09636f6c756d6e49447310041a1d080f104018003000380150f8075a0c08011040180030005014600060002000300068007000780080010088010098010042420a0963726561746564417410051a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422d0a08726f77436f756e7410061a0c0801104018003000501460002000300068007000780080010088010098010042320a0d64697374696e6374436f756e7410071a0c08011040180030005014600020003000680070007800800100880100980100422e0a096e756c6c436f756e7410081a0c08011040180030005014600020003000680070007800800100880100980100422e0a09686973746f6772616d10091a0c0808100018003000501160002001300068007000780080010088010098010042360a0761766753697a65100a1a0c08011040180030005014600020002a08303a3a3a494e5438300068007000780080010088010098010042350a107061727469616c507265646963617465100b1a0c0807100018003000501960002001300068007000780080010088010098010042340a0f66756c6c5374617469737469634944100c1a0c08011040180030005014600020013000680070007800800100880100980100480d52fa010a077072696d6172791001180122077461626c654944220b73746174697374696349442a046e616d652a09636f6c756d6e4944732a096372656174656441742a08726f77436f756e742a0d64697374696e6374436f756e742a096e756c6c436f756e742a09686973746f6772616d2a0761766753697a652a107061727469616c5072656469636174652a0f66756c6c537461746973746963494430013002400040004a10080010001a00200028003000380040005a007003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b20188020a5d66616d5f305f7461626c6549445f73746174697374696349445f6e616d655f636f6c756d6e4944735f6372656174656441745f726f77436f756e745f64697374696e6374436f756e745f6e756c6c436f756e745f686973746f6772616d10001a077461626c6549441a0b73746174697374696349441a046e616d651a09636f6c756d6e4944731a096372656174656441741a08726f77436f756e741a0d64697374696e6374436f756e741a096e756c6c436f756e741a09686973746f6772616d1a0761766753697a651a107061727469616c5072656469636174651a0f66756c6c5374617469737469634944200120022003200420052006200720082009200a200b200c2800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899d8a89","value":"030aca040a096c6f636174696f6e731815200128013a0042300a0b6c6f63616c6974794b657910011a0c0807100018003000501960002000300068007000780080010088010098010042320a0d6c6f63616c69747956616c756510021a0c08071000180030005019600020003000680070007800800100880100980100422e0a086c6174697475646510031a0d0803100f1812300050a40d600020003000680070007800800100880100980100422f0a096c6f6e67697475646510041a0d0803100f1812300050a40d6000200030006800700078008001008801009801004805528e010a077072696d61727910011801220b6c6f63616c6974794b6579220d6c6f63616c69747956616c75652a086c617469747564652a096c6f6e67697475646530013002400040004a10080010001a00200028003000380040005a00700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201710a3266616d5f305f6c6f63616c6974794b65795f6c6f63616c69747956616c75655f6c617469747564655f6c6f6e67697475646510001a0b6c6f63616c6974794b65791a0d6c6f63616c69747956616c75651a086c617469747564651a096c6f6e67697475646520012002200320042800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} @@ -175,7 +175,7 @@ system hash=53d96661ba72bfc04ab9365c988d3ef71e25de490c00ff43eb267b91422c2717 ,{"key":"c5"} ] -tenant hash=15c6717507feba07abf595e6fb5ecf16364a3888aa462a0e924d03523d892b3f +tenant hash=238142119bd7cc980ba280eba1ac2cefeab072aff41563a2c7d839066baa2e81 ---- [{"key":""} ,{"key":"8b89898a89","value":"0312390a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518022200280140004a00"} @@ -188,7 +188,7 @@ tenant hash=15c6717507feba07abf595e6fb5ecf16364a3888aa462a0e924d03523d892b3f ,{"key":"8b89948a89","value":"030afa050a086576656e746c6f67180c200128013a00422f0a0974696d657374616d7010011a0d080510001800300050da08600020003000680070007800800100880100980100422e0a096576656e745479706510021a0c08071000180030005019600020003000680070007800800100880100980100422d0a08746172676574494410031a0c0801104018003000501460002000300068007000780080010088010098010042300a0b7265706f7274696e67494410041a0c0801104018003000501460002000300068007000780080010088010098010042290a04696e666f10051a0c0807100018003000501960002001300068007000780080010088010098010042380a08756e69717565494410061a0c08081000180030005011600020002a09757569645f7634282930006800700078008001008801009801004807529e010a077072696d61727910011801220974696d657374616d702208756e6971756549442a096576656e74547970652a0874617267657449442a0b7265706f7274696e6749442a04696e666f30013006400040004a10080010001a00200028003000380040005a0070027003700470057a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201260a077072696d61727910001a0974696d657374616d701a08756e697175654944200120062800b201220a0f66616d5f325f6576656e745479706510021a096576656e745479706520022802b201200a0e66616d5f335f746172676574494410031a08746172676574494420032803b201260a1166616d5f345f7265706f7274696e67494410041a0b7265706f7274696e67494420042804b201180a0a66616d5f355f696e666f10051a04696e666f20052805b80106c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b89958a89","value":"030ad9060a0872616e67656c6f67180d200128013a00422f0a0974696d657374616d7010011a0d080510001800300050da08600020003000680070007800800100880100980100422c0a0772616e6765494410021a0c08011040180030005014600020003000680070007800800100880100980100422c0a0773746f7265494410031a0c08011040180030005014600020003000680070007800800100880100980100422e0a096576656e745479706510041a0c0807100018003000501960002000300068007000780080010088010098010042310a0c6f7468657252616e6765494410051a0c0801104018003000501460002001300068007000780080010088010098010042290a04696e666f10061a0c08071000180030005019600020013000680070007800800100880100980100423d0a08756e69717565494410071a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100480852a9010a077072696d61727910011801220974696d657374616d702208756e6971756549442a0772616e676549442a0773746f726549442a096576656e74547970652a0c6f7468657252616e676549442a04696e666f30013007400040004a10080010001a00200028003000380040005a00700270037004700570067a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201260a077072696d61727910001a0974696d657374616d701a08756e697175654944200120072800b2011e0a0d66616d5f325f72616e6765494410021a0772616e6765494420022802b2011e0a0d66616d5f335f73746f7265494410031a0773746f7265494420032803b201220a0f66616d5f345f6576656e745479706510041a096576656e745479706520042804b201280a1266616d5f355f6f7468657252616e6765494410051a0c6f7468657252616e6765494420052805b201180a0a66616d5f365f696e666f10061a04696e666f20062806b80107c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b89968a89","value":"030ad1030a027569180e200128013a0042280a036b657910011a0c08071000180030005019600020003000680070007800800100880100980100422a0a0576616c756510021a0c0808100018003000501160002001300068007000780080010088010098010042310a0b6c6173745570646174656410031a0d080510001800300050da08600020003000680070007800800100880100980100480452720a077072696d6172791001180122036b65792a0576616c75652a0b6c61737455706461746564300140004a10080010001a00200028003000380040005a00700270037a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201140a077072696d61727910001a036b657920012800b2011a0a0b66616d5f325f76616c756510021a0576616c756520022802b201260a1166616d5f335f6c6173745570646174656410031a0b6c6173745570646174656420032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} -,{"key":"8b89978a89","value":"030a990f0a046a6f6273180f200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100422b0a0673746174757310021a0c0807100018003000501960002000300068007000780080010088010098010042400a076372656174656410031a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422c0a077061796c6f616410041a0c08081000180030005011600020003000680070007800800100880100980100422d0a0870726f677265737310051a0c0808100018003000501160002001300068007000780080010088010098010042340a0f637265617465645f62795f7479706510061a0c0807100018003000501960002001300068007000780080010088010098010042320a0d637265617465645f62795f696410071a0c0801104018003000501460002001300068007000780080010088010098010042350a10636c61696d5f73657373696f6e5f696410081a0c0808100018003000501160002001300068007000780080010088010098010042360a11636c61696d5f696e7374616e63655f696410091a0c08011040180030005014600020013000680070007800800100880100980100422d0a086e756d5f72756e73100a1a0c08011040180030005014600020013000680070007800800100880100980100422e0a086c6173745f72756e100b1a0d080510001800300050da08600020013000680070007800800100880100980100422d0a086a6f625f74797065100c1a0c08071000180030005019600020013000680070007800800100880100980100480d52f6010a077072696d61727910011801220269642a067374617475732a07637265617465642a077061796c6f61642a0870726f67726573732a0f637265617465645f62795f747970652a0d637265617465645f62795f69642a10636c61696d5f73657373696f6e5f69642a11636c61696d5f696e7374616e63655f69642a086e756d5f72756e732a086c6173745f72756e2a086a6f625f74797065300140004a10080010001a00200028003000380040005a0070027003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a7c0a176a6f62735f7374617475735f637265617465645f696478100218002206737461747573220763726561746564300230033801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005aa4010a266a6f62735f637265617465645f62795f747970655f637265617465645f62795f69645f69647810031800220f637265617465645f62795f74797065220d637265617465645f62795f69642a06737461747573300630073801400040004a10080010001a00200028003000380040005a0070027a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005abc020a126a6f62735f72756e5f73746174735f696478100418002210636c61696d5f73657373696f6e5f696422067374617475732207637265617465642a086c6173745f72756e2a086e756d5f72756e732a11636c61696d5f696e7374616e63655f696430083002300338014000400040004a10080010001a00200028003000380040005a00700b700a70097a0408002000800100880100900103980100a20106080012001800a80100b20100ba01810173746174757320494e20282772756e6e696e67273a3a3a535452494e472c2027726576657274696e67273a3a3a535452494e472c202770656e64696e67273a3a3a535452494e472c202770617573652d726571756573746564273a3a3a535452494e472c202763616e63656c2d726571756573746564273a3a3a535452494e4729c00100c80100d00100e001005a6b0a116a6f62735f6a6f625f747970655f6964781005180022086a6f625f74797065300c380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b2017b0a1f66616d5f305f69645f7374617475735f637265617465645f7061796c6f616410001a0269641a067374617475731a07637265617465641a077061796c6f61641a0f637265617465645f62795f747970651a0d637265617465645f62795f69641a086a6f625f74797065200120022003200420062007200c2800b2011a0a0870726f677265737310011a0870726f677265737320052805b2014c0a05636c61696d10021a10636c61696d5f73657373696f6e5f69641a11636c61696d5f696e7374616e63655f69641a086e756d5f72756e731a086c6173745f72756e20082009200a200b2800b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} +,{"key":"8b89978a89","value":"030a990f0a046a6f6273180f200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f77696428293000680070007800800100880100980100422b0a0673746174757310021a0c0807100018003000501960002000300068007000780080010088010098010042400a076372656174656410031a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422c0a077061796c6f616410041a0c08081000180030005011600020013000680070007800800100880100980100422d0a0870726f677265737310051a0c0808100018003000501160002001300068007000780080010088010098010042340a0f637265617465645f62795f7479706510061a0c0807100018003000501960002001300068007000780080010088010098010042320a0d637265617465645f62795f696410071a0c0801104018003000501460002001300068007000780080010088010098010042350a10636c61696d5f73657373696f6e5f696410081a0c0808100018003000501160002001300068007000780080010088010098010042360a11636c61696d5f696e7374616e63655f696410091a0c08011040180030005014600020013000680070007800800100880100980100422d0a086e756d5f72756e73100a1a0c08011040180030005014600020013000680070007800800100880100980100422e0a086c6173745f72756e100b1a0d080510001800300050da08600020013000680070007800800100880100980100422d0a086a6f625f74797065100c1a0c08071000180030005019600020013000680070007800800100880100980100480d52f6010a077072696d61727910011801220269642a067374617475732a07637265617465642a077061796c6f61642a0870726f67726573732a0f637265617465645f62795f747970652a0d637265617465645f62795f69642a10636c61696d5f73657373696f6e5f69642a11636c61696d5f696e7374616e63655f69642a086e756d5f72756e732a086c6173745f72756e2a086a6f625f74797065300140004a10080010001a00200028003000380040005a0070027003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a7c0a176a6f62735f7374617475735f637265617465645f696478100218002206737461747573220763726561746564300230033801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005aa4010a266a6f62735f637265617465645f62795f747970655f637265617465645f62795f69645f69647810031800220f637265617465645f62795f74797065220d637265617465645f62795f69642a06737461747573300630073801400040004a10080010001a00200028003000380040005a0070027a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005abc020a126a6f62735f72756e5f73746174735f696478100418002210636c61696d5f73657373696f6e5f696422067374617475732207637265617465642a086c6173745f72756e2a086e756d5f72756e732a11636c61696d5f696e7374616e63655f696430083002300338014000400040004a10080010001a00200028003000380040005a00700b700a70097a0408002000800100880100900103980100a20106080012001800a80100b20100ba01810173746174757320494e20282772756e6e696e67273a3a3a535452494e472c2027726576657274696e67273a3a3a535452494e472c202770656e64696e67273a3a3a535452494e472c202770617573652d726571756573746564273a3a3a535452494e472c202763616e63656c2d726571756573746564273a3a3a535452494e4729c00100c80100d00100e001005a6b0a116a6f62735f6a6f625f747970655f6964781005180022086a6f625f74797065300c380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b2017b0a1f66616d5f305f69645f7374617475735f637265617465645f7061796c6f616410001a0269641a067374617475731a07637265617465641a077061796c6f61641a0f637265617465645f62795f747970651a0d637265617465645f62795f69641a086a6f625f74797065200120022003200420062007200c2800b2011a0a0870726f677265737310011a0870726f677265737320052805b2014c0a05636c61696d10021a10636c61696d5f73657373696f6e5f69641a11636c61696d5f696e7374616e63655f69641a086e756d5f72756e731a086c6173745f72756e20082009200a200b2800b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899b8a89","value":"030ad20b0a0c7765625f73657373696f6e731813200128013a0042370a02696410011a0c08011040180030005014600020002a0e756e697175655f726f7769642829300068007000780080010088010098010042310a0c68617368656453656372657410021a0c08081000180030005011600020003000680070007800800100880100980100422d0a08757365726e616d6510031a0c0807100018003000501960002000300068007000780080010088010098010042420a0963726561746564417410041a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422f0a0965787069726573417410051a0d080510001800300050da08600020003000680070007800800100880100980100422f0a097265766f6b6564417410061a0d080510001800300050da0860002001300068007000780080010088010098010042430a0a6c61737455736564417410071a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422e0a096175646974496e666f10081a0c08071000180030005019600020013000680070007800800100880100980100422c0a07757365725f696410091a0c080c100018003000501a600020003000680070007800800100880100980100480a52c2010a077072696d61727910011801220269642a0c6861736865645365637265742a08757365726e616d652a096372656174656441742a096578706972657341742a097265766f6b656441742a0a6c6173745573656441742a096175646974496e666f2a07757365725f6964300140004a10080010001a00200028003000380040005a00700270037004700570067007700870097a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e001005a750a1a7765625f73657373696f6e735f6578706972657341745f6964781002180022096578706972657341743005380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a750a1a7765625f73657373696f6e735f6372656174656441745f6964781003180022096372656174656441743004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a750a1a7765625f73657373696f6e735f7265766f6b656441745f6964781004180022097265766f6b656441743006380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e001005a770a1b7765625f73657373696f6e735f6c6173745573656441745f69647810051800220a6c6173745573656441743007380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00100e0010060066a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201c6010a5166616d5f305f69645f6861736865645365637265745f757365726e616d655f6372656174656441745f6578706972657341745f7265766f6b656441745f6c6173745573656441745f6175646974496e666f10001a0269641a0c6861736865645365637265741a08757365726e616d651a096372656174656441741a096578706972657341741a097265766f6b656441741a0a6c6173745573656441741a096175646974496e666f1a07757365725f69642001200220032004200520062007200820092800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899c8a89","value":"030a960a0a107461626c655f737461746973746963731814200128013a00422c0a077461626c65494410011a0c0801104018003000501460002000300068007000780080010088010098010042400a0b737461746973746963494410021a0c08011040180030005014600020002a0e756e697175655f726f7769642829300068007000780080010088010098010042290a046e616d6510031a0c08071000180030005019600020013000680070007800800100880100980100423f0a09636f6c756d6e49447310041a1d080f104018003000380150f8075a0c08011040180030005014600060002000300068007000780080010088010098010042420a0963726561746564417410051a0d080510001800300050da08600020002a116e6f7728293a3a3a54494d455354414d503000680070007800800100880100980100422d0a08726f77436f756e7410061a0c0801104018003000501460002000300068007000780080010088010098010042320a0d64697374696e6374436f756e7410071a0c08011040180030005014600020003000680070007800800100880100980100422e0a096e756c6c436f756e7410081a0c08011040180030005014600020003000680070007800800100880100980100422e0a09686973746f6772616d10091a0c0808100018003000501160002001300068007000780080010088010098010042360a0761766753697a65100a1a0c08011040180030005014600020002a08303a3a3a494e5438300068007000780080010088010098010042350a107061727469616c507265646963617465100b1a0c0807100018003000501960002001300068007000780080010088010098010042340a0f66756c6c5374617469737469634944100c1a0c08011040180030005014600020013000680070007800800100880100980100480d52fa010a077072696d6172791001180122077461626c654944220b73746174697374696349442a046e616d652a09636f6c756d6e4944732a096372656174656441742a08726f77436f756e742a0d64697374696e6374436f756e742a096e756c6c436f756e742a09686973746f6772616d2a0761766753697a652a107061727469616c5072656469636174652a0f66756c6c537461746973746963494430013002400040004a10080010001a00200028003000380040005a007003700470057006700770087009700a700b700c7a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b20188020a5d66616d5f305f7461626c6549445f73746174697374696349445f6e616d655f636f6c756d6e4944735f6372656174656441745f726f77436f756e745f64697374696e6374436f756e745f6e756c6c436f756e745f686973746f6772616d10001a077461626c6549441a0b73746174697374696349441a046e616d651a09636f6c756d6e4944731a096372656174656441741a08726f77436f756e741a0d64697374696e6374436f756e741a096e756c6c436f756e741a09686973746f6772616d1a0761766753697a651a107061727469616c5072656469636174651a0f66756c6c5374617469737469634944200120022003200420052006200720082009200a200b200c2800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} ,{"key":"8b899d8a89","value":"030aca040a096c6f636174696f6e731815200128013a0042300a0b6c6f63616c6974794b657910011a0c0807100018003000501960002000300068007000780080010088010098010042320a0d6c6f63616c69747956616c756510021a0c08071000180030005019600020003000680070007800800100880100980100422e0a086c6174697475646510031a0d0803100f1812300050a40d600020003000680070007800800100880100980100422f0a096c6f6e67697475646510041a0d0803100f1812300050a40d6000200030006800700078008001008801009801004805528e010a077072696d61727910011801220b6c6f63616c6974794b6579220d6c6f63616c69747956616c75652a086c617469747564652a096c6f6e67697475646530013002400040004a10080010001a00200028003000380040005a00700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e0010060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201710a3266616d5f305f6c6f63616c6974794b65795f6c6f63616c69747956616c75655f6c617469747564655f6c6f6e67697475646510001a0b6c6f63616c6974794b65791a0d6c6f63616c69747956616c75651a086c617469747564651a096c6f6e67697475646520012002200320042800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300"} diff --git a/pkg/sql/catalog/systemschema/system.go b/pkg/sql/catalog/systemschema/system.go index 24152331f90a..e2a7c80f10f2 100644 --- a/pkg/sql/catalog/systemschema/system.go +++ b/pkg/sql/catalog/systemschema/system.go @@ -208,7 +208,7 @@ CREATE TABLE system.jobs ( id INT8 DEFAULT unique_rowid(), status STRING NOT NULL, created TIMESTAMP NOT NULL DEFAULT now(), - payload BYTES NOT NULL, + payload BYTES, progress BYTES, created_by_type STRING, created_by_id INT, @@ -1646,7 +1646,7 @@ var ( {Name: "id", ID: 1, Type: types.Int, DefaultExpr: &uniqueRowIDString}, {Name: "status", ID: 2, Type: types.String}, {Name: "created", ID: 3, Type: types.Timestamp, DefaultExpr: &nowString}, - {Name: "payload", ID: 4, Type: types.Bytes}, + {Name: "payload", ID: 4, Type: types.Bytes, Nullable: true}, {Name: "progress", ID: 5, Type: types.Bytes, Nullable: true}, {Name: "created_by_type", ID: 6, Type: types.String, Nullable: true}, {Name: "created_by_id", ID: 7, Type: types.Int, Nullable: true}, diff --git a/pkg/sql/catalog/systemschema_test/testdata/bootstrap b/pkg/sql/catalog/systemschema_test/testdata/bootstrap index 306ad9202e2e..25b51807404c 100644 --- a/pkg/sql/catalog/systemschema_test/testdata/bootstrap +++ b/pkg/sql/catalog/systemschema_test/testdata/bootstrap @@ -97,7 +97,7 @@ CREATE TABLE public.jobs ( id INT8 NOT NULL DEFAULT unique_rowid(), status STRING NOT NULL, created TIMESTAMP NOT NULL DEFAULT now():::TIMESTAMP, - payload BYTES NOT NULL, + payload BYTES NULL, progress BYTES NULL, created_by_type STRING NULL, created_by_id INT8 NULL, @@ -562,7 +562,7 @@ schema_telemetry {"table":{"name":"eventlog","id":12,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"timestamp","id":1,"type":{"family":"TimestampFamily","oid":1114}},{"name":"eventType","id":2,"type":{"family":"StringFamily","oid":25}},{"name":"targetID","id":3,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"reportingID","id":4,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"info","id":5,"type":{"family":"StringFamily","oid":25},"nullable":true},{"name":"uniqueID","id":6,"type":{"family":"BytesFamily","oid":17},"defaultExpr":"uuid_v4()"}],"nextColumnId":7,"families":[{"name":"primary","columnNames":["timestamp","uniqueID"],"columnIds":[1,6]},{"name":"fam_2_eventType","id":2,"columnNames":["eventType"],"columnIds":[2],"defaultColumnId":2},{"name":"fam_3_targetID","id":3,"columnNames":["targetID"],"columnIds":[3],"defaultColumnId":3},{"name":"fam_4_reportingID","id":4,"columnNames":["reportingID"],"columnIds":[4],"defaultColumnId":4},{"name":"fam_5_info","id":5,"columnNames":["info"],"columnIds":[5],"defaultColumnId":5}],"nextFamilyId":6,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["timestamp","uniqueID"],"keyColumnDirections":["ASC","ASC"],"storeColumnNames":["eventType","targetID","reportingID","info"],"keyColumnIds":[1,6],"storeColumnIds":[2,3,4,5],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":2,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} {"table":{"name":"external_connections","id":52,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"connection_name","id":1,"type":{"family":"StringFamily","oid":25}},{"name":"created","id":2,"type":{"family":"TimestampFamily","oid":1114},"defaultExpr":"now():::TIMESTAMP"},{"name":"updated","id":3,"type":{"family":"TimestampFamily","oid":1114},"defaultExpr":"now():::TIMESTAMP"},{"name":"connection_type","id":4,"type":{"family":"StringFamily","oid":25}},{"name":"connection_details","id":5,"type":{"family":"BytesFamily","oid":17}},{"name":"owner","id":6,"type":{"family":"StringFamily","oid":25}},{"name":"owner_id","id":7,"type":{"family":"OidFamily","oid":26}}],"nextColumnId":8,"families":[{"name":"primary","columnNames":["connection_name","created","updated","connection_type","connection_details","owner","owner_id"],"columnIds":[1,2,3,4,5,6,7]}],"nextFamilyId":1,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["connection_name"],"keyColumnDirections":["ASC"],"storeColumnNames":["created","updated","connection_type","connection_details","owner","owner_id"],"keyColumnIds":[1],"storeColumnIds":[2,3,4,5,6,7],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":2,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} {"table":{"name":"job_info","id":53,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"job_id","id":1,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"info_key","id":2,"type":{"family":"BytesFamily","oid":17}},{"name":"written","id":3,"type":{"family":"TimestampTZFamily","oid":1184},"defaultExpr":"now():::TIMESTAMPTZ"},{"name":"value","id":4,"type":{"family":"BytesFamily","oid":17},"nullable":true}],"nextColumnId":5,"families":[{"name":"primary","columnNames":["job_id","info_key","written","value"],"columnIds":[1,2,3,4],"defaultColumnId":4}],"nextFamilyId":1,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["job_id","info_key","written"],"keyColumnDirections":["ASC","ASC","DESC"],"storeColumnNames":["value"],"keyColumnIds":[1,2,3],"storeColumnIds":[4],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":2,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} -{"table":{"name":"jobs","id":15,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"id","id":1,"type":{"family":"IntFamily","width":64,"oid":20},"defaultExpr":"unique_rowid()"},{"name":"status","id":2,"type":{"family":"StringFamily","oid":25}},{"name":"created","id":3,"type":{"family":"TimestampFamily","oid":1114},"defaultExpr":"now():::TIMESTAMP"},{"name":"payload","id":4,"type":{"family":"BytesFamily","oid":17}},{"name":"progress","id":5,"type":{"family":"BytesFamily","oid":17},"nullable":true},{"name":"created_by_type","id":6,"type":{"family":"StringFamily","oid":25},"nullable":true},{"name":"created_by_id","id":7,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"claim_session_id","id":8,"type":{"family":"BytesFamily","oid":17},"nullable":true},{"name":"claim_instance_id","id":9,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"num_runs","id":10,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"last_run","id":11,"type":{"family":"TimestampFamily","oid":1114},"nullable":true},{"name":"job_type","id":12,"type":{"family":"StringFamily","oid":25},"nullable":true}],"nextColumnId":13,"families":[{"name":"fam_0_id_status_created_payload","columnNames":["id","status","created","payload","created_by_type","created_by_id","job_type"],"columnIds":[1,2,3,4,6,7,12]},{"name":"progress","id":1,"columnNames":["progress"],"columnIds":[5],"defaultColumnId":5},{"name":"claim","id":2,"columnNames":["claim_session_id","claim_instance_id","num_runs","last_run"],"columnIds":[8,9,10,11]}],"nextFamilyId":3,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["id"],"keyColumnDirections":["ASC"],"storeColumnNames":["status","created","payload","progress","created_by_type","created_by_id","claim_session_id","claim_instance_id","num_runs","last_run","job_type"],"keyColumnIds":[1],"storeColumnIds":[2,3,4,5,6,7,8,9,10,11,12],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"indexes":[{"name":"jobs_status_created_idx","id":2,"version":3,"keyColumnNames":["status","created"],"keyColumnDirections":["ASC","ASC"],"keyColumnIds":[2,3],"keySuffixColumnIds":[1],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}},{"name":"jobs_created_by_type_created_by_id_idx","id":3,"version":3,"keyColumnNames":["created_by_type","created_by_id"],"keyColumnDirections":["ASC","ASC"],"storeColumnNames":["status"],"keyColumnIds":[6,7],"keySuffixColumnIds":[1],"storeColumnIds":[2],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}},{"name":"jobs_run_stats_idx","id":4,"version":3,"keyColumnNames":["claim_session_id","status","created"],"keyColumnDirections":["ASC","ASC","ASC"],"storeColumnNames":["last_run","num_runs","claim_instance_id"],"keyColumnIds":[8,2,3],"keySuffixColumnIds":[1],"storeColumnIds":[11,10,9],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{},"predicate":"status IN ('_':::STRING, '_':::STRING, '_':::STRING, '_':::STRING, '_':::STRING)"},{"name":"jobs_job_type_idx","id":5,"version":3,"keyColumnNames":["job_type"],"keyColumnDirections":["ASC"],"keyColumnIds":[12],"keySuffixColumnIds":[1],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}}],"nextIndexId":6,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} +{"table":{"name":"jobs","id":15,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"id","id":1,"type":{"family":"IntFamily","width":64,"oid":20},"defaultExpr":"unique_rowid()"},{"name":"status","id":2,"type":{"family":"StringFamily","oid":25}},{"name":"created","id":3,"type":{"family":"TimestampFamily","oid":1114},"defaultExpr":"now():::TIMESTAMP"},{"name":"payload","id":4,"type":{"family":"BytesFamily","oid":17},"nullable":true},{"name":"progress","id":5,"type":{"family":"BytesFamily","oid":17},"nullable":true},{"name":"created_by_type","id":6,"type":{"family":"StringFamily","oid":25},"nullable":true},{"name":"created_by_id","id":7,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"claim_session_id","id":8,"type":{"family":"BytesFamily","oid":17},"nullable":true},{"name":"claim_instance_id","id":9,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"num_runs","id":10,"type":{"family":"IntFamily","width":64,"oid":20},"nullable":true},{"name":"last_run","id":11,"type":{"family":"TimestampFamily","oid":1114},"nullable":true},{"name":"job_type","id":12,"type":{"family":"StringFamily","oid":25},"nullable":true}],"nextColumnId":13,"families":[{"name":"fam_0_id_status_created_payload","columnNames":["id","status","created","payload","created_by_type","created_by_id","job_type"],"columnIds":[1,2,3,4,6,7,12]},{"name":"progress","id":1,"columnNames":["progress"],"columnIds":[5],"defaultColumnId":5},{"name":"claim","id":2,"columnNames":["claim_session_id","claim_instance_id","num_runs","last_run"],"columnIds":[8,9,10,11]}],"nextFamilyId":3,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["id"],"keyColumnDirections":["ASC"],"storeColumnNames":["status","created","payload","progress","created_by_type","created_by_id","claim_session_id","claim_instance_id","num_runs","last_run","job_type"],"keyColumnIds":[1],"storeColumnIds":[2,3,4,5,6,7,8,9,10,11,12],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"indexes":[{"name":"jobs_status_created_idx","id":2,"version":3,"keyColumnNames":["status","created"],"keyColumnDirections":["ASC","ASC"],"keyColumnIds":[2,3],"keySuffixColumnIds":[1],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}},{"name":"jobs_created_by_type_created_by_id_idx","id":3,"version":3,"keyColumnNames":["created_by_type","created_by_id"],"keyColumnDirections":["ASC","ASC"],"storeColumnNames":["status"],"keyColumnIds":[6,7],"keySuffixColumnIds":[1],"storeColumnIds":[2],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}},{"name":"jobs_run_stats_idx","id":4,"version":3,"keyColumnNames":["claim_session_id","status","created"],"keyColumnDirections":["ASC","ASC","ASC"],"storeColumnNames":["last_run","num_runs","claim_instance_id"],"keyColumnIds":[8,2,3],"keySuffixColumnIds":[1],"storeColumnIds":[11,10,9],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{},"predicate":"status IN ('_':::STRING, '_':::STRING, '_':::STRING, '_':::STRING, '_':::STRING)"},{"name":"jobs_job_type_idx","id":5,"version":3,"keyColumnNames":["job_type"],"keyColumnDirections":["ASC"],"keyColumnIds":[12],"keySuffixColumnIds":[1],"foreignKey":{},"interleave":{},"partitioning":{},"sharded":{},"geoConfig":{}}],"nextIndexId":6,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} {"table":{"name":"join_tokens","id":41,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"id","id":1,"type":{"family":"UuidFamily","oid":2950}},{"name":"secret","id":2,"type":{"family":"BytesFamily","oid":17}},{"name":"expiration","id":3,"type":{"family":"TimestampTZFamily","oid":1184}}],"nextColumnId":4,"families":[{"name":"primary","columnNames":["id","secret","expiration"],"columnIds":[1,2,3]}],"nextFamilyId":1,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["id"],"keyColumnDirections":["ASC"],"storeColumnNames":["secret","expiration"],"keyColumnIds":[1],"storeColumnIds":[2,3],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":2,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} {"table":{"name":"lease","id":11,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"descID","id":1,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"version","id":2,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"nodeID","id":3,"type":{"family":"IntFamily","width":64,"oid":20}},{"name":"expiration","id":4,"type":{"family":"TimestampFamily","oid":1114}},{"name":"crdb_region","id":5,"type":{"family":"BytesFamily","oid":17}}],"nextColumnId":6,"families":[{"name":"primary","columnNames":["descID","version","nodeID","expiration","crdb_region"],"columnIds":[1,2,3,4,5]}],"nextFamilyId":1,"primaryIndex":{"name":"primary","id":2,"unique":true,"version":4,"keyColumnNames":["crdb_region","descID","version","expiration","nodeID"],"keyColumnDirections":["ASC","ASC","ASC","ASC","ASC"],"keyColumnIds":[5,1,2,4,3],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":3,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} {"table":{"name":"locations","id":21,"version":"1","modificationTime":{"wallTime":"0"},"parentId":1,"unexposedParentSchemaId":29,"columns":[{"name":"localityKey","id":1,"type":{"family":"StringFamily","oid":25}},{"name":"localityValue","id":2,"type":{"family":"StringFamily","oid":25}},{"name":"latitude","id":3,"type":{"family":"DecimalFamily","width":15,"precision":18,"oid":1700}},{"name":"longitude","id":4,"type":{"family":"DecimalFamily","width":15,"precision":18,"oid":1700}}],"nextColumnId":5,"families":[{"name":"fam_0_localityKey_localityValue_latitude_longitude","columnNames":["localityKey","localityValue","latitude","longitude"],"columnIds":[1,2,3,4]}],"nextFamilyId":1,"primaryIndex":{"name":"primary","id":1,"unique":true,"version":4,"keyColumnNames":["localityKey","localityValue"],"keyColumnDirections":["ASC","ASC"],"storeColumnNames":["latitude","longitude"],"keyColumnIds":[1,2],"storeColumnIds":[3,4],"foreignKey":{},"interleave":{},"partitioning":{},"encodingType":1,"sharded":{},"geoConfig":{},"constraintId":1},"nextIndexId":2,"privileges":{"users":[{"userProto":"admin","privileges":"480","withGrantOption":"480"},{"userProto":"root","privileges":"480","withGrantOption":"480"}],"ownerProto":"node","version":2},"nextMutationId":1,"formatVersion":3,"replacementOf":{"time":{}},"createAsOfTime":{"wallTime":"0"},"nextConstraintId":2}} diff --git a/pkg/sql/crdb_internal_test.go b/pkg/sql/crdb_internal_test.go index 02dd43ff2693..994ed75e6ef1 100644 --- a/pkg/sql/crdb_internal_test.go +++ b/pkg/sql/crdb_internal_test.go @@ -833,8 +833,8 @@ func TestInternalJobsTableRetryColumns(t *testing.T) { payloadBytes, err := protoutil.Marshal(&payload) assert.NoError(t, err) tdb.Exec(t, - "INSERT INTO system.jobs (id, status, created, payload) values ($1, $2, $3, $4)", - 1, jobs.StatusRunning, timeutil.Now(), payloadBytes, + "INSERT INTO system.jobs (id, status, created) values ($1, $2, $3)", + 1, jobs.StatusRunning, timeutil.Now(), ) tdb.Exec(t, "INSERT INTO system.job_info (job_id, info_key, value) values ($1, $2, $3)", @@ -1328,8 +1328,8 @@ func TestInternalSystemJobsTableMirrorsSystemJobsTable(t *testing.T) { assert.NoError(t, err) tdb.Exec(t, - "INSERT INTO system.jobs (id, status, created, payload) values ($1, $2, $3, $4)", - 1, jobs.StatusRunning, timeutil.Now(), payloadBytes, + "INSERT INTO system.jobs (id, status, created) values ($1, $2, $3)", + 1, jobs.StatusRunning, timeutil.Now(), ) tdb.Exec(t, "INSERT INTO system.job_info (job_id, info_key, value) values ($1, $2, $3)", @@ -1337,8 +1337,9 @@ func TestInternalSystemJobsTableMirrorsSystemJobsTable(t *testing.T) { ) tdb.Exec(t, - "INSERT INTO system.jobs values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", - 2, jobs.StatusRunning, timeutil.Now(), payloadBytes, []byte("progress"), "created by", 2, []byte("claim session id"), + `INSERT INTO system.jobs (id, status, created, created_by_type, created_by_id, + claim_session_id, claim_instance_id, num_runs, last_run, job_type) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + 2, jobs.StatusRunning, timeutil.Now(), "created by", 2, []byte("claim session id"), 2, 2, timeutil.Now(), jobspb.TypeImport.String(), ) tdb.Exec(t, @@ -1350,20 +1351,19 @@ func TestInternalSystemJobsTableMirrorsSystemJobsTable(t *testing.T) { 2, jobs.GetLegacyProgressKey(), []byte("progress"), ) - res := tdb.QueryStr(t, "SELECT * FROM system.jobs ORDER BY id") - tdb.CheckQueryResults(t, `SELECT * FROM crdb_internal.system_jobs ORDER BY id`, res) + res := tdb.QueryStr(t, ` + SELECT id, status, created, created_by_type, created_by_id, claim_session_id, + claim_instance_id, num_runs, last_run, job_type + FROM system.jobs ORDER BY id`, + ) tdb.CheckQueryResults(t, ` - SELECT id, status, created, payload, progress, created_by_type, created_by_id, claim_session_id, + SELECT id, status, created, created_by_type, created_by_id, claim_session_id, claim_instance_id, num_runs, last_run, job_type FROM crdb_internal.system_jobs ORDER BY id`, res, ) - tdb.CheckQueryResults(t, ` - SELECT id, status, created, payload, progress, created_by_type, created_by_id, claim_session_id, - claim_instance_id, num_runs, last_run, job_type - FROM system.jobs ORDER BY id`, - res, - ) + + // TODO(adityamaru): add checks for payload and progress } // TestInternalSystemJobsTableWorksWithVersionPreV23_1BackfillTypeColumnInJobsTable @@ -1435,8 +1435,8 @@ func TestCorruptPayloadError(t *testing.T) { tdb := sqlutils.MakeSQLRunner(db) tdb.Exec(t, - "INSERT INTO system.jobs (id, status, created, payload) values ($1, $2, $3, $4)", - 1, jobs.StatusRunning, timeutil.Now(), []byte("invalid payload"), + "INSERT INTO system.jobs (id, status, created) values ($1, $2, $3)", + 1, jobs.StatusRunning, timeutil.Now(), ) tdb.Exec(t, "INSERT INTO system.job_info (job_id, info_key, value) values ($1, $2, $3)", diff --git a/pkg/sql/delegate/job_control.go b/pkg/sql/delegate/job_control.go index d1d2b2ca0d53..d5aa83fdb7ab 100644 --- a/pkg/sql/delegate/job_control.go +++ b/pkg/sql/delegate/job_control.go @@ -91,7 +91,7 @@ AND jobs.status IN (%s) AND jobs.created_by_id IN (%s)`, payload, false, true )->'%s' ) IS NOT NULL AS correct_type - FROM system.jobs + FROM crdb_internal.system_jobs WHERE status IN (%s) ) WHERE correct_type diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog b/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog index 0ac51101924a..9531fcc613e5 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog @@ -114,7 +114,7 @@ SELECT id, strip_volatile(descriptor) FROM crdb_internal.kv_catalog_descriptor 12 {"table": {"columns": [{"id": 1, "name": "timestamp", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 2, "name": "eventType", "type": {"family": "StringFamily", "oid": 25}}, {"id": 3, "name": "targetID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "reportingID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 5, "name": "info", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"defaultExpr": "uuid_v4()", "id": 6, "name": "uniqueID", "type": {"family": "BytesFamily", "oid": 17}}], "formatVersion": 3, "id": 12, "name": "eventlog", "nextColumnId": 7, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [1, 6], "keyColumnNames": ["timestamp", "uniqueID"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4, 5], "storeColumnNames": ["eventType", "targetID", "reportingID", "info"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} 13 {"table": {"columns": [{"id": 1, "name": "timestamp", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 2, "name": "rangeID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "storeID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "eventType", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "otherRangeID", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 6, "name": "info", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"defaultExpr": "unique_rowid()", "id": 7, "name": "uniqueID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 13, "name": "rangelog", "nextColumnId": 8, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [1, 7], "keyColumnNames": ["timestamp", "uniqueID"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4, 5, 6], "storeColumnNames": ["rangeID", "storeID", "eventType", "otherRangeID", "info"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} 14 {"table": {"columns": [{"id": 1, "name": "key", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "value", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 3, "name": "lastUpdated", "type": {"family": "TimestampFamily", "oid": 1114}}], "formatVersion": 3, "id": 14, "name": "ui", "nextColumnId": 4, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["key"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3], "storeColumnNames": ["value", "lastUpdated"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} -15 {"table": {"columns": [{"defaultExpr": "unique_rowid()", "id": 1, "name": "id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"defaultExpr": "now():::TIMESTAMP", "id": 3, "name": "created", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 4, "name": "payload", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 5, "name": "progress", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "created_by_type", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "created_by_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 8, "name": "claim_session_id", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 9, "name": "claim_instance_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 10, "name": "num_runs", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 11, "name": "last_run", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "job_type", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 15, "indexes": [{"foreignKey": {}, "geoConfig": {}, "id": 2, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [2, 3], "keyColumnNames": ["status", "created"], "keySuffixColumnIds": [1], "name": "jobs_status_created_idx", "partitioning": {}, "sharded": {}, "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 3, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [6, 7], "keyColumnNames": ["created_by_type", "created_by_id"], "keySuffixColumnIds": [1], "name": "jobs_created_by_type_created_by_id_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [2], "storeColumnNames": ["status"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 4, "interleave": {}, "keyColumnDirections": ["ASC", "ASC", "ASC"], "keyColumnIds": [8, 2, 3], "keyColumnNames": ["claim_session_id", "status", "created"], "keySuffixColumnIds": [1], "name": "jobs_run_stats_idx", "partitioning": {}, "predicate": "status IN ('running':::STRING, 'reverting':::STRING, 'pending':::STRING, 'pause-requested':::STRING, 'cancel-requested':::STRING)", "sharded": {}, "storeColumnIds": [11, 10, 9], "storeColumnNames": ["last_run", "num_runs", "claim_instance_id"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 5, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [12], "keyColumnNames": ["job_type"], "keySuffixColumnIds": [1], "name": "jobs_job_type_idx", "partitioning": {}, "sharded": {}, "version": 3}], "name": "jobs", "nextColumnId": 13, "nextConstraintId": 2, "nextIndexId": 6, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["id"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "storeColumnNames": ["status", "created", "payload", "progress", "created_by_type", "created_by_id", "claim_session_id", "claim_instance_id", "num_runs", "last_run", "job_type"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} +15 {"table": {"columns": [{"defaultExpr": "unique_rowid()", "id": 1, "name": "id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"defaultExpr": "now():::TIMESTAMP", "id": 3, "name": "created", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 4, "name": "payload", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 5, "name": "progress", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "created_by_type", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "created_by_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 8, "name": "claim_session_id", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"id": 9, "name": "claim_instance_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 10, "name": "num_runs", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 11, "name": "last_run", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "job_type", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 15, "indexes": [{"foreignKey": {}, "geoConfig": {}, "id": 2, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [2, 3], "keyColumnNames": ["status", "created"], "keySuffixColumnIds": [1], "name": "jobs_status_created_idx", "partitioning": {}, "sharded": {}, "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 3, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [6, 7], "keyColumnNames": ["created_by_type", "created_by_id"], "keySuffixColumnIds": [1], "name": "jobs_created_by_type_created_by_id_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [2], "storeColumnNames": ["status"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 4, "interleave": {}, "keyColumnDirections": ["ASC", "ASC", "ASC"], "keyColumnIds": [8, 2, 3], "keyColumnNames": ["claim_session_id", "status", "created"], "keySuffixColumnIds": [1], "name": "jobs_run_stats_idx", "partitioning": {}, "predicate": "status IN ('running':::STRING, 'reverting':::STRING, 'pending':::STRING, 'pause-requested':::STRING, 'cancel-requested':::STRING)", "sharded": {}, "storeColumnIds": [11, 10, 9], "storeColumnNames": ["last_run", "num_runs", "claim_instance_id"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 5, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [12], "keyColumnNames": ["job_type"], "keySuffixColumnIds": [1], "name": "jobs_job_type_idx", "partitioning": {}, "sharded": {}, "version": 3}], "name": "jobs", "nextColumnId": 13, "nextConstraintId": 2, "nextIndexId": 6, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["id"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "storeColumnNames": ["status", "created", "payload", "progress", "created_by_type", "created_by_id", "claim_session_id", "claim_instance_id", "num_runs", "last_run", "job_type"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} 19 {"table": {"columns": [{"defaultExpr": "unique_rowid()", "id": 1, "name": "id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "hashedSecret", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 3, "name": "username", "type": {"family": "StringFamily", "oid": 25}}, {"defaultExpr": "now():::TIMESTAMP", "id": 4, "name": "createdAt", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 5, "name": "expiresAt", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 6, "name": "revokedAt", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"defaultExpr": "now():::TIMESTAMP", "id": 7, "name": "lastUsedAt", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 8, "name": "auditInfo", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "user_id", "type": {"family": "OidFamily", "oid": 26}}], "formatVersion": 3, "id": 19, "indexes": [{"foreignKey": {}, "geoConfig": {}, "id": 2, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [5], "keyColumnNames": ["expiresAt"], "keySuffixColumnIds": [1], "name": "web_sessions_expiresAt_idx", "partitioning": {}, "sharded": {}, "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 3, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [4], "keyColumnNames": ["createdAt"], "keySuffixColumnIds": [1], "name": "web_sessions_createdAt_idx", "partitioning": {}, "sharded": {}, "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 4, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [6], "keyColumnNames": ["revokedAt"], "keySuffixColumnIds": [1], "name": "web_sessions_revokedAt_idx", "partitioning": {}, "sharded": {}, "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 5, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [7], "keyColumnNames": ["lastUsedAt"], "keySuffixColumnIds": [1], "name": "web_sessions_lastUsedAt_idx", "partitioning": {}, "sharded": {}, "version": 3}], "name": "web_sessions", "nextColumnId": 10, "nextConstraintId": 2, "nextIndexId": 6, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["id"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4, 5, 6, 7, 8, 9], "storeColumnNames": ["hashedSecret", "username", "createdAt", "expiresAt", "revokedAt", "lastUsedAt", "auditInfo", "user_id"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} 20 {"table": {"columns": [{"id": 1, "name": "tableID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"defaultExpr": "unique_rowid()", "id": 2, "name": "statisticID", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "columnIDs", "type": {"arrayContents": {"family": "IntFamily", "oid": 20, "width": 64}, "arrayElemType": "IntFamily", "family": "ArrayFamily", "oid": 1016, "width": 64}}, {"defaultExpr": "now():::TIMESTAMP", "id": 5, "name": "createdAt", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 6, "name": "rowCount", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 7, "name": "distinctCount", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 8, "name": "nullCount", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 9, "name": "histogram", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}, {"defaultExpr": "0:::INT8", "id": 10, "name": "avgSize", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 11, "name": "partialPredicate", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 12, "name": "fullStatisticID", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 20, "name": "table_statistics", "nextColumnId": 13, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [1, 2], "keyColumnNames": ["tableID", "statisticID"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "storeColumnNames": ["name", "columnIDs", "createdAt", "rowCount", "distinctCount", "nullCount", "histogram", "avgSize", "partialPredicate", "fullStatisticID"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} 21 {"table": {"columns": [{"id": 1, "name": "localityKey", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "localityValue", "type": {"family": "StringFamily", "oid": 25}}, {"id": 3, "name": "latitude", "type": {"family": "DecimalFamily", "oid": 1700, "precision": 18, "width": 15}}, {"id": 4, "name": "longitude", "type": {"family": "DecimalFamily", "oid": 1700, "precision": 18, "width": 15}}], "formatVersion": 3, "id": 21, "name": "locations", "nextColumnId": 5, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC", "ASC"], "keyColumnIds": [1, 2], "keyColumnNames": ["localityKey", "localityValue"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [3, 4], "storeColumnNames": ["latitude", "longitude"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "480", "userProto": "admin", "withGrantOption": "480"}, {"privileges": "480", "userProto": "root", "withGrantOption": "480"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}} diff --git a/pkg/sql/logictest/testdata/logic_test/create_index b/pkg/sql/logictest/testdata/logic_test/create_index index bca10b313251..5b03c4fb1528 100644 --- a/pkg/sql/logictest/testdata/logic_test/create_index +++ b/pkg/sql/logictest/testdata/logic_test/create_index @@ -1,4 +1,4 @@ -# LogicTest: default-configs local-mixed-22.2-23.1 +# LogicTest: default-configs statement ok CREATE TABLE t ( @@ -284,7 +284,7 @@ let $spans SELECT j->'schemaChange'->'resumeSpanList'->0->'resumeSpans'->0 AS span FROM ( SELECT crdb_internal.pb_to_json('payload', payload) AS j - FROM system.jobs + FROM crdb_internal.system_jobs ) WHERE j->>'description' LIKE 'CREATE INDEX pauseidx%' ), @@ -329,16 +329,16 @@ SELECT json_build_array( # sure that on resume it re-keys the spans correctly. We pretty_key these below # to confirm/show what is in them. statement ok -UPDATE system.jobs - SET payload = crdb_internal.json_to_pb( +UPDATE system.job_info + SET value = crdb_internal.json_to_pb( 'cockroach.sql.jobs.jobspb.Payload', json_set( - crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload), + crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', value), ARRAY['schemaChange', 'resumeSpanList', '0'], '{"resumeSpans": $spans}'::jsonb ) ) -WHERE crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload)->>'description' LIKE 'CREATE INDEX pauseidx%'; +WHERE info_key = 'legacy_payload'::BYTES AND crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', value)->>'description' LIKE 'CREATE INDEX pauseidx%'; # confirm we see these bogus start and end keys in the job, both for the wrong # tenant and for no tenant. @@ -349,7 +349,7 @@ SELECT crdb_internal.pretty_key(decode(j->'schemaChange'->'resumeSpanList'->0->'resumeSpans'->1->>'key', 'base64'), 0), crdb_internal.pretty_key(decode(j->'schemaChange'->'resumeSpanList'->0->'resumeSpans'->1->>'endKey', 'base64'), 0) FROM ( - SELECT crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload) j FROM system.jobs + SELECT crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload) j FROM crdb_internal.system_jobs ) WHERE j->>'description' LIKE 'CREATE INDEX pauseidx%'; ---- /103/Table/114/1 /103/Table/114/1/100 /114/1/100 /114/2 diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index 34a3b2820331..f4424e891664 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -1556,7 +1556,6 @@ system public primary system public 29_15_1_not_null system public jobs CHECK NO NO system public 29_15_2_not_null system public jobs CHECK NO NO system public 29_15_3_not_null system public jobs CHECK NO NO -system public 29_15_4_not_null system public jobs CHECK NO NO system public primary system public jobs PRIMARY KEY NO NO system public 29_41_1_not_null system public join_tokens CHECK NO NO system public 29_41_2_not_null system public join_tokens CHECK NO NO @@ -1856,7 +1855,6 @@ system public 29_14_3_not_null system public 29_15_1_not_null id IS NOT NULL system public 29_15_2_not_null status IS NOT NULL system public 29_15_3_not_null created IS NOT NULL -system public 29_15_4_not_null payload IS NOT NULL system public 29_19_1_not_null id IS NOT NULL system public 29_19_2_not_null hashedSecret IS NOT NULL system public 29_19_3_not_null username IS NOT NULL diff --git a/pkg/sql/logictest/testdata/logic_test/system b/pkg/sql/logictest/testdata/logic_test/system index 8ccf338b4351..2ec71f669396 100644 --- a/pkg/sql/logictest/testdata/logic_test/system +++ b/pkg/sql/logictest/testdata/logic_test/system @@ -316,7 +316,7 @@ SHOW COLUMNS FROM system.jobs id INT8 false unique_rowid() · {jobs_created_by_type_created_by_id_idx,jobs_job_type_idx,jobs_run_stats_idx,jobs_status_created_idx,primary} false status STRING false NULL · {jobs_created_by_type_created_by_id_idx,jobs_run_stats_idx,jobs_status_created_idx,primary} false created TIMESTAMP false now() · {jobs_run_stats_idx,jobs_status_created_idx,primary} false -payload BYTES false NULL · {primary} false +payload BYTES true NULL · {primary} false progress BYTES true NULL · {primary} false created_by_type STRING true NULL · {jobs_created_by_type_created_by_id_idx,primary} false created_by_id INT8 true NULL · {jobs_created_by_type_created_by_id_idx,primary} false diff --git a/pkg/sql/logictest/tests/local-mixed-22.2-23.1/BUILD.bazel b/pkg/sql/logictest/tests/local-mixed-22.2-23.1/BUILD.bazel index 1b985d842f1f..fe1bc2d3dfb5 100644 --- a/pkg/sql/logictest/tests/local-mixed-22.2-23.1/BUILD.bazel +++ b/pkg/sql/logictest/tests/local-mixed-22.2-23.1/BUILD.bazel @@ -10,7 +10,7 @@ go_test( "//c-deps:libgeos", # keep "//pkg/sql/logictest:testdata", # keep ], - shard_count = 13, + shard_count = 12, tags = [ "cpu:1", ], diff --git a/pkg/sql/logictest/tests/local-mixed-22.2-23.1/generated_test.go b/pkg/sql/logictest/tests/local-mixed-22.2-23.1/generated_test.go index 0205120baf7a..157a2871a1d7 100644 --- a/pkg/sql/logictest/tests/local-mixed-22.2-23.1/generated_test.go +++ b/pkg/sql/logictest/tests/local-mixed-22.2-23.1/generated_test.go @@ -93,13 +93,6 @@ func TestLogic_comment_on( runLogicTest(t, "comment_on") } -func TestLogic_create_index( - t *testing.T, -) { - defer leaktest.AfterTest(t)() - runLogicTest(t, "create_index") -} - func TestLogic_drop_database( t *testing.T, ) { diff --git a/pkg/sql/row/expr_walker_test.go b/pkg/sql/row/expr_walker_test.go index 1601ad3c2145..0ad01c19546c 100644 --- a/pkg/sql/row/expr_walker_test.go +++ b/pkg/sql/row/expr_walker_test.go @@ -210,7 +210,7 @@ func TestJobBackedSeqChunkProvider(t *testing.T) { CurVal: 0, } require.NoError(t, j.RequestChunk(ctx, evalCtx, annot, seqMetadata)) - getJobProgressQuery := `SELECT progress FROM system.jobs J WHERE J.id = $1` + getJobProgressQuery := `SELECT progress FROM crdb_internal.system_jobs J WHERE J.id = $1` var progressBytes []byte require.NoError(t, sqlDB.QueryRow(getJobProgressQuery, job.ID()).Scan(&progressBytes)) diff --git a/pkg/sql/schema_changer_test.go b/pkg/sql/schema_changer_test.go index 80a23bc0c7c4..2523353ee625 100644 --- a/pkg/sql/schema_changer_test.go +++ b/pkg/sql/schema_changer_test.go @@ -4584,7 +4584,7 @@ func TestIndexBackfillAfterGC(t *testing.T) { got := sqlDB.QueryStr(t, ` SELECT p->'schemaChange'->'writeTimestamp'->>'wallTime' < $1, jsonb_pretty(p) - FROM (SELECT crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload) AS p FROM system.jobs) + FROM (SELECT crdb_internal.pb_to_json('cockroach.sql.jobs.jobspb.Payload', payload) AS p FROM crdb_internal.system_jobs) WHERE p->>'description' LIKE 'CREATE UNIQUE INDEX index_created_in_test%'`, gcAt.WallTime, )[0] diff --git a/pkg/sql/schemachanger/schemachanger_test.go b/pkg/sql/schemachanger/schemachanger_test.go index 58064023df6d..07258c6aedb0 100644 --- a/pkg/sql/schemachanger/schemachanger_test.go +++ b/pkg/sql/schemachanger/schemachanger_test.go @@ -773,7 +773,7 @@ func TestSchemaChangerJobErrorDetails(t *testing.T) { require.Regexp(t, "^stages graphviz: https.*", ed[1]) require.Regexp(t, "^dependencies graphviz: https.*", ed[2]) } - results = tdb.QueryStr(t, `SELECT encode(payload, 'hex') FROM system.jobs WHERE id = $1`, jobID) + results = tdb.QueryStr(t, `SELECT encode(payload, 'hex') FROM crdb_internal.system_jobs WHERE id = $1`, jobID) require.Len(t, results, 1) b, err := hex.DecodeString(results[0][0]) require.NoError(t, err) diff --git a/pkg/sql/ttl/ttljob/ttljob_test.go b/pkg/sql/ttl/ttljob/ttljob_test.go index cb5173abd1a5..277e0fa51757 100644 --- a/pkg/sql/ttl/ttljob/ttljob_test.go +++ b/pkg/sql/ttl/ttljob/ttljob_test.go @@ -201,9 +201,8 @@ func (h *rowLevelTTLTestJobTestHelper) verifyExpiredRowsJobOnly( t *testing.T, expectedNumExpiredRows int, ) { rows := h.sqlDB.Query(t, ` - SELECT sys_j.status, sys_j.progress - FROM crdb_internal.jobs AS crdb_j - JOIN system.jobs as sys_j ON crdb_j.job_id = sys_j.id + SELECT crdb_j.status, crdb_j.progress + FROM crdb_internal.system_jobs AS crdb_j WHERE crdb_j.job_type = 'ROW LEVEL TTL' `) jobCount := 0 @@ -233,9 +232,8 @@ func (h *rowLevelTTLTestJobTestHelper) verifyExpiredRows( t *testing.T, expectedSQLInstanceIDToProcessorMap map[base.SQLInstanceID]*processor, ) { rows := h.sqlDB.Query(t, ` - SELECT sys_j.status, sys_j.progress - FROM crdb_internal.jobs AS crdb_j - JOIN system.jobs as sys_j ON crdb_j.job_id = sys_j.id + SELECT crdb_j.status, crdb_j.progress + FROM crdb_internal.system_jobs AS crdb_j WHERE crdb_j.job_type = 'ROW LEVEL TTL' `) jobCount := 0 diff --git a/pkg/testutils/jobutils/jobs_verification.go b/pkg/testutils/jobutils/jobs_verification.go index b56ea316a56e..b820a4ccaebd 100644 --- a/pkg/testutils/jobutils/jobs_verification.go +++ b/pkg/testutils/jobutils/jobs_verification.go @@ -264,7 +264,7 @@ func GetLastJobID(t testing.TB, db *sqlutils.SQLRunner) jobspb.JobID { func GetJobProgress(t *testing.T, db *sqlutils.SQLRunner, jobID jobspb.JobID) *jobspb.Progress { ret := &jobspb.Progress{} var buf []byte - db.QueryRow(t, `SELECT progress FROM system.jobs WHERE id = $1`, jobID).Scan(&buf) + db.QueryRow(t, `SELECT progress FROM crdb_internal.system_jobs WHERE id = $1`, jobID).Scan(&buf) if err := protoutil.Unmarshal(buf, ret); err != nil { t.Fatal(err) } diff --git a/pkg/upgrade/upgrademanager/manager.go b/pkg/upgrade/upgrademanager/manager.go index 4a71604d3ec1..9b56927b561a 100644 --- a/pkg/upgrade/upgrademanager/manager.go +++ b/pkg/upgrade/upgrademanager/manager.go @@ -227,7 +227,7 @@ func (m *Manager) RunPermanentUpgrades(ctx context.Context, upToVersion roachpb. } else { // The stale read said that the upgrades had not run. Let's try a consistent read too since log.Infof(ctx, - "the last last permanent upgrade (v%s) does not appear to have completed; attempting to run all upgrades", + "the last permanent upgrade (v%s) does not appear to have completed; attempting to run all upgrades", lastVer) } @@ -756,7 +756,7 @@ SELECT id, status payload, false -- emit_defaults ) AS pl - FROM system.jobs + FROM crdb_internal.system_jobs WHERE status IN ` + jobs.NonTerminalStatusTupleString + ` ) WHERE pl->'migration'->'clusterVersion' = $1::JSON;` diff --git a/pkg/upgrade/upgrades/system_job_info.go b/pkg/upgrade/upgrades/system_job_info.go index 42886d0ec4d8..71aa80027a6b 100644 --- a/pkg/upgrade/upgrades/system_job_info.go +++ b/pkg/upgrade/upgrades/system_job_info.go @@ -15,6 +15,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/upgrade" ) @@ -26,3 +27,17 @@ func systemJobInfoTableMigration( ctx, d.DB.KV(), d.Settings, d.Codec, systemschema.SystemJobInfoTable, ) } + +const alterPayloadToNullableQuery = ` +ALTER TABLE system.jobs ALTER COLUMN payload DROP NOT NULL +` + +// alterPayloadColumnToNullable runs a schema change to drop the NOT NULL +// constraint on the system.jobs payload column. +func alterPayloadColumnToNullable( + ctx context.Context, _ clusterversion.ClusterVersion, d upgrade.TenantDeps, +) error { + _, err := d.InternalExecutor.ExecEx(ctx, "set-job-payload-nullable", nil, + sessiondata.NodeUserSessionDataOverride, alterPayloadToNullableQuery) + return err +} diff --git a/pkg/upgrade/upgrades/upgrades.go b/pkg/upgrade/upgrades/upgrades.go index 0e901736962a..2cd0c300df62 100644 --- a/pkg/upgrade/upgrades/upgrades.go +++ b/pkg/upgrade/upgrades/upgrades.go @@ -304,6 +304,12 @@ var upgrades = []upgradebase.Upgrade{ upgrade.NoPrecondition, systemStatisticsActivityTableMigration, ), + upgrade.NewTenantUpgrade( + "stop writing payload and progress to system.jobs", + toCV(clusterversion.V23_2StopWritingPayloadAndProgressToSystemJobs), + upgrade.NoPrecondition, + alterPayloadColumnToNullable, + ), } func init() { From 2b1482e39aa088408441a5a9cfcfc36e8d6e983c Mon Sep 17 00:00:00 2001 From: ajwerner Date: Mon, 27 Mar 2023 11:12:57 -0400 Subject: [PATCH 3/5] sql/catalog: add String and SafeFormat methods to DescriptorIDSet Epic: none Release note: None --- pkg/sql/catalog/BUILD.bazel | 1 + pkg/sql/catalog/descriptor_id_set.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/sql/catalog/BUILD.bazel b/pkg/sql/catalog/BUILD.bazel index 257de924276d..0e8c681059c8 100644 --- a/pkg/sql/catalog/BUILD.bazel +++ b/pkg/sql/catalog/BUILD.bazel @@ -46,6 +46,7 @@ go_library( "//pkg/util/iterutil", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_redact//:redact", + "@com_github_cockroachdb_redact//interfaces", ], ) diff --git a/pkg/sql/catalog/descriptor_id_set.go b/pkg/sql/catalog/descriptor_id_set.go index 07eb0cac6198..ed56cc96fa48 100644 --- a/pkg/sql/catalog/descriptor_id_set.go +++ b/pkg/sql/catalog/descriptor_id_set.go @@ -13,6 +13,8 @@ package catalog import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/util/intsets" + "github.com/cockroachdb/redact" + "github.com/cockroachdb/redact/interfaces" ) // DescriptorIDSet efficiently stores an unordered set of descriptor ids. @@ -20,6 +22,16 @@ type DescriptorIDSet struct { set intsets.Fast } +// SafeFormat implements SafeFormatter for DescriptorIDSet. +func (d *DescriptorIDSet) SafeFormat(s interfaces.SafePrinter, verb rune) { + s.SafeString(redact.SafeString(d.String())) +} + +// String implement fmt.Stringer for DescriptorIDSet. +func (d *DescriptorIDSet) String() string { + return d.set.String() +} + // MakeDescriptorIDSet returns a set initialized with the given values. func MakeDescriptorIDSet(ids ...descpb.ID) DescriptorIDSet { s := DescriptorIDSet{} @@ -32,6 +44,8 @@ func MakeDescriptorIDSet(ids ...descpb.ID) DescriptorIDSet { // Suppress the linter. var _ = MakeDescriptorIDSet +var _ redact.SafeFormatter = (*DescriptorIDSet)(nil) + // Add adds an id to the set. No-op if the id is already in the set. func (d *DescriptorIDSet) Add(id descpb.ID) { d.set.Add(int(id)) From 8d7babc8247ea899bca63334d900990f96520b9c Mon Sep 17 00:00:00 2001 From: ajwerner Date: Mon, 27 Mar 2023 11:14:15 -0400 Subject: [PATCH 4/5] sql: do not drop table descriptor independently if we're in drop schema If we have dropped schema IDs, we know that this is not an individual drop table schema change. We only have more than one dropped table when we drop a database or a schema. Before this change, we'd drop the table on its own, and then create another GC job to drop all the tables. This is not actually a bug because we should be robust to this, but it's also bad. Epic: none Release note (bug fix): DROP SCHEMA ... CASCADE could create multiple GC jobs: one for every table and one for the cascaded drop itself. This has been fixed. --- pkg/sql/schema_changer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sql/schema_changer.go b/pkg/sql/schema_changer.go index 8d0e3114248d..14899c474b56 100644 --- a/pkg/sql/schema_changer.go +++ b/pkg/sql/schema_changer.go @@ -544,6 +544,8 @@ func (sc *SchemaChanger) execLogTags() *logtags.Buffer { } if sc.droppedDatabaseID != descpb.InvalidID { buf = buf.Add("db", sc.droppedDatabaseID) + } else if !sc.droppedSchemaIDs.Empty() { + buf = buf.Add("schema", sc.droppedSchemaIDs) } return buf } @@ -720,7 +722,7 @@ func (sc *SchemaChanger) exec(ctx context.Context) error { } // Otherwise, continue with the rest of the schema change state machine. - if tableDesc.Dropped() && sc.droppedDatabaseID == descpb.InvalidID { + if tableDesc.Dropped() && sc.droppedDatabaseID == descpb.InvalidID && sc.droppedSchemaIDs.Empty() { if tableDesc.IsPhysicalTable() { // We've dropped this physical table, let's kick off a GC job. dropTime := timeutil.Now().UnixNano() From fb2a3ccf7be79c323f522ab042eb061581e1d2c9 Mon Sep 17 00:00:00 2001 From: ajwerner Date: Mon, 27 Mar 2023 11:25:02 -0400 Subject: [PATCH 5/5] sql/gc_job,sqlerrors: make GC job robust to missing descriptors The check used for missing descriptors became incorrect in the course of https://github.com/cockroachdb/cockroach/pull/94695. That change updated the underlying error code used in getters by the GC job. The GC job would subsequently retry forever when the descriptor was missing. This bug has not been shipped yet, so not writing a release note. Fixes: #99590 Release note: None --- pkg/sql/gcjob/gc_job.go | 14 +++++++- pkg/sql/gcjob/index_garbage_collection.go | 11 +++---- pkg/sql/gcjob/refresh_statuses.go | 2 +- pkg/sql/gcjob/table_garbage_collection.go | 4 +-- pkg/sql/gcjob_test/gc_job_test.go | 40 +++++++++++++++-------- pkg/sql/sem/builtins/BUILD.bazel | 1 + pkg/sql/sem/builtins/pg_builtins.go | 9 +++-- pkg/sql/sqlerrors/errors.go | 15 +++++++++ 8 files changed, 66 insertions(+), 30 deletions(-) diff --git a/pkg/sql/gcjob/gc_job.go b/pkg/sql/gcjob/gc_job.go index 8e5fdee0c36b..f6ae37d73646 100644 --- a/pkg/sql/gcjob/gc_job.go +++ b/pkg/sql/gcjob/gc_job.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" "github.com/cockroachdb/cockroach/pkg/sql/isql" + "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -137,7 +138,7 @@ func deleteTableData( table, err = col.ByID(txn.KV()).Get().Table(ctx, droppedTable.ID) return err }); err != nil { - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { // This can happen if another GC job created for the same table got to // the table first. See #50344. log.Warningf(ctx, "table descriptor %d not found while attempting to GC, skipping", droppedTable.ID) @@ -566,6 +567,17 @@ func waitForWork( return ctx.Err() } +// isMissingDescriptorError checks whether the error has a code corresponding +// to a missing descriptor or if there is a lower-level catalog error with +// the same meaning. +// +// TODO(ajwerner,postamar): Nail down when we expect the lower-level error +// and tighten up the collection. +func isMissingDescriptorError(err error) bool { + return errors.Is(err, catalog.ErrDescriptorNotFound) || + sqlerrors.IsMissingDescriptorError(err) +} + // OnFailOrCancel is part of the jobs.Resumer interface. func (r schemaChangeGCResumer) OnFailOrCancel(context.Context, interface{}, error) error { return nil diff --git a/pkg/sql/gcjob/index_garbage_collection.go b/pkg/sql/gcjob/index_garbage_collection.go index cb64c80b5aa8..7c195bcc58cf 100644 --- a/pkg/sql/gcjob/index_garbage_collection.go +++ b/pkg/sql/gcjob/index_garbage_collection.go @@ -23,7 +23,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" "github.com/cockroachdb/cockroach/pkg/sql/isql" - "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" ) @@ -45,7 +44,7 @@ func deleteIndexData( // are no longer in use. This is necessary in the case of truncate, where we // schedule a GC Job in the transaction that commits the truncation. parentDesc, err := sql.WaitToUpdateLeases(ctx, execCfg.LeaseManager, parentID) - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { handleTableDescriptorDeleted(ctx, parentID, progress) return nil } @@ -94,7 +93,7 @@ func gcIndexes( // are no longer in use. This is necessary in the case of truncate, where we // schedule a GC Job in the transaction that commits the truncation. parentDesc, err := sql.WaitToUpdateLeases(ctx, execCfg.LeaseManager, parentID) - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { handleTableDescriptorDeleted(ctx, parentID, progress) return nil } @@ -131,8 +130,7 @@ func gcIndexes( ) } err := sql.DescsTxn(ctx, execCfg, removeIndexZoneConfigs) - if errors.Is(err, catalog.ErrDescriptorNotFound) || - sqlerrors.IsUndefinedRelationError(err) { + if isMissingDescriptorError(err) { handleTableDescriptorDeleted(ctx, parentID, progress) return nil } @@ -213,8 +211,7 @@ func deleteIndexZoneConfigsAfterGC( } err := sql.DescsTxn(ctx, execCfg, removeIndexZoneConfigs) switch { - case errors.Is(err, catalog.ErrDescriptorNotFound), - sqlerrors.IsUndefinedRelationError(err): + case isMissingDescriptorError(err): log.Infof(ctx, "removing index %d zone config from table %d failed: %v", index.IndexID, parentID, err) case err != nil: diff --git a/pkg/sql/gcjob/refresh_statuses.go b/pkg/sql/gcjob/refresh_statuses.go index 2b9549a54481..350003be1c05 100644 --- a/pkg/sql/gcjob/refresh_statuses.go +++ b/pkg/sql/gcjob/refresh_statuses.go @@ -138,7 +138,7 @@ func updateStatusForGCElements( return nil }); err != nil { - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { log.Warningf(ctx, "table %d not found, marking as GC'd", tableID) markTableGCed(ctx, tableID, progress, jobspb.SchemaChangeGCProgress_CLEARED) return false, true, maxDeadline diff --git a/pkg/sql/gcjob/table_garbage_collection.go b/pkg/sql/gcjob/table_garbage_collection.go index 2edbf2bdb627..b9edd5fed38f 100644 --- a/pkg/sql/gcjob/table_garbage_collection.go +++ b/pkg/sql/gcjob/table_garbage_collection.go @@ -54,7 +54,7 @@ func gcTables( table, err = col.ByID(txn.KV()).Get().Table(ctx, droppedTable.ID) return err }); err != nil { - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { // This can happen if another GC job created for the same table got to // the table first. See #50344. log.Warningf(ctx, "table descriptor %d not found while attempting to GC, skipping", droppedTable.ID) @@ -293,7 +293,7 @@ func deleteTableDescriptorsAfterGC( table, err = col.ByID(txn.KV()).Get().Table(ctx, droppedTable.ID) return err }); err != nil { - if errors.Is(err, catalog.ErrDescriptorNotFound) { + if isMissingDescriptorError(err) { // This can happen if another GC job created for the same table got to // the table first. See #50344. log.Warningf(ctx, "table descriptor %d not found while attempting to GC, skipping", droppedTable.ID) diff --git a/pkg/sql/gcjob_test/gc_job_test.go b/pkg/sql/gcjob_test/gc_job_test.go index 643128618946..768bac9fcec5 100644 --- a/pkg/sql/gcjob_test/gc_job_test.go +++ b/pkg/sql/gcjob_test/gc_job_test.go @@ -542,23 +542,15 @@ func TestGCTenant(t *testing.T) { }) } -// This test exercises code whereby an index GC job is running, and, in the +// This test exercises code whereby an GC job is running, and, in the // meantime, the descriptor is removed. We want to ensure that the GC job -// finishes without an error. -func TestDropIndexWithDroppedDescriptor(t *testing.T) { +// finishes without an error. We want to test this both for index drops +// and for table drops. +func TestDropWithDeletedDescriptor(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - // The way the GC job works is that it initially clears the index - // data, then it waits for the background MVCC GC to run and remove - // the underlying tombstone, and then finally it removes any relevant - // zone configurations for the index from system.zones. In the first - // and final phases, the job resolves the descriptor. This test ensures - // that the code is robust to the descriptor being removed both before - // the initial DelRange, and after, when going to remove the zone config. - testutils.RunTrueAndFalse(t, "before DelRange", func( - t *testing.T, beforeDelRange bool, - ) { + runTest := func(t *testing.T, dropIndex bool, beforeDelRange bool) { ctx, cancel := context.WithCancel(context.Background()) gcJobID := make(chan jobspb.JobID) knobs := base.TestingKnobs{ @@ -628,7 +620,12 @@ SELECT descriptor_id, index_id WHERE descriptor_name = 'foo' AND index_name = 'foo_j_i_idx';`).Scan(&tableID, &indexID) // Drop the index. - tdb.Exec(t, "DROP INDEX foo@foo_j_i_idx") + if dropIndex { + tdb.Exec(t, "DROP INDEX foo@foo_j_i_idx") + } else { + tdb.Exec(t, "DROP TABLE foo") + } + codec := s.ExecutorConfig().(sql.ExecutorConfig).Codec tablePrefix.Store(codec.TablePrefix(uint32(tableID))) @@ -654,5 +651,20 @@ SELECT descriptor_id, index_id // Ensure that the job completes successfully in either case. jr := s.JobRegistry().(*jobs.Registry) require.NoError(t, jr.WaitForJobs(ctx, []jobspb.JobID{jobID})) + } + + // The way the GC job works is that it initially clears the index + // data, then it waits for the background MVCC GC to run and remove + // the underlying tombstone, and then finally it removes any relevant + // zone configurations for the index from system.zones. In the first + // and final phases, the job resolves the descriptor. This test ensures + // that the code is robust to the descriptor being removed both before + // the initial DelRange, and after, when going to remove the zone config. + testutils.RunTrueAndFalse(t, "before DelRange", func( + t *testing.T, beforeDelRange bool, + ) { + testutils.RunTrueAndFalse(t, "drop index", func(t *testing.T, dropIndex bool) { + runTest(t, dropIndex, beforeDelRange) + }) }) } diff --git a/pkg/sql/sem/builtins/BUILD.bazel b/pkg/sql/sem/builtins/BUILD.bazel index 7ddf32853374..e2f50ffa10d7 100644 --- a/pkg/sql/sem/builtins/BUILD.bazel +++ b/pkg/sql/sem/builtins/BUILD.bazel @@ -91,6 +91,7 @@ go_library( "//pkg/sql/sem/volatility", "//pkg/sql/sessiondata", "//pkg/sql/sessiondatapb", + "//pkg/sql/sqlerrors", "//pkg/sql/sqlliveness", "//pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil", "//pkg/sql/sqltelemetry", diff --git a/pkg/sql/sem/builtins/pg_builtins.go b/pkg/sql/sem/builtins/pg_builtins.go index 8b1b307dcaa3..460e59f40c19 100644 --- a/pkg/sql/sem/builtins/pg_builtins.go +++ b/pkg/sql/sem/builtins/pg_builtins.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" + "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/ipaddr" "github.com/cockroachdb/errors" @@ -990,12 +991,10 @@ var pgBuiltins = map[string]builtinDefinition{ typ, err = evalCtx.Planner.ResolveTypeByOID(ctx, oid) if err != nil { // If the error is a descriptor does not exist error, then swallow it. - unknown := tree.NewDString(fmt.Sprintf("unknown (OID=%s)", oidArg)) switch { - case errors.Is(err, catalog.ErrDescriptorNotFound): - return unknown, nil - case pgerror.GetPGCode(err) == pgcode.UndefinedObject: - return unknown, nil + case sqlerrors.IsMissingDescriptorError(err), + errors.Is(err, catalog.ErrDescriptorNotFound): + return tree.NewDString(fmt.Sprintf("unknown (OID=%s)", oidArg)), nil default: return nil, err } diff --git a/pkg/sql/sqlerrors/errors.go b/pkg/sql/sqlerrors/errors.go index c3f8d04f379c..3369aae2b99f 100644 --- a/pkg/sql/sqlerrors/errors.go +++ b/pkg/sql/sqlerrors/errors.go @@ -378,6 +378,21 @@ func IsUndefinedSchemaError(err error) bool { return errHasCode(err, pgcode.UndefinedSchema) } +// IsMissingDescriptorError checks whether the error has any indication +// that it corresponds to a missing descriptor of any kind. +// +// Note that this does not deal with the lower-level +// catalog.ErrDescriptorNotFound error. That error should be transformed +// by this package for all uses in the SQL layer and coming out of +// descs.Collection functions. +func IsMissingDescriptorError(err error) bool { + return IsUndefinedRelationError(err) || + IsUndefinedSchemaError(err) || + IsUndefinedDatabaseError(err) || + errHasCode(err, pgcode.UndefinedObject) || + errHasCode(err, pgcode.UndefinedFunction) +} + func errHasCode(err error, code ...pgcode.Code) bool { pgCode := pgerror.GetPGCode(err) for _, c := range code {