Merge #97218

97218: jobs: write payload and progress to system.job_info r=dt,fqazi a=adityamaru

jobs: introduces an InfoStorage wrapper

This change introduces an InfoStorage wrapper that
is initialized in the context of a Job and optionally a txn.
This wrapper can be used to read/write rows in
the system.job_info table that correspond to the
Job.

Additionally, the write path of InfoStorage checks that
the caller updating the row is part of the currently running
instance of the job. We do this by checking that the job's sqlliveness
session is the same as the one persisted in the job record.

Release note: None

jobs: write payload and progress to system.job_info

Newly created jobs will write their Payload and Progress
to the system.job_info table, in addition to the job record
that is already created in the system.jobs table. Running jobs
that update their Payload or Progress in system.jobs, during their
execution, will also write this updated state to the job_info table.

Double writes for the Payload and Progress of jobs is the first step
to migrating existing job state to the job_info table.

Follow up changes will:

1.  Add a migration to backfill job_info with the Payload and Progress
of all job records in the system.jobs table.

2. Modify read methods in the registry to pull the Payload and Progress
from the job_info table instead of the jobs table.

Release note: None

Epic: CRDB-24874


Co-authored-by: adityamaru <[email protected]>

pkg/bench/rttanalysis/testdata/benchmark_expectations

@@ -1,65 +1,65 @@
 exp,benchmark
-10,AlterRole/alter_role_with_1_option
-13,AlterRole/alter_role_with_2_options
-17,AlterRole/alter_role_with_3_options
-9,AlterTableAddCheckConstraint/alter_table_add_1_check_constraint
-9,AlterTableAddCheckConstraint/alter_table_add_2_check_constraints
-9,AlterTableAddCheckConstraint/alter_table_add_3_check_constraints
-9,AlterTableAddColumn/alter_table_add_1_column
-9,AlterTableAddColumn/alter_table_add_2_columns
-9,AlterTableAddColumn/alter_table_add_3_columns
-11,AlterTableAddForeignKey/alter_table_add_1_foreign_key
-13,AlterTableAddForeignKey/alter_table_add_2_foreign_keys
-15,AlterTableAddForeignKey/alter_table_add_3_foreign_keys
-11,AlterTableAddForeignKey/alter_table_add_foreign_key_with_3_columns
+14,AlterRole/alter_role_with_1_option
+17,AlterRole/alter_role_with_2_options
+25,AlterRole/alter_role_with_3_options
+15,AlterTableAddCheckConstraint/alter_table_add_1_check_constraint
+15,AlterTableAddCheckConstraint/alter_table_add_2_check_constraints
+15,AlterTableAddCheckConstraint/alter_table_add_3_check_constraints
+15,AlterTableAddColumn/alter_table_add_1_column
+15,AlterTableAddColumn/alter_table_add_2_columns
+15,AlterTableAddColumn/alter_table_add_3_columns
+17,AlterTableAddForeignKey/alter_table_add_1_foreign_key
+19,AlterTableAddForeignKey/alter_table_add_2_foreign_keys
+21,AlterTableAddForeignKey/alter_table_add_3_foreign_keys
+17,AlterTableAddForeignKey/alter_table_add_foreign_key_with_3_columns
 8,AlterTableConfigureZone/alter_table_configure_zone_5_replicas
 8,AlterTableConfigureZone/alter_table_configure_zone_7_replicas_
 8,AlterTableConfigureZone/alter_table_configure_zone_ranges
-13,AlterTableDropColumn/alter_table_drop_1_column
-13,AlterTableDropColumn/alter_table_drop_2_columns
-13,AlterTableDropColumn/alter_table_drop_3_columns
-9,AlterTableDropConstraint/alter_table_drop_1_check_constraint
-9,AlterTableDropConstraint/alter_table_drop_2_check_constraints
-9,AlterTableDropConstraint/alter_table_drop_3_check_constraints
+19,AlterTableDropColumn/alter_table_drop_1_column
+19,AlterTableDropColumn/alter_table_drop_2_columns
+19,AlterTableDropColumn/alter_table_drop_3_columns
+15,AlterTableDropConstraint/alter_table_drop_1_check_constraint
+15,AlterTableDropConstraint/alter_table_drop_2_check_constraints
+15,AlterTableDropConstraint/alter_table_drop_3_check_constraints
 9,AlterTableSplit/alter_table_split_at_1_value
 13,AlterTableSplit/alter_table_split_at_2_values
 17,AlterTableSplit/alter_table_split_at_3_values
 7,AlterTableUnsplit/alter_table_unsplit_at_1_value
 9,AlterTableUnsplit/alter_table_unsplit_at_2_values
 11,AlterTableUnsplit/alter_table_unsplit_at_3_values
-12,CreateRole/create_role_with_1_option
-15,CreateRole/create_role_with_2_options
-18,CreateRole/create_role_with_3_options
-13,CreateRole/create_role_with_no_options
-12,"Discard/DISCARD_ALL,_1_tables_in_1_db"
-15,"Discard/DISCARD_ALL,_2_tables_in_2_dbs"
+20,CreateRole/create_role_with_1_option
+23,CreateRole/create_role_with_2_options
+26,CreateRole/create_role_with_3_options
+21,CreateRole/create_role_with_no_options
+16,"Discard/DISCARD_ALL,_1_tables_in_1_db"
+23,"Discard/DISCARD_ALL,_2_tables_in_2_dbs"
 0,"Discard/DISCARD_ALL,_no_tables"
-10,DropDatabase/drop_database_0_tables
-11,DropDatabase/drop_database_1_table
-11,DropDatabase/drop_database_2_tables
-11,DropDatabase/drop_database_3_tables
-18,DropRole/drop_1_role
-26,DropRole/drop_2_roles
-34,DropRole/drop_3_roles
-9,DropSequence/drop_1_sequence
-11,DropSequence/drop_2_sequences
-13,DropSequence/drop_3_sequences
-9,DropTable/drop_1_table
-11,DropTable/drop_2_tables
-13,DropTable/drop_3_tables
-10,DropView/drop_1_view
-11,DropView/drop_2_views
-11,DropView/drop_3_views
+16,DropDatabase/drop_database_0_tables
+17,DropDatabase/drop_database_1_table
+17,DropDatabase/drop_database_2_tables
+17,DropDatabase/drop_database_3_tables
+26,DropRole/drop_1_role
+34,DropRole/drop_2_roles
+42,DropRole/drop_3_roles
+15,DropSequence/drop_1_sequence
+17,DropSequence/drop_2_sequences
+19,DropSequence/drop_3_sequences
+15,DropTable/drop_1_table
+17,DropTable/drop_2_tables
+19,DropTable/drop_3_tables
+16,DropView/drop_1_view
+17,DropView/drop_2_views
+17,DropView/drop_3_views
 5,GenerateObjects/generate_1000_tables_-_this_test_should_use_the_same_number_of_RTTs_as_for_10_tables
 11,GenerateObjects/generate_100_tables_from_template
 5,GenerateObjects/generate_10_tables
 16,GenerateObjects/generate_10x10_schemas_and_tables_in_existing_db
 5,GenerateObjects/generate_50000_tables
-9,Grant/grant_all_on_1_table
-9,Grant/grant_all_on_2_tables
-9,Grant/grant_all_on_3_tables
-13,GrantRole/grant_1_role
-18,GrantRole/grant_2_roles
+13,Grant/grant_all_on_1_table
+17,Grant/grant_all_on_2_tables
+21,Grant/grant_all_on_3_tables
+17,GrantRole/grant_1_role
+22,GrantRole/grant_2_roles
 3,ORMQueries/activerecord_type_introspection_query
 4,ORMQueries/django_column_introspection_1_table
 4,ORMQueries/django_column_introspection_4_tables
@@ -83,21 +83,21 @@ exp,benchmark
 3,ORMQueries/pg_my_temp_schema_multiple_times
 3,ORMQueries/pg_namespace
 4,ORMQueries/pg_type
-9,Revoke/revoke_all_on_1_table
-9,Revoke/revoke_all_on_2_tables
-9,Revoke/revoke_all_on_3_tables
-10,RevokeRole/revoke_1_role
-12,RevokeRole/revoke_2_roles
+13,Revoke/revoke_all_on_1_table
+17,Revoke/revoke_all_on_2_tables
+21,Revoke/revoke_all_on_3_tables
+14,RevokeRole/revoke_1_role
+16,RevokeRole/revoke_2_roles
 1,SystemDatabaseQueries/select_system.users_with_empty_database_Name
 1,SystemDatabaseQueries/select_system.users_with_schema_Name
 1,SystemDatabaseQueries/select_system.users_without_schema_Name
-12,Truncate/truncate_1_column_0_rows
-12,Truncate/truncate_1_column_1_row
-12,Truncate/truncate_1_column_2_rows
-12,Truncate/truncate_2_column_0_rows
-12,Truncate/truncate_2_column_1_rows
-12,Truncate/truncate_2_column_2_rows
+20,Truncate/truncate_1_column_0_rows
+20,Truncate/truncate_1_column_1_row
+20,Truncate/truncate_1_column_2_rows
+20,Truncate/truncate_2_column_0_rows
+20,Truncate/truncate_2_column_1_rows
+20,Truncate/truncate_2_column_2_rows
 1,VirtualTableQueries/select_crdb_internal.invalid_objects_with_1_fk
 1,VirtualTableQueries/select_crdb_internal.tables_with_1_fk
 9,VirtualTableQueries/virtual_table_cache_with_point_lookups
-7,VirtualTableQueries/virtual_table_cache_with_schema_change
+15,VirtualTableQueries/virtual_table_cache_with_schema_change

pkg/ccl/benchccl/rttanalysisccl/testdata/benchmark_expectations

@@ -1,19 +1,19 @@
 exp,benchmark
-10,AlterPrimaryRegion/alter_empty_database_alter_primary_region
-12,AlterPrimaryRegion/alter_empty_database_set_initial_primary_region
-10,AlterPrimaryRegion/alter_populated_database_alter_primary_region
-13,AlterPrimaryRegion/alter_populated_database_set_initial_primary_region
-9,AlterRegions/alter_empty_database_add_region
-11,AlterRegions/alter_empty_database_drop_region
-9,AlterRegions/alter_populated_database_add_region
-11,AlterRegions/alter_populated_database_drop_region
-10,AlterSurvivalGoals/alter_empty_database_from_region_to_zone
-10,AlterSurvivalGoals/alter_empty_database_from_zone_to_region
-20,AlterSurvivalGoals/alter_populated_database_from_region_to_zone
-20,AlterSurvivalGoals/alter_populated_database_from_zone_to_region
-12,AlterTableLocality/alter_from_global_to_rbr
-11,AlterTableLocality/alter_from_global_to_regional_by_table
-8,AlterTableLocality/alter_from_rbr_to_global
-8,AlterTableLocality/alter_from_rbr_to_regional_by_table
-11,AlterTableLocality/alter_from_regional_by_table_to_global
-12,AlterTableLocality/alter_from_regional_by_table_to_rbr
+14,AlterPrimaryRegion/alter_empty_database_alter_primary_region
+15,AlterPrimaryRegion/alter_empty_database_set_initial_primary_region
+14,AlterPrimaryRegion/alter_populated_database_alter_primary_region
+16,AlterPrimaryRegion/alter_populated_database_set_initial_primary_region
+13,AlterRegions/alter_empty_database_add_region
+18,AlterRegions/alter_empty_database_drop_region
+13,AlterRegions/alter_populated_database_add_region
+18,AlterRegions/alter_populated_database_drop_region
+14,AlterSurvivalGoals/alter_empty_database_from_region_to_zone
+14,AlterSurvivalGoals/alter_empty_database_from_zone_to_region
+24,AlterSurvivalGoals/alter_populated_database_from_region_to_zone
+24,AlterSurvivalGoals/alter_populated_database_from_zone_to_region
+24,AlterTableLocality/alter_from_global_to_rbr
+15,AlterTableLocality/alter_from_global_to_regional_by_table
+12,AlterTableLocality/alter_from_rbr_to_global
+12,AlterTableLocality/alter_from_rbr_to_regional_by_table
+15,AlterTableLocality/alter_from_regional_by_table_to_global
+24,AlterTableLocality/alter_from_regional_by_table_to_rbr

pkg/ccl/kvccl/kvtenantccl/tenant_upgrade_test.go

@@ -255,6 +255,7 @@ func TestTenantUpgradeFailure(t *testing.T) {
 			TestingKnobs: base.TestingKnobs{
 				JobsTestingKnobs: jobs.NewTestingKnobsWithShortIntervals(),
 				UpgradeManager: &upgradebase.TestingKnobs{
+					DontUseJobs: true,
 					ListBetweenOverride: func(from, to roachpb.Version) []roachpb.Version {
 						return []roachpb.Version{v1, v2}
 					},

pkg/jobs/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
         "errors.go",
         "executor_impl.go",
         "helpers.go",
+        "job_info_storage.go",
         "job_scheduler.go",
         "jobs.go",
         "metrics.go",
@@ -87,6 +88,7 @@ go_test(
         "delegate_control_test.go",
         "executor_impl_test.go",
         "helpers_test.go",
+        "job_info_storage_test.go",
         "job_scheduler_test.go",
         "jobs_test.go",
         "lease_test.go",
@@ -96,6 +98,7 @@ go_test(
         "scheduled_job_executor_test.go",
         "scheduled_job_test.go",
         "testutils_test.go",
+        "update_test.go",
     ],
     args = ["-test.timeout=895s"],
     embed = [":jobs"],
@@ -150,6 +153,7 @@ go_test(
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "//pkg/util/tracing",
+        "//pkg/util/tracing/tracingpb",
         "//pkg/util/uuid",
         "@com_github_cockroachdb_apd_v3//:apd",
         "@com_github_cockroachdb_errors//:errors",