Merge #125158

125158: backupccl: Replace DEBUG_PAUSE_ON option with standard pause point wh… r=dt a=navsetlur …en pausing a restore operation on error The DEBUG_PAUSE_ON option was previously used to pause a restore operation when it failed before returning the error. We currently have a standardized way of pausing operations via the pause point interface and it made sense to update this feature to use this approach while fixing a test failure. This change removes the DEBUG_PAUSE_ON option and replaces it with a 'restore.restore_after_failure' pause point. Fixes: #121342 Release note (enterprise): The DEBUG_PAUSE_ON option has been removed entirely and replaced with the 'restore.restore_after_failure' pause point to match other pause points used throughout the codebase. You can set this pause point by running `SET CLUSTER SETTING jobs.debug.pausepoints = 'restore.after_restore_failure'` Co-authored-by: Naveen Setlur <[email protected]>
cockroachdb · Jun 10, 2024 · e09f535 · e09f535
2 parents ffb0466 + 49a1a4b
commit e09f535
Show file tree

Hide file tree

Showing 11 changed files with 19 additions and 177 deletions.
diff --git a/docs/generated/sql/bnf/restore_options.bnf b/docs/generated/sql/bnf/restore_options.bnf
@@ -9,7 +9,6 @@ restore_options ::=
 	| 'SKIP_MISSING_UDFS'
 	| 'DETACHED'
 	| 'SKIP_LOCALITIES_CHECK'
-	| 'DEBUG_PAUSE_ON' '=' string_or_placeholder
 	| 'NEW_DB_NAME' '=' string_or_placeholder
 	| 'INCREMENTAL_LOCATION' '=' string_or_placeholder_opt_list
 	| 'VIRTUAL_CLUSTER_NAME' '=' string_or_placeholder

diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf
@@ -1121,7 +1121,6 @@ unreserved_keyword ::=
 	| 'DAY'
 	| 'DEALLOCATE'
 	| 'DEBUG_IDS'
-	| 'DEBUG_PAUSE_ON'
 	| 'DEBUG_DUMP_METADATA_SST'
 	| 'DECLARE'
 	| 'DELETE'
@@ -2718,7 +2717,6 @@ restore_options ::=
 	| 'SKIP_MISSING_UDFS'
 	| 'DETACHED'
 	| 'SKIP_LOCALITIES_CHECK'
-	| 'DEBUG_PAUSE_ON' '=' string_or_placeholder
 	| 'NEW_DB_NAME' '=' string_or_placeholder
 	| 'INCREMENTAL_LOCATION' '=' string_or_placeholder_opt_list
 	| virtual_cluster_name '=' string_or_placeholder
@@ -3701,7 +3699,6 @@ bare_label_keywords ::=
 	| 'DEALLOCATE'
 	| 'DEBUG_DUMP_METADATA_SST'
 	| 'DEBUG_IDS'
-	| 'DEBUG_PAUSE_ON'
 	| 'DEC'
 	| 'DECIMAL'
 	| 'DECLARE'

diff --git a/pkg/ccl/backupccl/backup_telemetry.go b/pkg/ccl/backupccl/backup_telemetry.go
@@ -318,7 +318,6 @@ func logRestoreTelemetry(
 	opts tree.RestoreOptions,
 	descsByTablePattern map[tree.TablePattern]catalog.Descriptor,
 	restoreDBs []catalog.DatabaseDescriptor,
-	debugPauseOn string,
 	applicationName string,
 ) {
 	var requestedTargets []descpb.Descriptor
@@ -419,7 +418,6 @@ func logRestoreTelemetry(
 		HasEncryptionPassphrase: passphrase,
 		KMSType:                 kmsType,
 		KMSCount:                uint32(kmsCount),
-		DebugPauseOn:            debugPauseOn,
 		JobID:                   uint64(jobID),
 		Options:                 options,
 		ApplicationName:         applicationName,

diff --git a/pkg/ccl/backupccl/backup_test.go b/pkg/ccl/backupccl/backup_test.go
@@ -9118,103 +9118,6 @@ DROP INDEX idx_3;
 	sqlDB.Exec(t, `BACKUP test.t TO 'nodelocal://1/backup_test' WITH revision_history`)
 }
 
-func waitForStatus(t *testing.T, db *sqlutils.SQLRunner, jobID int64, status jobs.Status) error {
-	return testutils.SucceedsSoonError(func() error {
-		var jobStatus string
-		db.QueryRow(t, `SELECT status FROM system.jobs WHERE id = $1`, jobID).Scan(&jobStatus)
-
-		if status != jobs.Status(jobStatus) {
-			return errors.Newf("expected jobID %d to have status %, got %s", jobID, status, jobStatus)
-		}
-		return nil
-	})
-}
-
-// Test to verify that RESTORE jobs self pause on error when given the
-// DEBUG_PAUSE_ON = 'error' option.
-func TestRestorePauseOnError(t *testing.T) {
-	defer leaktest.AfterTest(t)()
-	defer log.ScopeWithoutShowLogs(t).Close(t)
-
-	skip.WithIssue(t, 121342)
-
-	defer jobs.TestingSetProgressThresholds()()
-
-	baseDir := "testdata"
-	args := base.TestServerArgs{
-		ExternalIODir: baseDir,
-		Knobs:         base.TestingKnobs{JobsTestingKnobs: jobs.NewTestingKnobsWithShortIntervals()}}
-	params := base.TestClusterArgs{ServerArgs: args}
-	tc, sqlDB, _, cleanupFn := backupRestoreTestSetupWithParams(t, singleNode, 1,
-		InitManualReplication, params)
-	defer cleanupFn()
-
-	var forceFailure bool
-	for i := range tc.Servers {
-		jobRegistry := tc.ApplicationLayer(i).JobRegistry()
-
-		jobRegistry.(*jobs.Registry).TestingWrapResumerConstructor(
-			jobspb.TypeRestore,
-			func(raw jobs.Resumer) jobs.Resumer {
-				r := raw.(*restoreResumer)
-				r.testingKnobs.beforePublishingDescriptors = func() error {
-					if forceFailure {
-						return errors.New("testing injected failure")
-					}
-					return nil
-				}
-				return r
-			})
-	}
-
-	sqlDB.Exec(t, `CREATE DATABASE r1`)
-	sqlDB.Exec(t, `CREATE TABLE r1.foo (id INT)`)
-	sqlDB.Exec(t, `BACKUP DATABASE r1 TO 'nodelocal://1/eventlogging'`)
-	sqlDB.Exec(t, `DROP DATABASE r1`)
-
-	restoreQuery := `RESTORE DATABASE r1 FROM 'nodelocal://1/eventlogging' WITH DEBUG_PAUSE_ON = 'error'`
-	findJobQuery := `SELECT job_id FROM [SHOW JOBS] WHERE description LIKE '%RESTORE DATABASE%' ORDER BY created DESC`
-
-	// Verify that a RESTORE job will self pause on an error, but can be resumed
-	// after the source of error is fixed.
-	{
-		var jobID int64
-		forceFailure = true
-
-		sqlDB.QueryRow(t, restoreQuery)
-
-		sqlDB.QueryRow(t, findJobQuery).Scan(&jobID)
-		if err := waitForStatus(t, sqlDB, jobID, jobs.StatusPaused); err != nil {
-			t.Fatal(err)
-		}
-
-		forceFailure = false
-		sqlDB.Exec(t, "RESUME JOB $1", jobID)
-
-		if err := waitForStatus(t, sqlDB, jobID, jobs.StatusSucceeded); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Verify that a RESTORE job will self pause on an error and can be canceled.
-	{
-		var jobID int64
-		forceFailure = true
-		sqlDB.Exec(t, `DROP DATABASE r1`)
-		sqlDB.QueryRow(t, restoreQuery)
-		sqlDB.QueryRow(t, findJobQuery).Scan(&jobID)
-		if err := waitForStatus(t, sqlDB, jobID, jobs.StatusPaused); err != nil {
-			t.Fatal(err)
-		}
-
-		sqlDB.Exec(t, "CANCEL JOB $1", jobID)
-
-		if err := waitForStatus(t, sqlDB, jobID, jobs.StatusCanceled); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
 // TestDroppedDescriptorRevisionAndSystemDBIDClash is a regression test for a
 // discrepancy in the descriptor resolution logic during restore planning and
 // execution.

diff --git a/pkg/ccl/backupccl/restore_job.go b/pkg/ccl/backupccl/restore_job.go
@@ -1716,19 +1716,11 @@ func remapPublicSchemas(
 
 // Resume is part of the jobs.Resumer interface.
 func (r *restoreResumer) Resume(ctx context.Context, execCtx interface{}) error {
+	p := execCtx.(sql.JobExecContext)
 	if err := r.doResume(ctx, execCtx); err != nil {
-		details := r.job.Details().(jobspb.RestoreDetails)
-		if details.DebugPauseOn == "error" {
-			const errorFmt = "job failed with error (%v) but is being paused due to the %s=%s setting"
-			log.Warningf(ctx, errorFmt, err, restoreOptDebugPauseOn, details.DebugPauseOn)
-
-			return jobs.MarkPauseRequestError(errors.Wrapf(err,
-				"pausing job due to the %s=%s setting",
-				restoreOptDebugPauseOn, details.DebugPauseOn))
-		}
-		return err
+		// Need to return the pause "error" as the main error here
+		return errors.CombineErrors(p.ExecCfg().JobRegistry.CheckPausepoint("restore.after_restore_failure"), err)
 	}
-
 	return nil
 }
 

diff --git a/pkg/ccl/backupccl/restore_planning.go b/pkg/ccl/backupccl/restore_planning.go
@@ -79,7 +79,6 @@ const (
 	restoreOptSkipMissingSequenceOwners = "skip_missing_sequence_owners"
 	restoreOptSkipMissingViews          = "skip_missing_views"
 	restoreOptSkipLocalitiesCheck       = "skip_localities_check"
-	restoreOptDebugPauseOn              = "debug_pause_on"
 	restoreOptAsTenant                  = "virtual_cluster_name"
 	restoreOptForceTenantID             = "virtual_cluster"
 
@@ -88,10 +87,6 @@ const (
 	restoreTempSystemDB = "crdb_temp_system"
 )
 
-var allowedDebugPauseOnValues = map[string]struct{}{
-	"error": {},
-}
-
 // featureRestoreEnabled is used to enable and disable the RESTORE feature.
 var featureRestoreEnabled = settings.RegisterBoolSetting(
 	settings.ApplicationLevel,
@@ -1079,7 +1074,6 @@ func resolveOptionsForRestoreJobDescription(
 		SkipMissingUDFs:                  opts.SkipMissingUDFs,
 		Detached:                         opts.Detached,
 		SkipLocalitiesCheck:              opts.SkipLocalitiesCheck,
-		DebugPauseOn:                     opts.DebugPauseOn,
 		AsTenant:                         opts.AsTenant,
 		ForceTenantID:                    opts.ForceTenantID,
 		SchemaOnly:                       opts.SchemaOnly,
@@ -1187,7 +1181,6 @@ func restoreTypeCheck(
 			restoreStmt.Options.NewDBName,
 			restoreStmt.Options.ForceTenantID,
 			restoreStmt.Options.AsTenant,
-			restoreStmt.Options.DebugPauseOn,
 			restoreStmt.Options.ExecutionLocality,
 		},
 	); err != nil {
@@ -1994,19 +1987,6 @@ func doRestorePlan(
 		}
 	}
 
-	var debugPauseOn string
-	if restoreStmt.Options.DebugPauseOn != nil {
-		var err error
-		debugPauseOn, err = exprEval.String(ctx, restoreStmt.Options.DebugPauseOn)
-		if err != nil {
-			return err
-		}
-
-		if _, ok := allowedDebugPauseOnValues[debugPauseOn]; len(debugPauseOn) > 0 && !ok {
-			return errors.Newf("%s cannot be set with the value %s", restoreOptDebugPauseOn, debugPauseOn)
-		}
-	}
-
 	var asOfInterval int64
 	if !endTime.IsEmpty() {
 		asOfInterval = endTime.WallTime - p.ExtendedEvalContext().StmtTimestamp.UnixNano()
@@ -2143,7 +2123,6 @@ func doRestorePlan(
 		DescriptorCoverage: restoreStmt.DescriptorCoverage,
 		Encryption:         encryption,
 		DatabaseModifiers:  databaseModifiers,
-		DebugPauseOn:       debugPauseOn,
 
 		// A RESTORE SYSTEM USERS planned on a 22.1 node will use the
 		// RestoreSystemUsers field in the job details to identify this flavour of
@@ -2187,7 +2166,7 @@ func doRestorePlan(
 		}
 		resultsCh <- tree.Datums{tree.NewDInt(tree.DInt(jobID))}
 		collectRestoreTelemetry(ctx, jobID, restoreDetails, intoDB, newDBName, subdir, restoreStmt,
-			descsByTablePattern, restoreDBs, asOfInterval, debugPauseOn, p.SessionData().ApplicationName)
+			descsByTablePattern, restoreDBs, asOfInterval, p.SessionData().ApplicationName)
 		return nil
 	}
 
@@ -2234,7 +2213,7 @@ func doRestorePlan(
 	// execution.
 	p.InternalSQLTxn().Descriptors().ReleaseAll(ctx)
 	collectRestoreTelemetry(ctx, sj.ID(), restoreDetails, intoDB, newDBName, subdir, restoreStmt,
-		descsByTablePattern, restoreDBs, asOfInterval, debugPauseOn, p.SessionData().ApplicationName)
+		descsByTablePattern, restoreDBs, asOfInterval, p.SessionData().ApplicationName)
 	if err := sj.Start(ctx); err != nil {
 		return err
 	}
@@ -2255,7 +2234,6 @@ func collectRestoreTelemetry(
 	descsByTablePattern map[tree.TablePattern]catalog.Descriptor,
 	restoreDBs []catalog.DatabaseDescriptor,
 	asOfInterval int64,
-	debugPauseOn string,
 	applicationName string,
 ) {
 	telemetry.Count("restore.total.started")
@@ -2269,7 +2247,7 @@ func collectRestoreTelemetry(
 	}
 
 	logRestoreTelemetry(ctx, jobID, details, intoDB, newDBName, subdir, asOfInterval, restoreStmt.Options,
-		descsByTablePattern, restoreDBs, debugPauseOn, applicationName)
+		descsByTablePattern, restoreDBs, applicationName)
 }
 
 // checkForConflictingDescriptors checks for user-created descriptors that would

diff --git a/pkg/ccl/backupccl/restore_planning_test.go b/pkg/ccl/backupccl/restore_planning_test.go
@@ -54,7 +54,6 @@ func TestRestoreResolveOptionsForJobDescription(t *testing.T) {
 		SkipMissingUDFs:                  true,
 		Detached:                         true,
 		SkipLocalitiesCheck:              true,
-		DebugPauseOn:                     tree.NewDString("test expr"),
 		AsTenant:                         tree.NewDString("test expr"),
 		ForceTenantID:                    tree.NewDInt(42),
 		SchemaOnly:                       true,

diff --git a/pkg/jobs/jobspb/jobs.proto b/pkg/jobs/jobspb/jobs.proto
@@ -518,9 +518,6 @@ message RestoreDetails {
     (gogoproto.castkey) = "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb.ID"
   ];
 
-  // DebugPauseOn describes the events that the job should pause itself on for debugging purposes.
-  string debug_pause_on = 20;
-
   // RestoreSystemUsers is set to true if user runs RESTORE SYSTEM USERS.
   // TODO(msbutler): delete in 23.1
   bool restore_system_users = 22;

diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y
@@ -926,7 +926,7 @@ func (u *sqlSymUnion) showFingerprintOptions() *tree.ShowFingerprintOptions {
 %token <str> CURRENT_ROLE CURRENT_TIME CURRENT_TIMESTAMP
 %token <str> CURRENT_USER CURSOR CYCLE
 
-%token <str> DATA DATABASE DATABASES DATE DAY DEBUG_IDS DEBUG_PAUSE_ON DEC DEBUG_DUMP_METADATA_SST DECIMAL DEFAULT DEFAULTS DEFINER
+%token <str> DATA DATABASE DATABASES DATE DAY DEBUG_IDS DEC DEBUG_DUMP_METADATA_SST DECIMAL DEFAULT DEFAULTS DEFINER
 %token <str> DEALLOCATE DECLARE DEFERRABLE DEFERRED DELETE DELIMITER DEPENDS DESC DESTINATION DETACHED DETAILS
 %token <str> DISCARD DISTINCT DO DOMAIN DOUBLE DROP
 
@@ -3771,7 +3771,6 @@ drop_external_connection_stmt:
 //    kms="[kms_provider]://[kms_host]/[master_key_identifier]?[parameters]" : decrypt backups using KMS
 //    detached: execute restore job asynchronously, without waiting for its completion
 //    skip_localities_check: ignore difference of zone configuration between restore cluster and backup cluster
-//    debug_pause_on: describes the events that the job should pause itself on for debugging purposes.
 //    new_db_name: renames the restored database. only applies to database restores
 //    include_all_virtual_clusters: enable backups of all virtual clusters during a cluster backup
 // %SeeAlso: BACKUP, WEBDOCS/restore.html
@@ -3925,10 +3924,6 @@ restore_options:
   {
     $$.val = &tree.RestoreOptions{SkipLocalitiesCheck: true}
   }
-| DEBUG_PAUSE_ON '=' string_or_placeholder
-  {
-    $$.val = &tree.RestoreOptions{DebugPauseOn: $3.expr()}
-  }
 | NEW_DB_NAME '=' string_or_placeholder
   {
     $$.val = &tree.RestoreOptions{NewDBName: $3.expr()}
@@ -17121,7 +17116,6 @@ unreserved_keyword:
 | DAY
 | DEALLOCATE
 | DEBUG_IDS
-| DEBUG_PAUSE_ON
 | DEBUG_DUMP_METADATA_SST
 | DECLARE
 | DELETE
@@ -17626,7 +17620,6 @@ bare_label_keywords:
 | DEALLOCATE
 | DEBUG_DUMP_METADATA_SST
 | DEBUG_IDS
-| DEBUG_PAUSE_ON
 | DEC
 | DECIMAL
 | DECLARE

diff --git a/pkg/sql/parser/testdata/backup_restore b/pkg/sql/parser/testdata/backup_restore
@@ -740,24 +740,24 @@ RESTORE FROM '_' WITH OPTIONS (into_db = '_', skip_missing_foreign_keys, skip_lo
 RESTORE FROM 'a' WITH OPTIONS (into_db = 'foo', skip_missing_foreign_keys, skip_localities_check) -- identifiers removed
 
 parse
-RESTORE foo FROM 'bar' WITH OPTIONS (encryption_passphrase='secret', into_db='baz', debug_pause_on='error',
+RESTORE foo FROM 'bar' WITH OPTIONS (encryption_passphrase='secret', into_db='baz',
 skip_missing_foreign_keys, skip_missing_sequences, skip_missing_sequence_owners, skip_missing_views, skip_missing_udfs, detached, skip_localities_check)
 ----
-RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- normalized!
-RESTORE TABLE (foo) FROM ('bar') WITH OPTIONS (encryption_passphrase = '*****', into_db = ('baz'), debug_pause_on = ('error'), skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- fully parenthesized
-RESTORE TABLE foo FROM '_' WITH OPTIONS (encryption_passphrase = '*****', into_db = '_', debug_pause_on = '_', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- literals removed
-RESTORE TABLE _ FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- identifiers removed
-RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = 'secret', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- passwords exposed
+RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- normalized!
+RESTORE TABLE (foo) FROM ('bar') WITH OPTIONS (encryption_passphrase = '*****', into_db = ('baz'), skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- fully parenthesized
+RESTORE TABLE foo FROM '_' WITH OPTIONS (encryption_passphrase = '*****', into_db = '_', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- literals removed
+RESTORE TABLE _ FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- identifiers removed
+RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = 'secret', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, detached, skip_localities_check) -- passwords exposed
 
 parse
-RESTORE foo FROM 'bar' WITH ENCRYPTION_PASSPHRASE = 'secret', INTO_DB=baz, DEBUG_PAUSE_ON='error',
+RESTORE foo FROM 'bar' WITH ENCRYPTION_PASSPHRASE = 'secret', INTO_DB=baz,
 SKIP_MISSING_FOREIGN_KEYS, SKIP_MISSING_SEQUENCES, SKIP_MISSING_SEQUENCE_OWNERS, SKIP_MISSING_VIEWS, SKIP_LOCALITIES_CHECK, SKIP_MISSING_UDFS
 ----
-RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- normalized!
-RESTORE TABLE (foo) FROM ('bar') WITH OPTIONS (encryption_passphrase = '*****', into_db = ('baz'), debug_pause_on = ('error'), skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- fully parenthesized
-RESTORE TABLE foo FROM '_' WITH OPTIONS (encryption_passphrase = '*****', into_db = '_', debug_pause_on = '_', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- literals removed
-RESTORE TABLE _ FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- identifiers removed
-RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = 'secret', into_db = 'baz', debug_pause_on = 'error', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- passwords exposed
+RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- normalized!
+RESTORE TABLE (foo) FROM ('bar') WITH OPTIONS (encryption_passphrase = '*****', into_db = ('baz'), skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- fully parenthesized
+RESTORE TABLE foo FROM '_' WITH OPTIONS (encryption_passphrase = '*****', into_db = '_', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- literals removed
+RESTORE TABLE _ FROM 'bar' WITH OPTIONS (encryption_passphrase = '*****', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- identifiers removed
+RESTORE TABLE foo FROM 'bar' WITH OPTIONS (encryption_passphrase = 'secret', into_db = 'baz', skip_missing_foreign_keys, skip_missing_sequence_owners, skip_missing_sequences, skip_missing_views, skip_missing_udfs, skip_localities_check) -- passwords exposed
 
 parse
 RESTORE TENANT 36 FROM ($1, $2) AS OF SYSTEM TIME '1'