Skip to content

Commit

Permalink
upgrademanager: unskip TestMigrationFailure
Browse files Browse the repository at this point in the history
The test no longer fails with our change in

Fixes: #106648
Fixes: #106762
Release note: None
  • Loading branch information
adityamaru committed Aug 18, 2023
1 parent 63a71dc commit ba3cf0b
Showing 1 changed file with 18 additions and 6 deletions.
24 changes: 18 additions & 6 deletions pkg/jobs/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,25 +151,37 @@ func isJobTypeColumnDoesNotExistError(err error) bool {
strings.Contains(err.Error(), "column \"job_type\" does not exist")
}

// isJobInfoTableDoesNotExistError returns true if the error is of the form
// `related "job_info" does not exist`.
func isJobInfoTableDoesNotExistError(err error) bool {
return pgerror.GetPGCode(err) == pgcode.UndefinedTable &&
strings.Contains(err.Error(), "relation \"system.job_info\" does not exist")
}

// MaybeGenerateForcedRetryableError returns a
// TransactionRetryWithProtoRefreshError that will cause the txn to be retried
// if the error is because of an undefined job_type column.
// if the error is because of an undefined job_type column or missing job_info
// table.
//
// In https://github.com/cockroachdb/cockroach/issues/106762 we noticed that if
// a query is executed with an AS OF SYSTEM TIME clause that picks a transaction
// timestamp before the job_type migration, then parts of the jobs
// infrastructure will attempt to query the job_type column even though it
// doesn't exist at the transaction's timestamp.
//
// As a short term fix, when we encounter an `UndefinedColumn` error we
// generate a synthetic retryable error so that the txn is pushed to a
// higher timestamp at which the upgrade will have completed and the
// `job_type` column will be visible. The longer term fix is being tracked
// in https://github.com/cockroachdb/cockroach/issues/106764.
// As a short term fix, when we encounter an `UndefinedTable` or
// `UndefinedColumn` error we generate a synthetic retryable error so that the
// txn is pushed to a higher timestamp at which the upgrade will have completed
// and the table/column will be visible. The longer term fix is being tracked in
// https://github.com/cockroachdb/cockroach/issues/106764.
func MaybeGenerateForcedRetryableError(ctx context.Context, txn *kv.Txn, err error) error {
if err != nil && isJobTypeColumnDoesNotExistError(err) {
return txn.GenerateForcedRetryableError(ctx, "synthetic error "+
"to push timestamp to after the `job_type` upgrade has run")
}
if err != nil && isJobInfoTableDoesNotExistError(err) {
return txn.GenerateForcedRetryableError(ctx, "synthetic error "+
"to push timestamp to after the `job_info` upgrade has run")
}
return err
}

0 comments on commit ba3cf0b

Please sign in to comment.