diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 709deda57a47..8d963b96be9d 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -149,7 +149,6 @@ ALL_TESTS = [ "//pkg/kv/kvserver/protectedts/ptcache:ptcache_test", "//pkg/kv/kvserver/protectedts/ptreconcile:ptreconcile_test", "//pkg/kv/kvserver/protectedts/ptstorage:ptstorage_test", - "//pkg/kv/kvserver/protectedts/ptverifier:ptverifier_test", "//pkg/kv/kvserver/protectedts:protectedts_test", "//pkg/kv/kvserver/raftentry:raftentry_test", "//pkg/kv/kvserver/rangefeed:rangefeed_test", diff --git a/pkg/kv/kvserver/protectedts/protectedts.go b/pkg/kv/kvserver/protectedts/protectedts.go index 5ed957ae2fe1..2fedb889f32f 100644 --- a/pkg/kv/kvserver/protectedts/protectedts.go +++ b/pkg/kv/kvserver/protectedts/protectedts.go @@ -39,7 +39,6 @@ var ErrExists = errors.New("protected timestamp record already exists") type Provider interface { Storage Cache - Verifier Reconciler Start(context.Context, *stop.Stopper) error @@ -134,16 +133,6 @@ type Cache interface { Refresh(_ context.Context, asOf hlc.Timestamp) error } -// Verifier provides a mechanism to verify that a created Record will certainly -// apply. -type Verifier interface { - - // Verify returns an error if the record of the provided ID cannot be - // verified. If nil is returned then the record has been proven to apply - // until it is removed. - Verify(context.Context, uuid.UUID) error -} - // Reconciler provides a mechanism to reconcile protected timestamp records with // external state. type Reconciler interface { diff --git a/pkg/kv/kvserver/protectedts/protectedts_test.go b/pkg/kv/kvserver/protectedts/protectedts_test.go index 8b3b6f8752ee..8fd0ff51496e 100644 --- a/pkg/kv/kvserver/protectedts/protectedts_test.go +++ b/pkg/kv/kvserver/protectedts/protectedts_test.go @@ -17,7 +17,6 @@ func TestProtectedTimestamps(t *testing.T) { var ( _ Provider _ Cache - _ Verifier _ Storage _ = EmptyCache(nil) _ = ErrNotExists diff --git a/pkg/kv/kvserver/protectedts/ptprovider/BUILD.bazel b/pkg/kv/kvserver/protectedts/ptprovider/BUILD.bazel index e3620f19cb9e..cd021496f27c 100644 --- a/pkg/kv/kvserver/protectedts/ptprovider/BUILD.bazel +++ b/pkg/kv/kvserver/protectedts/ptprovider/BUILD.bazel @@ -12,7 +12,6 @@ go_library( "//pkg/kv/kvserver/protectedts/ptcache", "//pkg/kv/kvserver/protectedts/ptreconcile", "//pkg/kv/kvserver/protectedts/ptstorage", - "//pkg/kv/kvserver/protectedts/ptverifier", "//pkg/settings/cluster", "//pkg/sql/sqlutil", "//pkg/util/metric", diff --git a/pkg/kv/kvserver/protectedts/ptprovider/provider.go b/pkg/kv/kvserver/protectedts/ptprovider/provider.go index 1f55a2557808..377039e80c53 100644 --- a/pkg/kv/kvserver/protectedts/ptprovider/provider.go +++ b/pkg/kv/kvserver/protectedts/ptprovider/provider.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptstorage" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptverifier" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" "github.com/cockroachdb/cockroach/pkg/util/metric" @@ -42,7 +41,6 @@ type Config struct { // Provider is the concrete implementation of protectedts.Provider interface. type Provider struct { protectedts.Storage - protectedts.Verifier protectedts.Cache protectedts.Reconciler metric.Struct @@ -54,7 +52,6 @@ func New(cfg Config) (protectedts.Provider, error) { return nil, err } storage := ptstorage.New(cfg.Settings, cfg.InternalExecutor, cfg.Knobs) - verifier := ptverifier.New(cfg.DB, storage) reconciler := ptreconcile.New(cfg.Settings, cfg.DB, storage, cfg.ReconcileStatusFuncs) cache := ptcache.New(ptcache.Config{ DB: cfg.DB, @@ -65,7 +62,6 @@ func New(cfg Config) (protectedts.Provider, error) { return &Provider{ Storage: storage, Cache: cache, - Verifier: verifier, Reconciler: reconciler, Struct: reconciler.Metrics(), }, nil diff --git a/pkg/kv/kvserver/protectedts/ptverifier/BUILD.bazel b/pkg/kv/kvserver/protectedts/ptverifier/BUILD.bazel deleted file mode 100644 index 969c0cba6634..000000000000 --- a/pkg/kv/kvserver/protectedts/ptverifier/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "ptverifier", - srcs = ["verifier.go"], - importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptverifier", - visibility = ["//visibility:public"], - deps = [ - "//pkg/kv", - "//pkg/kv/kvserver/protectedts", - "//pkg/kv/kvserver/protectedts/ptpb", - "//pkg/roachpb", - "//pkg/util/hlc", - "//pkg/util/uuid", - "@com_github_cockroachdb_errors//:errors", - ], -) - -go_test( - name = "ptverifier_test", - size = "small", - srcs = [ - "main_test.go", - "verifier_test.go", - ], - deps = [ - ":ptverifier", - "//pkg/base", - "//pkg/keys", - "//pkg/kv", - "//pkg/kv/kvclient/kvcoord", - "//pkg/kv/kvserver/protectedts", - "//pkg/kv/kvserver/protectedts/ptpb", - "//pkg/kv/kvserver/protectedts/ptstorage", - "//pkg/roachpb", - "//pkg/security", - "//pkg/security/securitytest", - "//pkg/server", - "//pkg/sql/sqlutil", - "//pkg/testutils/serverutils", - "//pkg/testutils/testcluster", - "//pkg/util/leaktest", - "//pkg/util/randutil", - "//pkg/util/uuid", - "@com_github_cockroachdb_errors//:errors", - "@com_github_stretchr_testify//require", - ], -) diff --git a/pkg/kv/kvserver/protectedts/ptverifier/main_test.go b/pkg/kv/kvserver/protectedts/ptverifier/main_test.go deleted file mode 100644 index 8fff74647ba4..000000000000 --- a/pkg/kv/kvserver/protectedts/ptverifier/main_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package ptverifier_test - -import ( - "os" - "testing" - - "github.com/cockroachdb/cockroach/pkg/security" - "github.com/cockroachdb/cockroach/pkg/security/securitytest" - "github.com/cockroachdb/cockroach/pkg/server" - "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" - "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" - "github.com/cockroachdb/cockroach/pkg/util/randutil" -) - -func TestMain(m *testing.M) { - security.SetAssetLoader(securitytest.EmbeddedAssets) - randutil.SeedForTests() - serverutils.InitTestServerFactory(server.TestServerFactory) - serverutils.InitTestClusterFactory(testcluster.TestClusterFactory) - os.Exit(m.Run()) -} diff --git a/pkg/kv/kvserver/protectedts/ptverifier/verifier.go b/pkg/kv/kvserver/protectedts/ptverifier/verifier.go deleted file mode 100644 index c3f46349c80d..000000000000 --- a/pkg/kv/kvserver/protectedts/ptverifier/verifier.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package ptverifier - -import ( - "bytes" - "context" - "fmt" - - "github.com/cockroachdb/cockroach/pkg/kv" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" - "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/util/hlc" - "github.com/cockroachdb/cockroach/pkg/util/uuid" - "github.com/cockroachdb/errors" -) - -// verifier implements protectedts.Verifier. -type verifier struct { - db *kv.DB - s protectedts.Storage -} - -// New returns a new Verifier. -func New(db *kv.DB, s protectedts.Storage) protectedts.Verifier { - return &verifier{db: db, s: s} -} - -// Verify verifies that a record with the provided id is verified. -// If it is not verified this call will perform verification and mark the -// record as verified. -func (v *verifier) Verify(ctx context.Context, id uuid.UUID) error { - // First we go read the record and note the timestamp at which we read it. - r, ts, err := getRecordWithTimestamp(ctx, v.s, v.db, id) - if err != nil { - return errors.Wrapf(err, "failed to fetch record %s", id) - } - - // TODO(adityamaru): Remove this once we delete all `Verify` calls. The new - // subsystem is not going to provide verification semantics. Until then mark - // the record as verified so it is a noop. - if r.DeprecatedSpans == nil { - return nil - } - - if r.Verified { // already verified - return nil - } - - b := makeVerificationBatch(r, ts) - if err := v.db.Run(ctx, &b); err != nil { - return err - } - - // Check the responses and synthesize an error if one occurred. - if err := parseResponse(&b, r); err != nil { - return err - } - // Mark the record as verified. - return errors.Wrapf(v.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { - return v.s.MarkVerified(ctx, txn, id) - }), "failed to mark %v as verified", id) -} - -// getRecordWithTimestamp fetches the record with the provided id and returns -// the hlc timestamp at which that read occurred. -func getRecordWithTimestamp( - ctx context.Context, s protectedts.Storage, db *kv.DB, id uuid.UUID, -) (r *ptpb.Record, readAt hlc.Timestamp, err error) { - if err = db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { - r, err = s.GetRecord(ctx, txn, id) - readAt = txn.ReadTimestamp() - return err - }); err != nil { - return nil, hlc.Timestamp{}, err - } - return r, readAt, nil -} - -func makeVerificationBatch(r *ptpb.Record, aliveAt hlc.Timestamp) kv.Batch { - // Need to perform validation, build a batch and run it. - mergedSpans, _ := roachpb.MergeSpans(&r.DeprecatedSpans) - var b kv.Batch - for _, s := range mergedSpans { - var req roachpb.AdminVerifyProtectedTimestampRequest - req.RecordAliveAt = aliveAt - req.Protected = r.Timestamp - req.RecordID = r.ID.GetUUID() - req.Key = s.Key - req.EndKey = s.EndKey - b.AddRawRequest(&req) - } - return b -} - -func parseResponse(b *kv.Batch, r *ptpb.Record) error { - rawResponse := b.RawResponse() - var errBuilder bytes.Buffer - for _, resp := range rawResponse.Responses { - resp := resp.GetInner().(*roachpb.AdminVerifyProtectedTimestampResponse) - if len(resp.DeprecatedFailedRanges) == 0 && len(resp.VerificationFailedRanges) == 0 { - continue - } - - // Write the error header the first time we encounter failed ranges. - if errBuilder.Len() == 0 { - _, _ = errBuilder.WriteString(fmt.Sprintf("failed to verify protection record %s with ts: %s:\n", - r.ID.String(), r.Timestamp.String())) - } - - useDeprecated := len(resp.VerificationFailedRanges) == 0 - for _, failedRange := range resp.VerificationFailedRanges { - if failedRange.Reason != "" { - // Write the per range reason for failure. - _, _ = errBuilder.WriteString(fmt.Sprintf("range ID: %d, range span: %s - %s: %s\n", - failedRange.RangeID, failedRange.StartKey.String(), failedRange.EndKey.String(), - failedRange.Reason)) - } else { - // If no reason was saved, dump relevant information. - _, _ = errBuilder.WriteString(fmt.Sprintf("range ID: %d, range span: %s - %s\n", - failedRange.RangeID, failedRange.StartKey.String(), failedRange.EndKey.String())) - } - } - - if !useDeprecated { - continue - } - - for _, rangeDesc := range resp.DeprecatedFailedRanges { - _, _ = errBuilder.WriteString(fmt.Sprintf("range ID: %d, range span: %s - %s\n", - rangeDesc.RangeID, rangeDesc.StartKey.String(), rangeDesc.EndKey.String())) - } - } - if errBuilder.Len() > 0 { - return errors.Newf("protected ts verification error: %s", errBuilder.String()) - } - return nil -} diff --git a/pkg/kv/kvserver/protectedts/ptverifier/verifier_test.go b/pkg/kv/kvserver/protectedts/ptverifier/verifier_test.go deleted file mode 100644 index de87dfbd5339..000000000000 --- a/pkg/kv/kvserver/protectedts/ptverifier/verifier_test.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package ptverifier_test - -import ( - "context" - "sync/atomic" - "testing" - "time" - - "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/kv" - "github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptstorage" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptverifier" - "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" - "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" - "github.com/cockroachdb/cockroach/pkg/util/leaktest" - "github.com/cockroachdb/cockroach/pkg/util/uuid" - "github.com/cockroachdb/errors" - "github.com/stretchr/testify/require" -) - -// TestVerifier tests the business logic of verification by mocking out the -// actual verification requests but using a real implementation of -// protectedts.Storage. -func TestVerifier(t *testing.T) { - defer leaktest.AfterTest(t)() - - ctx := context.Background() - tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{}) - defer tc.Stopper().Stop(ctx) - - s := tc.Server(0) - var senderFunc atomic.Value - senderFunc.Store(kv.SenderFunc(nil)) - ds := s.DistSenderI().(*kvcoord.DistSender) - tsf := kvcoord.NewTxnCoordSenderFactory( - kvcoord.TxnCoordSenderFactoryConfig{ - AmbientCtx: s.DB().AmbientContext, - HeartbeatInterval: time.Second, - Settings: s.ClusterSettings(), - Clock: s.Clock(), - Stopper: s.Stopper(), - }, - kv.SenderFunc(func(ctx context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { - if f := senderFunc.Load().(kv.SenderFunc); f != nil { - return f(ctx, ba) - } - return ds.Send(ctx, ba) - }), - ) - - pts := ptstorage.New(s.ClusterSettings(), s.InternalExecutor().(sqlutil.InternalExecutor), nil /* knobs */) - withDB := ptstorage.WithDatabase(pts, s.DB()) - db := kv.NewDB(s.DB().AmbientContext, tsf, s.Clock(), s.Stopper()) - ptv := ptverifier.New(db, pts) - makeTableSpan := func(tableID uint32) roachpb.Span { - k := keys.SystemSQLCodec.TablePrefix(tableID) - return roachpb.Span{Key: k, EndKey: k.PrefixEnd()} - } - - createRecord := func(t *testing.T, tables ...uint32) *ptpb.Record { - spans := make([]roachpb.Span, len(tables)) - for i, tid := range tables { - spans[i] = makeTableSpan(tid) - } - r := ptpb.Record{ - ID: uuid.MakeV4().GetBytes(), - Timestamp: s.Clock().Now(), - Mode: ptpb.PROTECT_AFTER, - DeprecatedSpans: spans, - } - require.Nil(t, s.DB().Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { - return pts.Protect(ctx, txn, &r) - })) - return &r - } - ensureVerified := func(t *testing.T, id uuid.UUID, verified bool) { - got, err := withDB.GetRecord(ctx, nil, id) - require.NoError(t, err) - require.Equal(t, verified, got.Verified) - } - release := func(t *testing.T, id uuid.UUID) { - require.NoError(t, withDB.Release(ctx, nil, id)) - } - for _, c := range []struct { - name string - test func(t *testing.T) - }{ - { - name: "record doesn't exist", - test: func(t *testing.T) { - require.Regexp(t, protectedts.ErrNotExists.Error(), - ptv.Verify(ctx, uuid.MakeV4()).Error()) - }, - }, - { - name: "verification failed with injected error", - test: func(t *testing.T) { - defer senderFunc.Store(senderFunc.Load()) - r := createRecord(t, 42) - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - return nil, roachpb.NewError(errors.New("boom")) - } - return ds.Send(ctx, ba) - })) - require.Regexp(t, "boom", ptv.Verify(ctx, r.ID.GetUUID()).Error()) - ensureVerified(t, r.ID.GetUUID(), false) - release(t, r.ID.GetUUID()) - }, - }, - { - name: "verification failed with injected response", - test: func(t *testing.T) { - defer senderFunc.Store(senderFunc.Load()) - r := createRecord(t, 42) - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - var resp roachpb.BatchResponse - resp.Add(&roachpb.AdminVerifyProtectedTimestampResponse{ - VerificationFailedRanges: []roachpb.AdminVerifyProtectedTimestampResponse_FailedRange{{ - RangeID: 42, - StartKey: roachpb.RKey(r.DeprecatedSpans[0].Key), - EndKey: roachpb.RKey(r.DeprecatedSpans[0].EndKey), - }}, - }) - return &resp, nil - } - return ds.Send(ctx, ba) - })) - require.Regexp(t, "protected ts verification error: failed to verify protection.*\n"+ - "range ID: 42, range span: /Table/42 - /Table/43", - ptv.Verify(ctx, r.ID.GetUUID()).Error()) - ensureVerified(t, r.ID.GetUUID(), false) - release(t, r.ID.GetUUID()) - }, - }, - { - name: "verification failed with injected response over two spans", - test: func(t *testing.T) { - defer senderFunc.Store(senderFunc.Load()) - r := createRecord(t, 42, 12) - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - var resp roachpb.BatchResponse - resp.Add(&roachpb.AdminVerifyProtectedTimestampResponse{ - VerificationFailedRanges: []roachpb.AdminVerifyProtectedTimestampResponse_FailedRange{{ - RangeID: 42, - StartKey: roachpb.RKey(r.DeprecatedSpans[0].Key), - EndKey: roachpb.RKey(r.DeprecatedSpans[0].EndKey), - Reason: "foo", - }}, - }) - resp.Add(&roachpb.AdminVerifyProtectedTimestampResponse{ - VerificationFailedRanges: []roachpb.AdminVerifyProtectedTimestampResponse_FailedRange{{ - RangeID: 12, - StartKey: roachpb.RKey(r.DeprecatedSpans[1].Key), - EndKey: roachpb.RKey(r.DeprecatedSpans[1].EndKey), - Reason: "bar", - }}, - }) - return &resp, nil - } - return ds.Send(ctx, ba) - })) - require.Regexp(t, "protected ts verification error: failed to verify protection.*\n"+ - "range ID: 42, "+ - "range span: /Table/42 - /Table/43: foo\nrange ID: 12, "+ - "range span: /Table/12 - /Table/13: bar", - ptv.Verify(ctx, r.ID.GetUUID()).Error()) - ensureVerified(t, r.ID.GetUUID(), false) - release(t, r.ID.GetUUID()) - }, - }, - { - // TODO(adityamaru): Remove in 21.2. - name: "verification failed with deprecated failed ranges response", - test: func(t *testing.T) { - defer senderFunc.Store(senderFunc.Load()) - r := createRecord(t, 42) - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - var resp roachpb.BatchResponse - resp.Add(&roachpb.AdminVerifyProtectedTimestampResponse{ - DeprecatedFailedRanges: []roachpb.RangeDescriptor{{ - RangeID: 42, - StartKey: roachpb.RKey(r.DeprecatedSpans[0].Key), - EndKey: roachpb.RKey(r.DeprecatedSpans[0].EndKey), - }}, - }) - return &resp, nil - } - return ds.Send(ctx, ba) - })) - require.Regexp(t, "protected ts verification error: failed to verify protection."+ - "*\nrange ID: 42, range span: /Table/42 - /Table/43", - ptv.Verify(ctx, r.ID.GetUUID()).Error()) - ensureVerified(t, r.ID.GetUUID(), false) - release(t, r.ID.GetUUID()) - }, - }, - { - name: "verification succeeded", - test: func(t *testing.T) { - defer senderFunc.Store(senderFunc.Load()) - r := createRecord(t, 42) - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - var resp roachpb.BatchResponse - resp.Add(&roachpb.AdminVerifyProtectedTimestampResponse{}) - return &resp, nil - } - return ds.Send(ctx, ba) - })) - require.NoError(t, ptv.Verify(ctx, r.ID.GetUUID())) - ensureVerified(t, r.ID.GetUUID(), true) - // Show that we don't send again once we've already verified. - sawVerification := false - senderFunc.Store(kv.SenderFunc(func( - ctx context.Context, ba roachpb.BatchRequest, - ) (*roachpb.BatchResponse, *roachpb.Error) { - if _, ok := ba.GetArg(roachpb.AdminVerifyProtectedTimestamp); ok { - sawVerification = true - } - return ds.Send(ctx, ba) - })) - require.NoError(t, ptv.Verify(ctx, r.ID.GetUUID())) - require.False(t, sawVerification) - release(t, r.ID.GetUUID()) - }, - }, - } { - t.Run(c.name, c.test) - } -}