From 8846aacd09b963f5f0eba421080cffa3285229e8 Mon Sep 17 00:00:00 2001 From: adamcfraser Date: Mon, 11 Sep 2023 16:36:05 -0700 Subject: [PATCH 1/3] Update XattrStore, remove subdocXattrStore Remove subdocXattrStore (previously used to support working with multiple gocb versions), moving required functions not already included in XattrStore into XattrStore. --- base/bucket_gocb_test.go | 109 ++++++++++++++++---------------- base/collection_xattr.go | 36 +++++------ base/collection_xattr_common.go | 59 ++++++----------- base/error.go | 4 +- base/leaky_datastore.go | 47 +++++--------- go.mod | 4 +- go.sum | 4 ++ rest/api_test.go | 4 +- rest/importtest/import_test.go | 37 ++++------- rest/user_api_test.go | 15 ++--- 10 files changed, 131 insertions(+), 188 deletions(-) diff --git a/base/bucket_gocb_test.go b/base/bucket_gocb_test.go index 574e6c3893..983124c3b2 100644 --- a/base/bucket_gocb_test.go +++ b/base/bucket_gocb_test.go @@ -441,10 +441,6 @@ func SkipXattrTestsIfNotEnabled(t *testing.T) { if !TestUseXattrs() { t.Skip("XATTR based tests not enabled. Enable via SG_TEST_USE_XATTRS=true environment variable") } - - if UnitTestUrlIsWalrus() { - t.Skip("This test won't work under walrus until https://github.com/couchbase/sync_gateway/issues/2390") - } } // TestXattrWriteCasSimple. Validates basic write of document with xattr, and retrieval of the same doc w/ xattr. @@ -500,7 +496,7 @@ func TestXattrWriteCasSimple(t *testing.T) { // Validate against $document.value_crc32c var retrievedVxattr map[string]interface{} - _, err = dataStore.GetWithXattr(key, "$document", "", retrievedVal, &retrievedVxattr, nil) + _, err = dataStore.GetWithXattr(key, "$document", "", &retrievedVal, &retrievedVxattr, nil) require.NoError(t, err) vxattrCrc32c, ok := retrievedVxattr["value_crc32c"].(string) assert.True(t, ok, "Unable to retrieve virtual xattr crc32c as string") @@ -701,6 +697,11 @@ func TestXattrWriteCasRaw(t *testing.T) { // TestWriteCasTombstoneResurrect. Verifies writing a new document body and xattr to a logically deleted document (xattr still exists) func TestXattrWriteCasTombstoneResurrect(t *testing.T) { + // Skipping on non-Couchbase until CBG-3392 is fixed + if UnitTestUrlIsWalrus() { + t.Skip("Test requires Couchbase Server bucket when using xattrs") + } + SkipXattrTestsIfNotEnabled(t) bucket := GetTestBucket(t) @@ -1150,8 +1151,6 @@ func TestXattrDeleteDocumentUpdate(t *testing.T) { // TestXattrDeleteDocumentAndUpdateXATTR. Delete the document body and update the xattr. func TestXattrDeleteDocumentAndUpdateXattr(t *testing.T) { - SkipXattrTestsIfNotEnabled(t) - bucket := GetTestBucket(t) defer bucket.Close() dataStore := bucket.GetSingleDataStore() @@ -1178,10 +1177,7 @@ func TestXattrDeleteDocumentAndUpdateXattr(t *testing.T) { t.Errorf("Error doing WriteCasWithXattr: %+v", err) } - subdocXattrStore, ok := dataStore.(SubdocXattrStore) - require.True(t, ok) - - _, mutateErr := subdocXattrStore.SubdocUpdateXattrDeleteBody(key, xattrName, 0, cas, xattrVal) + _, mutateErr := dataStore.UpdateXattrDeleteBody(key, xattrName, 0, cas, xattrVal) assert.NoError(t, mutateErr) // Verify delete of body and update of XATTR @@ -1263,22 +1259,26 @@ func TestXattrTombstoneDocAndUpdateXattr(t *testing.T) { updatedXattrVal := make(map[string]interface{}) updatedXattrVal["seq"] = 123 updatedXattrVal["rev"] = "2-1234" + xattrValBytes, err := JSONMarshal(updatedXattrVal) + require.NoError(t, err) // Attempt to delete DocExistsXattrExists, DocExistsNoXattr, and XattrExistsNoDoc // No errors should be returned when deleting these. keys := []string{key1, key2, key3} casValues := []uint64{cas1, cas2, cas3} shouldDeleteBody := []bool{true, true, false} - subdocStore, ok := dataStore.(SubdocXattrStore) - require.True(t, ok) for i, key := range keys { log.Printf("Delete testing for key: %v", key) + // First attempt to update with a bad cas value, and ensure we're getting the expected error - _, errCasMismatch := UpdateTombstoneXattr(subdocStore, key, xattrName, 0, uint64(1234), &updatedXattrVal, shouldDeleteBody[i]) + _, errCasMismatch := dataStore.WriteWithXattr(key, xattrName, 0, uint64(1234), nil, nil, xattrValBytes, true, shouldDeleteBody[i]) + + //_, errCasMismatch := UpdateTombstoneXattr(dataStore, key, xattrName, 0, uint64(1234), &updatedXattrVal, shouldDeleteBody[i]) assert.True(t, IsCasMismatch(errCasMismatch), fmt.Sprintf("Expected cas mismatch for %s", key)) - _, errDelete := UpdateTombstoneXattr(subdocStore, key, xattrName, 0, uint64(casValues[i]), &updatedXattrVal, shouldDeleteBody[i]) + _, errDelete := dataStore.WriteWithXattr(key, xattrName, 0, uint64(casValues[i]), nil, nil, xattrValBytes, true, shouldDeleteBody[i]) + //_, errDelete := UpdateTombstoneXattr(dataStore, key, xattrName, 0, uint64(casValues[i]), &updatedXattrVal, shouldDeleteBody[i]) log.Printf("Delete error: %v", errDelete) assert.NoError(t, errDelete, fmt.Sprintf("Unexpected error deleting %s", key)) @@ -1287,7 +1287,9 @@ func TestXattrTombstoneDocAndUpdateXattr(t *testing.T) { // Now attempt to tombstone key4 (NoDocNoXattr), should not return an error (per SG #3307). Should save xattr metadata. log.Printf("Deleting key: %v", key4) - _, errDelete := UpdateTombstoneXattr(subdocStore, key4, xattrName, 0, uint64(0), &updatedXattrVal, false) + _, errDelete := dataStore.WriteWithXattr(key4, xattrName, 0, uint64(0), nil, nil, xattrValBytes, true, false) + + //_, errDelete := UpdateTombstoneXattr(dataStore, key4, xattrName, 0, uint64(0), &updatedXattrVal, false) assert.NoError(t, errDelete, "Unexpected error tombstoning non-existent doc") assert.True(t, verifyDocDeletedXattrExists(dataStore, key4, xattrName), "Expected doc to be deleted, but xattrs to exist") @@ -1369,7 +1371,7 @@ func TestXattrDeleteDocAndXattr(t *testing.T) { log.Printf("Deleting key: %v", key4) errDelete := dataStore.DeleteWithXattr(key4, xattrName) assert.Error(t, errDelete, "Expected error when calling dataStore.DeleteWithXattr") - assert.Truef(t, pkgerrors.Cause(errDelete) == ErrNotFound, "Exepcted keynotfound error but got %v", errDelete) + assert.Truef(t, IsDocNotFoundError(errDelete), "Exepcted keynotfound error but got %v", errDelete) assert.True(t, verifyDocAndXattrDeleted(dataStore, key4, xattrName), "Expected doc to be deleted") } @@ -1377,6 +1379,9 @@ func TestXattrDeleteDocAndXattr(t *testing.T) { // callback function func TestDeleteWithXattrWithSimulatedRaceResurrect(t *testing.T) { + if UnitTestUrlIsWalrus() { + t.Skip("Test requires CBS in order to use deleteWithXattrInternal callback") + } SkipXattrTestsIfNotEnabled(t) bucket := GetTestBucket(t) @@ -1412,9 +1417,9 @@ func TestDeleteWithXattrWithSimulatedRaceResurrect(t *testing.T) { } // case to KvXattrStore to pass to deleteWithXattrInternal - kvXattrStore, ok := dataStore.(KvXattrStore) + collection, ok := dataStore.(*Collection) require.True(t, ok) - deleteErr := deleteWithXattrInternal(kvXattrStore, key, xattrName, callback) + deleteErr := deleteWithXattrInternal(collection, key, xattrName, callback) assert.Equal(t, 1, numTimesCalledBack) assert.True(t, deleteErr != nil, "We expected an error here, because deleteWithXattrInternal should have "+ " detected that the doc was resurrected during its execution") @@ -1503,7 +1508,7 @@ func TestXattrRetrieveDocumentAndXattr(t *testing.T) { var key4DocResult map[string]interface{} var key4XattrResult map[string]interface{} _, key4err := dataStore.GetWithXattr(key4, xattrName, "", &key4DocResult, &key4XattrResult, nil) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(key4err)) + assert.True(t, IsDocNotFoundError(key4err)) assert.Nil(t, key4DocResult) assert.Nil(t, key4XattrResult) @@ -1512,6 +1517,10 @@ func TestXattrRetrieveDocumentAndXattr(t *testing.T) { // TestXattrMutateDocAndXattr. Validates mutation of doc + xattr in various possible previous states of the document. func TestXattrMutateDocAndXattr(t *testing.T) { + // Skipping on non-Couchbase until CBG-3392 is fixed + if UnitTestUrlIsWalrus() { + t.Skip("Test requires Couchbase Server bucket when using xattrs") + } SkipXattrTestsIfNotEnabled(t) bucket := GetTestBucket(t) @@ -1666,8 +1675,6 @@ func TestGetXattr(t *testing.T) { var response map[string]interface{} - subdocStore, ok := dataStore.(SubdocXattrStore) - // Get Xattr From Existing Doc with Existing Xattr _, err = dataStore.GetXattr(key1, xattrName1, &response) assert.NoError(t, err) @@ -1678,12 +1685,12 @@ func TestGetXattr(t *testing.T) { // Get Xattr From Existing Doc With Non-Existent Xattr -> ErrSubDocBadMulti _, err = dataStore.GetXattr(key1, "non-exist", &response) assert.Error(t, err) - assert.Equal(t, ErrXattrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsXattrNotFoundError(err)) // Get Xattr From Non-Existent Doc With Non-Existent Xattr _, err = dataStore.GetXattr("non-exist", "non-exist", &response) assert.Error(t, err) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsDocNotFoundError(err)) // Get Xattr From Tombstoned Doc With Existing System Xattr (ErrSubDocSuccessDeleted) cas, err = dataStore.WriteCasWithXattr(key2, SyncXattrName, 0, uint64(0), nil, val2, xattrVal2) @@ -1696,14 +1703,13 @@ func TestGetXattr(t *testing.T) { // Get Xattr From Tombstoned Doc With Non-Existent System Xattr -> SubDocMultiPathFailureDeleted _, err = dataStore.GetXattr(key2, "_non-exist", &response) assert.Error(t, err) - assert.Equal(t, ErrXattrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsXattrNotFoundError(err)) // Get Xattr and Body From Tombstoned Doc With Non-Existent System Xattr -> SubDocMultiPathFailureDeleted var v, xv, userXv map[string]interface{} - require.True(t, ok) - _, err = subdocStore.SubdocGetBodyAndXattr(key2, "_non-exist", "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key2, "_non-exist", "", &v, &xv, &userXv) assert.Error(t, err) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsDocNotFoundError(err)) // Get Xattr From Tombstoned Doc With Deleted User Xattr cas, err = dataStore.WriteCasWithXattr(key3, xattrName3, 0, uint64(0), nil, val3, xattrVal3) @@ -1712,7 +1718,7 @@ func TestGetXattr(t *testing.T) { require.NoError(t, err) _, err = dataStore.GetXattr(key3, xattrName3, &response) assert.Error(t, err) - assert.Equal(t, ErrXattrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsXattrNotFoundError(err)) } func TestGetXattrAndBody(t *testing.T) { @@ -1759,48 +1765,45 @@ func TestGetXattrAndBody(t *testing.T) { t.Errorf("Error doing WriteCasWithXattr: %+v", err) } - subdocStore, ok := dataStore.(SubdocXattrStore) - require.True(t, ok) - // Get Xattr From Existing Doc with Existing Xattr var v, xv, userXv map[string]interface{} - _, err = subdocStore.SubdocGetBodyAndXattr(key1, xattrName1, "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key1, xattrName1, "", &v, &xv, &userXv) assert.NoError(t, err) assert.Equal(t, xattrVal1["seq"], xv["seq"]) assert.Equal(t, xattrVal1["rev"], xv["rev"]) // Get body and Xattr From Existing Doc With Non-Existent Xattr -> returns body only - _, err = subdocStore.SubdocGetBodyAndXattr(key1, "non-exist", "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key1, "non-exist", "", &v, &xv, &userXv) assert.NoError(t, err) assert.Equal(t, val1["type"], v["type"]) // Get Xattr From Non-Existent Doc With Non-Existent Xattr - _, err = subdocStore.SubdocGetBodyAndXattr("non-exist", "non-exist", "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr("non-exist", "non-exist", "", &v, &xv, &userXv) assert.Error(t, err) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsDocNotFoundError(err)) // Get Xattr From Tombstoned Doc With Existing System Xattr (ErrSubDocSuccessDeleted) cas, err = dataStore.WriteCasWithXattr(key2, SyncXattrName, 0, uint64(0), nil, val2, xattrVal2) require.NoError(t, err) _, err = dataStore.Remove(key2, cas) require.NoError(t, err) - _, err = subdocStore.SubdocGetBodyAndXattr(key2, SyncXattrName, "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key2, SyncXattrName, "", &v, &xv, &userXv) assert.NoError(t, err) // Get Xattr From Tombstoned Doc With Non-Existent System Xattr -> returns not found - _, err = subdocStore.SubdocGetBodyAndXattr(key2, "_non-exist", "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key2, "_non-exist", "", &v, &xv, &userXv) assert.Error(t, err) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsDocNotFoundError(err)) // Get Xattr From Tombstoned Doc With Deleted User Xattr -> returns not found cas, err = dataStore.WriteCasWithXattr(key3, xattrName3, 0, uint64(0), nil, val3, xattrVal3) require.NoError(t, err) _, err = dataStore.Remove(key3, cas) require.NoError(t, err) - _, err = subdocStore.SubdocGetBodyAndXattr(key3, xattrName3, "", &v, &xv, &userXv) + _, err = dataStore.GetWithXattr(key3, xattrName3, "", &v, &xv, &userXv) assert.Error(t, err) - assert.Equal(t, ErrNotFound, pkgerrors.Cause(err)) + assert.True(t, IsDocNotFoundError(err)) } func TestApplyViewQueryOptions(t *testing.T) { @@ -2075,9 +2078,8 @@ func createTombstonedDoc(t *testing.T, dataStore sgbucket.DataStore, key, xattrN cas, err := dataStore.WriteCasWithXattr(key, xattrName, 0, cas, nil, val, xattrVal) require.NoError(t, err) - subdocStore, _ := dataStore.(SubdocXattrStore) // Create tombstone revision which deletes doc body but preserves XATTR - _, mutateErr := subdocStore.SubdocDeleteBody(key, xattrName, 0, cas) + _, mutateErr := dataStore.DeleteBody(key, xattrName, 0, cas) /* flags := gocb.SubdocDocFlagAccessDeleted _, mutateErr := dataStore.dataStore.MutateInEx(key, flags, gocb.Cas(cas), uint32(0)). @@ -2103,9 +2105,7 @@ func verifyDocAndXattrDeleted(store sgbucket.XattrStore, key, xattrName string) var retrievedVal map[string]interface{} var retrievedXattr map[string]interface{} _, err := store.GetWithXattr(key, xattrName, "", &retrievedVal, &retrievedXattr, nil) - notFound := pkgerrors.Cause(err) == ErrNotFound - - return notFound + return IsDocNotFoundError(err) } func verifyDocDeletedXattrExists(store sgbucket.XattrStore, key, xattrName string) bool { @@ -2128,9 +2128,6 @@ func TestUpdateXattrWithDeleteBodyAndIsDelete(t *testing.T) { defer bucket.Close() dataStore := bucket.GetSingleDataStore() - subdocXattrStore, ok := dataStore.(SubdocXattrStore) - require.True(t, ok) - // Create a document with extended attributes key := "DocWithXattrAndIsDelete" val := make(map[string]interface{}) @@ -2151,7 +2148,10 @@ func TestUpdateXattrWithDeleteBodyAndIsDelete(t *testing.T) { updatedXattrVal["rev"] = "2-EmDC" // Attempt to delete the document body (deleteBody = true); isDelete is true to mark this doc as a tombstone. - _, errDelete := UpdateTombstoneXattr(subdocXattrStore, key, xattrKey, 0, cas, &updatedXattrVal, true) + + xattrValBytes, err := JSONMarshal(updatedXattrVal) + require.NoError(t, err) + _, errDelete := dataStore.WriteWithXattr(key, xattrKey, 0, cas, nil, nil, xattrValBytes, true, true) assert.NoError(t, errDelete, fmt.Sprintf("Unexpected error deleting %s", key)) assert.True(t, verifyDocDeletedXattrExists(dataStore, key, xattrKey), fmt.Sprintf("Expected doc %s to be deleted", key)) @@ -2234,9 +2234,6 @@ func TestInsertTombstoneWithXattr(t *testing.T) { defer bucket.Close() dataStore := bucket.GetSingleDataStore() - subdocXattrStore, ok := dataStore.(SubdocXattrStore) - require.True(t, ok) - // Create a document with extended attributes key := "InsertedTombstoneDoc" val := make(map[string]interface{}) @@ -2249,13 +2246,15 @@ func TestInsertTombstoneWithXattr(t *testing.T) { cas := uint64(0) // Attempt to delete the document body (deleteBody = true); isDelete is true to mark this doc as a tombstone. - _, errDelete := UpdateTombstoneXattr(subdocXattrStore, key, xattrKey, 0, cas, &xattrVal, false) + xattrValBytes, err := JSONMarshal(xattrVal) + require.NoError(t, err) + _, errDelete := dataStore.WriteWithXattr(key, xattrKey, 0, cas, nil, nil, xattrValBytes, true, false) assert.NoError(t, errDelete, fmt.Sprintf("Unexpected error deleting %s", key)) assert.True(t, verifyDocDeletedXattrExists(dataStore, key, xattrKey), fmt.Sprintf("Expected doc %s to be deleted", key)) var docResult map[string]interface{} var xattrResult map[string]interface{} - _, err := dataStore.GetWithXattr(key, xattrKey, "", &docResult, &xattrResult, nil) + _, err = dataStore.GetWithXattr(key, xattrKey, "", &docResult, &xattrResult, nil) assert.NoError(t, err) assert.Len(t, docResult, 0) assert.Equal(t, "1-EmDC", xattrResult["rev"]) diff --git a/base/collection_xattr.go b/base/collection_xattr.go index 40de35ac1a..0262fd6d9c 100644 --- a/base/collection_xattr.go +++ b/base/collection_xattr.go @@ -26,13 +26,12 @@ var UpsertSpecXattr = &gocb.UpsertSpecOptions{IsXattr: true} var RemoveSpecXattr = &gocb.RemoveSpecOptions{IsXattr: true} var LookupOptsAccessDeleted *gocb.LookupInOptions -var _ SubdocXattrStore = &Collection{} - -// IsSupported is a shim that queries the parent bucket's feature - required for implementing SubdocXattrStore +// IsSupported is a shim that queries the parent bucket's feature func (c *Collection) IsSupported(feature sgbucket.BucketStoreFeature) bool { return c.Bucket.IsSupported(feature) } +var _ sgbucket.XattrStore = &Collection{} var _ UserXattrStore = &Collection{} func init() { @@ -40,7 +39,6 @@ func init() { LookupOptsAccessDeleted.Internal.DocFlags = gocb.SubdocDocFlagAccessDeleted } -// GetSpec required for SubdocXattrStore func (c *Collection) GetSpec() BucketSpec { return c.Bucket.Spec } @@ -90,10 +88,6 @@ func (c *Collection) DeleteXattrs(k string, xattrKeys ...string) (err error) { return DeleteXattrs(c, k, xattrKeys...) } -func (c *Collection) UpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}, deleteBody bool, isDelete bool) (casOut uint64, err error) { - return UpdateTombstoneXattr(c, k, xattrKey, exp, cas, xv, deleteBody) -} - // SubdocGetXattr retrieves the named xattr // Notes on error handling // - gocb v2 returns subdoc errors at the op level, in the ContentAt response @@ -286,9 +280,9 @@ func (c *Collection) SubdocGetBodyAndXattr(k string, xattrKey string, userXattrK return cas, err } -// SubdocInsertXattr inserts a new server tombstone with an associated mobile xattr. Writes cas and crc32c to the xattr using +// InsertXattr inserts a new server tombstone with an associated mobile xattr. Writes cas and crc32c to the xattr using // macro expansion. -func (c *Collection) SubdocInsertXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { +func (c *Collection) InsertXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -319,9 +313,9 @@ func (c *Collection) SubdocInsertXattr(k string, xattrKey string, exp uint32, ca return uint64(result.Cas()), nil } -// SubdocInsertXattr inserts a document and associated mobile xattr in a single mutateIn operation. Writes cas and crc32c to the xattr using +// InsertBodyAndXattr inserts a document and associated mobile xattr in a single mutateIn operation. Writes cas and crc32c to the xattr using // macro expansion. -func (c *Collection) SubdocInsertBodyAndXattr(k string, xattrKey string, exp uint32, v interface{}, xv interface{}) (casOut uint64, err error) { +func (c *Collection) InsertBodyAndXattr(k string, xattrKey string, exp uint32, v interface{}, xv interface{}) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -390,9 +384,9 @@ func (c *Collection) SubdocSetXattr(k string, xattrKey string, xv interface{}) ( return uint64(result.Cas()), nil } -// SubdocUpdateXattr updates the xattr on an existing document. Writes cas and crc32c to the xattr using +// UpdateXattr updates the xattr on an existing document. Writes cas and crc32c to the xattr using // macro expansion. -func (c *Collection) SubdocUpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { +func (c *Collection) UpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -415,9 +409,9 @@ func (c *Collection) SubdocUpdateXattr(k string, xattrKey string, exp uint32, ca return uint64(result.Cas()), nil } -// SubdocUpdateBodyAndXattr updates the document body and xattr of an existing document. Writes cas and crc32c to the xattr using +// UpdateBodyAndXattr updates the document body and xattr of an existing document. Writes cas and crc32c to the xattr using // macro expansion. -func (c *Collection) SubdocUpdateBodyAndXattr(k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { +func (c *Collection) UpdateBodyAndXattr(k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -440,9 +434,9 @@ func (c *Collection) SubdocUpdateBodyAndXattr(k string, xattrKey string, exp uin return uint64(result.Cas()), nil } -// SubdocUpdateBodyAndXattr deletes the document body and updates the xattr of an existing document. Writes cas and crc32c to the xattr using +// UpdateXattrDeleteBody deletes the document body and updates the xattr of an existing document. Writes cas and crc32c to the xattr using // macro expansion. -func (c *Collection) SubdocUpdateXattrDeleteBody(k, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { +func (c *Collection) UpdateXattrDeleteBody(k, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -497,7 +491,7 @@ func (c *Collection) SubdocDeleteXattrs(k string, xattrKeys ...string) error { } // SubdocDeleteXattr deletes the document body and associated xattr of an existing document. -func (c *Collection) SubdocDeleteBodyAndXattr(k string, xattrKey string) (err error) { +func (c *Collection) DeleteBodyAndXattr(k string, xattrKey string) (err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() @@ -525,8 +519,8 @@ func (c *Collection) SubdocDeleteBodyAndXattr(k string, xattrKey string) (err er return mutateErr } -// SubdocDeleteBody deletes the document body of an existing document, and updates cas and crc32c in the associated xattr. -func (c *Collection) SubdocDeleteBody(k string, xattrKey string, exp uint32, cas uint64) (casOut uint64, err error) { +// DeleteBody deletes the document body of an existing document, and updates cas and crc32c in the associated xattr. +func (c *Collection) DeleteBody(k string, xattrKey string, exp uint32, cas uint64) (casOut uint64, err error) { c.Bucket.waitForAvailKvOp() defer c.Bucket.releaseKvOp() diff --git a/base/collection_xattr_common.go b/base/collection_xattr_common.go index 84d555d884..146842c128 100644 --- a/base/collection_xattr_common.go +++ b/base/collection_xattr_common.go @@ -22,45 +22,22 @@ const ( xattrMacroValueCrc32c = "value_crc32c" ) -// SubdocXattrStore interface defines the set of operations Sync Gateway uses to manage and interact with xattrs -type SubdocXattrStore interface { - SubdocGetXattr(k string, xattrKey string, xv interface{}) (casOut uint64, err error) - SubdocGetBodyAndXattr(k string, xattrKey string, userXattrKey string, rv interface{}, xv interface{}, uxv interface{}) (cas uint64, err error) - SubdocInsertXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) - SubdocInsertBodyAndXattr(k string, xattrKey string, exp uint32, v interface{}, xv interface{}) (casOut uint64, err error) - SubdocSetXattr(k string, xattrKey string, xv interface{}) (casOut uint64, err error) - SubdocUpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) - SubdocUpdateBodyAndXattr(k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) - SubdocUpdateXattrDeleteBody(k, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) - SubdocDeleteXattr(k string, xattrKey string, cas uint64) error - SubdocDeleteXattrs(k string, xattrKeys ...string) error - SubdocDeleteBodyAndXattr(k string, xattrKey string) error - SubdocDeleteBody(k string, xattrKey string, exp uint32, cas uint64) (casOut uint64, err error) - - // TODO: These could be factored out of this interface and passed in via other means - GetSpec on a DataStore doesn't fit well and IsSupported generally applies only to a bucket not a collection/datastore - GetSpec() BucketSpec // GetSpec is used for retry counts/times - IsSupported(feature sgbucket.BucketStoreFeature) bool // IsSupported typically defined on the bucket, not the datastore - isRecoverableReadError(err error) bool // TODO: Can isRecoverableReadError be handled by GoCB's retry logic? - isRecoverableWriteError(err error) bool // TODO: Can isRecoverableWriteError by handled by GoCB's retry logic? -} - // Utilities for creating/deleting user xattr. For test use type UserXattrStore = sgbucket.UserXattrStore // KvXattrStore is used for xattr_common functions that perform subdoc and standard kv operations type KvXattrStore interface { sgbucket.KVStore - SubdocXattrStore } // CAS-safe write of a document and it's associated named xattr -func WriteCasWithXattr(store SubdocXattrStore, k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { +func WriteCasWithXattr(store *Collection, k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { worker := func() (shouldRetry bool, err error, value uint64) { // cas=0 specifies an insert if cas == 0 { - casOut, err = store.SubdocInsertBodyAndXattr(k, xattrKey, exp, v, xv) + casOut, err = store.InsertBodyAndXattr(k, xattrKey, exp, v, xv) if err != nil { shouldRetry = store.isRecoverableWriteError(err) return shouldRetry, err, uint64(0) @@ -71,14 +48,14 @@ func WriteCasWithXattr(store SubdocXattrStore, k string, xattrKey string, exp ui // Otherwise, replace existing value if v != nil { // Have value and xattr value - update both - casOut, err = store.SubdocUpdateBodyAndXattr(k, xattrKey, exp, cas, opts, v, xv) + casOut, err = store.UpdateBodyAndXattr(k, xattrKey, exp, cas, opts, v, xv) if err != nil { shouldRetry = store.isRecoverableWriteError(err) return shouldRetry, err, uint64(0) } } else { // Update xattr only - casOut, err = store.SubdocUpdateXattr(k, xattrKey, exp, cas, xv) + casOut, err = store.UpdateXattr(k, xattrKey, exp, cas, xv) if err != nil { shouldRetry = store.isRecoverableWriteError(err) return shouldRetry, err, uint64(0) @@ -98,7 +75,7 @@ func WriteCasWithXattr(store SubdocXattrStore, k string, xattrKey string, exp ui // Single attempt to update a document and xattr. Setting isDelete=true and value=nil will delete the document body. Both // update types (UpdateTombstoneXattr, WriteCasWithXattr) include recoverable error retry. -func WriteWithXattr(store SubdocXattrStore, k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, value []byte, xattrValue []byte, isDelete bool, deleteBody bool) (casOut uint64, err error) { // If this is a tombstone, we want to delete the document and update the xattr +func WriteWithXattr(store *Collection, k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, value []byte, xattrValue []byte, isDelete bool, deleteBody bool) (casOut uint64, err error) { // If this is a tombstone, we want to delete the document and update the xattr if isDelete { return UpdateTombstoneXattr(store, k, xattrKey, exp, cas, xattrValue, deleteBody) } else { @@ -108,7 +85,7 @@ func WriteWithXattr(store SubdocXattrStore, k string, xattrKey string, exp uint3 } // CAS-safe update of a document's xattr (only). Deletes the document body if deleteBody is true. -func UpdateTombstoneXattr(store SubdocXattrStore, k string, xattrKey string, exp uint32, cas uint64, xv interface{}, deleteBody bool) (casOut uint64, err error) { +func UpdateTombstoneXattr(store *Collection, k string, xattrKey string, exp uint32, cas uint64, xv interface{}, deleteBody bool) (casOut uint64, err error) { // WriteCasWithXattr always stamps the xattr with the new cas using macro expansion, into a top-level property called 'cas'. // This is the only use case for macro expansion today - if more cases turn up, should change the sg-bucket API to handle this more generically. @@ -120,16 +97,16 @@ func UpdateTombstoneXattr(store SubdocXattrStore, k string, xattrKey string, exp // If deleteBody == true, remove the body and update xattr if deleteBody { - casOut, tombstoneErr = store.SubdocUpdateXattrDeleteBody(k, xattrKey, exp, cas, xv) + casOut, tombstoneErr = store.UpdateXattrDeleteBody(k, xattrKey, exp, cas, xv) } else { if cas == 0 { // if cas == 0, create a new server tombstone with xattr - casOut, tombstoneErr = store.SubdocInsertXattr(k, xattrKey, exp, cas, xv) + casOut, tombstoneErr = store.InsertXattr(k, xattrKey, exp, cas, xv) // If one-step tombstone creation is not supported, set flag for document body removal requiresBodyRemoval = !store.IsSupported(sgbucket.BucketStoreFeatureCreateDeletedWithXattr) } else { // If cas is non-zero, this is an already existing tombstone. Update xattr only - casOut, tombstoneErr = store.SubdocUpdateXattr(k, xattrKey, exp, cas, xv) + casOut, tombstoneErr = store.UpdateXattr(k, xattrKey, exp, cas, xv) } } @@ -154,7 +131,7 @@ func UpdateTombstoneXattr(store SubdocXattrStore, k string, xattrKey string, exp if requiresBodyRemoval { worker := func() (shouldRetry bool, err error, value uint64) { - casOut, removeErr := store.SubdocDeleteBody(k, xattrKey, exp, cas) + casOut, removeErr := store.DeleteBody(k, xattrKey, exp, cas) if removeErr != nil { // If there is a cas mismatch the body has since been updated and so we don't need to bother removing // body in this operation @@ -187,7 +164,7 @@ func UpdateTombstoneXattr(store SubdocXattrStore, k string, xattrKey string, exp // A zero CAS in `previous` is interpreted as no document existing; this can be used to short- // circuit the initial Get when the document is unlikely to already exist. -func WriteUpdateWithXattr(store SubdocXattrStore, k string, xattrKey string, userXattrKey string, exp uint32, opts *sgbucket.MutateInOptions, previous *sgbucket.BucketDocument, callback sgbucket.WriteUpdateWithXattrFunc) (casOut uint64, err error) { +func WriteUpdateWithXattr(store *Collection, k string, xattrKey string, userXattrKey string, exp uint32, opts *sgbucket.MutateInOptions, previous *sgbucket.BucketDocument, callback sgbucket.WriteUpdateWithXattrFunc) (casOut uint64, err error) { var value []byte var xattrValue []byte @@ -266,7 +243,7 @@ func WriteUpdateWithXattr(store SubdocXattrStore, k string, xattrKey string, use } // SetXattr performs a subdoc set on the supplied xattrKey. Implements a retry for recoverable failures. -func SetXattr(store SubdocXattrStore, k string, xattrKey string, xv []byte) (casOut uint64, err error) { +func SetXattr(store *Collection, k string, xattrKey string, xv []byte) (casOut uint64, err error) { worker := func() (shouldRetry bool, err error, value uint64) { casOut, writeErr := store.SubdocSetXattr(k, xattrKey, xv) @@ -292,7 +269,7 @@ func SetXattr(store SubdocXattrStore, k string, xattrKey string, xv []byte) (cas } // RemoveXattr performs a cas safe subdoc delete of the provided key. Will retry if a recoverable failure occurs. -func RemoveXattr(store SubdocXattrStore, k string, xattrKey string, cas uint64) error { +func RemoveXattr(store *Collection, k string, xattrKey string, cas uint64) error { worker := func() (shouldRetry bool, err error, value interface{}) { writeErr := store.SubdocDeleteXattr(k, xattrKey, cas) if writeErr == nil { @@ -317,7 +294,7 @@ func RemoveXattr(store SubdocXattrStore, k string, xattrKey string, cas uint64) // DeleteXattrs performs a subdoc delete of the provided keys. Retries any recoverable failures. Not cas safe does a // straight delete. -func DeleteXattrs(store SubdocXattrStore, k string, xattrKeys ...string) error { +func DeleteXattrs(store *Collection, k string, xattrKeys ...string) error { worker := func() (shouldRetry bool, err error, value interface{}) { writeErr := store.SubdocDeleteXattrs(k, xattrKeys...) if writeErr == nil { @@ -354,7 +331,7 @@ func DeleteXattrs(store SubdocXattrStore, k string, xattrKeys ...string) error { // Expected errors: // - Temporary server overloaded errors, in which case the caller should retry // - If the doc is in the the NoDoc and NoXattr state, it will return a KeyNotFound error -func DeleteWithXattr(store KvXattrStore, k string, xattrKey string) error { +func DeleteWithXattr(store *Collection, k string, xattrKey string) error { // Delegate to internal method that can take a testing-related callback return deleteWithXattrInternal(store, k, xattrKey, nil) } @@ -363,7 +340,7 @@ func DeleteWithXattr(store KvXattrStore, k string, xattrKey string) error { // to simulate the doc having changed state (artifiically injected race condition) type deleteWithXattrRaceInjection func(k string, xattrKey string) -func deleteWithXattrInternal(store KvXattrStore, k string, xattrKey string, callback deleteWithXattrRaceInjection) error { +func deleteWithXattrInternal(store *Collection, k string, xattrKey string, callback deleteWithXattrRaceInjection) error { DebugfCtx(context.TODO(), KeyCRUD, "DeleteWithXattr called with key: %v xattrKey: %v", UD(k), UD(xattrKey)) @@ -371,7 +348,7 @@ func deleteWithXattrInternal(store KvXattrStore, k string, xattrKey string, call // NOTE: ongoing discussion w/ KV Engine team on whether this should handle cases where the body // doesn't exist (eg, a tombstoned xattr doc) by just ignoring the "delete body" mutation, rather // than current behavior of returning gocb.ErrKeyNotFound - mutateErr := store.SubdocDeleteBodyAndXattr(k, xattrKey) + mutateErr := store.DeleteBodyAndXattr(k, xattrKey) if IsDocNotFoundError(mutateErr) { // Invoke the testing related callback. This is a no-op in non-test contexts. if callback != nil { @@ -393,7 +370,7 @@ func deleteWithXattrInternal(store KvXattrStore, k string, xattrKey string, call } -func deleteDocXattrOnly(store SubdocXattrStore, k string, xattrKey string, callback deleteWithXattrRaceInjection) error { +func deleteDocXattrOnly(store *Collection, k string, xattrKey string, callback deleteWithXattrRaceInjection) error { // Do get w/ xattr in order to get cas var retrievedVal map[string]interface{} diff --git a/base/error.go b/base/error.go index 00402471f4..7e29a9e1e0 100644 --- a/base/error.go +++ b/base/error.go @@ -43,11 +43,13 @@ var ( ErrUpdateCancel = &sgError{"Cancel update"} ErrImportCancelledPurged = &sgError{"Import Cancelled Due to Purge"} ErrChannelFeed = &sgError{"Error while building channel feed"} - ErrXattrNotFound = &sgError{"Xattr Not Found"} ErrTimeout = &sgError{"Operation timed out"} ErrPathNotFound = sgbucket.ErrPathNotFound ErrPathExists = sgbucket.ErrPathExists + // ErrXattrNotFound is returned if a requested xattr is not present on a DCP event + ErrXattrNotFound = &sgError{"Xattr Not Found"} + // ErrPartialViewErrors is returned if the view call contains any partial errors. // This is more of a warning, and inspecting ViewResult.Errors is required for detail. ErrPartialViewErrors = &sgError{"Partial errors in view"} diff --git a/base/leaky_datastore.go b/base/leaky_datastore.go index 9b0061ba85..1f8be97ba1 100644 --- a/base/leaky_datastore.go +++ b/base/leaky_datastore.go @@ -333,22 +333,12 @@ func (lds *LeakyDataStore) IsSupported(feature sgbucket.BucketStoreFeature) bool return lds.dataStore.IsSupported(feature) } -//-------- SubdocXattrStore methods: - -func (lds *LeakyDataStore) SubdocGetXattr(k string, xattrKey string, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.GetXattr(k, xattrKey, xv) -} - -func (lds *LeakyDataStore) SubdocGetBodyAndXattr(k string, xattrKey string, userXattrKey string, rv interface{}, xv interface{}, uxv interface{}) (cas uint64, err error) { - return lds.dataStore.GetWithXattr(k, xattrKey, userXattrKey, rv, xv, uxv) -} - -func (lds *LeakyDataStore) SubdocInsertXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).InsertXattr(k, xattrKey, exp, cas, xv) +func (lds *LeakyDataStore) InsertXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { + return lds.dataStore.InsertXattr(k, xattrKey, exp, cas, xv) } -func (lds *LeakyDataStore) SubdocInsertBodyAndXattr(k string, xattrKey string, exp uint32, v interface{}, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).InsertBodyAndXattr(k, xattrKey, exp, v, xv) +func (lds *LeakyDataStore) InsertBodyAndXattr(k string, xattrKey string, exp uint32, v interface{}, xv interface{}) (casOut uint64, err error) { + return lds.dataStore.InsertBodyAndXattr(k, xattrKey, exp, v, xv) } func (lds *LeakyDataStore) SubdocSetXattr(k string, xattrKey string, xv interface{}) (casOut uint64, err error) { @@ -359,32 +349,28 @@ func (lds *LeakyDataStore) SubdocSetXattr(k string, xattrKey string, xv interfac return } -func (lds *LeakyDataStore) SubdocUpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).UpdateXattr(k, xattrKey, exp, cas, xv) +func (lds *LeakyDataStore) UpdateXattr(k string, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { + return lds.dataStore.UpdateXattr(k, xattrKey, exp, cas, xv) } -func (lds *LeakyDataStore) SubdocUpdateBodyAndXattr(k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).UpdateBodyAndXattr(k, xattrKey, exp, cas, opts, v, xv) +func (lds *LeakyDataStore) UpdateBodyAndXattr(k string, xattrKey string, exp uint32, cas uint64, opts *sgbucket.MutateInOptions, v interface{}, xv interface{}) (casOut uint64, err error) { + return lds.dataStore.UpdateBodyAndXattr(k, xattrKey, exp, cas, opts, v, xv) } -func (lds *LeakyDataStore) SubdocUpdateXattrDeleteBody(k, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).UpdateXattrDeleteBody(k, xattrKey, exp, cas, xv) +func (lds *LeakyDataStore) UpdateXattrDeleteBody(k, xattrKey string, exp uint32, cas uint64, xv interface{}) (casOut uint64, err error) { + return lds.dataStore.UpdateXattrDeleteBody(k, xattrKey, exp, cas, xv) } -func (lds *LeakyDataStore) SubdocDeleteXattr(k string, xattrKey string, cas uint64) error { - return lds.dataStore.(sgbucket.XattrStore2).RemoveXattr(k, xattrKey, cas) -} - -func (lds *LeakyDataStore) SubdocDeleteXattrs(k string, xattrKeys ...string) error { - return lds.dataStore.DeleteXattrs(k, xattrKeys...) +func (lds *LeakyDataStore) DeleteXattr(k string, xattrKey string, cas uint64) error { + return lds.dataStore.RemoveXattr(k, xattrKey, cas) } -func (lds *LeakyDataStore) SubdocDeleteBodyAndXattr(k string, xattrKey string) error { - return lds.dataStore.(sgbucket.XattrStore2).DeleteBodyAndXattr(k, xattrKey) +func (lds *LeakyDataStore) DeleteBodyAndXattr(k string, xattrKey string) error { + return lds.dataStore.DeleteBodyAndXattr(k, xattrKey) } -func (lds *LeakyDataStore) SubdocDeleteBody(k string, xattrKey string, exp uint32, cas uint64) (casOut uint64, err error) { - return lds.dataStore.(sgbucket.XattrStore2).DeleteBody(k, xattrKey, exp, cas) +func (lds *LeakyDataStore) DeleteBody(k string, xattrKey string, exp uint32, cas uint64) (casOut uint64, err error) { + return lds.dataStore.DeleteBody(k, xattrKey, exp, cas) } func (lds *LeakyDataStore) GetSpec() BucketSpec { @@ -411,5 +397,4 @@ func (lds *LeakyDataStore) isRecoverableWriteError(err error) bool { // Assert interface compliance: var ( _ sgbucket.DataStore = &LeakyDataStore{} - _ SubdocXattrStore = &LeakyDataStore{} ) diff --git a/go.mod b/go.mod index b45e228c47..5cade5b44b 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/couchbase/gocbcore/v10 v10.2.4-0.20230511103754-8dd1a95f5f33 github.com/couchbase/gomemcached v0.2.1 github.com/couchbase/goutils v0.1.2 - github.com/couchbase/sg-bucket v0.0.0-20230824211334-789eb9635f76 + github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c github.com/couchbaselabs/go-fleecedelta v0.0.0-20200408160354-2ed3f45fde8f - github.com/couchbaselabs/rosmar v0.0.0-20230907142031-ad1bbc5eb635 + github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0 github.com/elastic/gosigar v0.14.2 github.com/felixge/fgprof v0.9.2 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 0b84029376..de1df9c459 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/couchbase/sg-bucket v0.0.0-20230821233142-3881c917c8f1 h1:8YvcPsuHgXE github.com/couchbase/sg-bucket v0.0.0-20230821233142-3881c917c8f1/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= github.com/couchbase/sg-bucket v0.0.0-20230824211334-789eb9635f76 h1:SkanGCttA5xsq43ekHTcIR57ZXvA+MjK5WfdbeDNWek= github.com/couchbase/sg-bucket v0.0.0-20230824211334-789eb9635f76/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= +github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c h1:gs5NrvLopWkZOoI7FdYHWjAuJ3KYQtey/++CTTfJBow= +github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= github.com/couchbase/tools-common v0.0.0-20220810163003-4c3c185822d4 h1:Ub0gZWccoNL+ag59v8S23tbHDknfN1l3CVl7Kb+hFO0= github.com/couchbase/tools-common v0.0.0-20220810163003-4c3c185822d4/go.mod h1:lW0Em1mo5xqQ0mRMzJlg013/6ruyISupTfW7nbLaa1E= github.com/couchbaselabs/go-fleecedelta v0.0.0-20200408160354-2ed3f45fde8f h1:al5DxXEBAUmINnP5dR950gL47424WzncuRpNdg0TWR0= @@ -98,6 +100,8 @@ github.com/couchbaselabs/rosmar v0.0.0-20230825143155-1a2061661a4b h1:lV0I66UKNg github.com/couchbaselabs/rosmar v0.0.0-20230825143155-1a2061661a4b/go.mod h1:abjH2Mv7BHf/OkvRJDtL3cNpF9tjXnpyKYaDIj7YGf4= github.com/couchbaselabs/rosmar v0.0.0-20230907142031-ad1bbc5eb635 h1:us+fVo+22Dg4F4HV6PfBmxRBH+af9UfHN5ABlVWGEAY= github.com/couchbaselabs/rosmar v0.0.0-20230907142031-ad1bbc5eb635/go.mod h1:abjH2Mv7BHf/OkvRJDtL3cNpF9tjXnpyKYaDIj7YGf4= +github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0 h1:iicxlq0vE6wfYDsQ4ziIfSLqpt/LMfDStv/gKPnAiRw= +github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0/go.mod h1:TZR8hpREKvIClLqJY22IbfX6PugLx9Z/HyY2iawyXP0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/rest/api_test.go b/rest/api_test.go index a8b7ce68a2..a2c505cca7 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -1659,11 +1659,9 @@ func TestWriteTombstonedDocUsingXattrs(t *testing.T) { } // Fetch the xattr and make sure it contains the above value - subdocXattrStore, ok := rt.GetSingleDataStore().(base.SubdocXattrStore) - require.True(t, ok) var retrievedVal map[string]interface{} var retrievedXattr map[string]interface{} - _, err = subdocXattrStore.SubdocGetBodyAndXattr("-21SK00U-ujxUO9fU2HezxL", base.SyncXattrName, "", &retrievedVal, &retrievedXattr, nil) + _, err = rt.GetSingleDataStore().GetWithXattr("-21SK00U-ujxUO9fU2HezxL", base.SyncXattrName, "", &retrievedVal, &retrievedXattr, nil) assert.NoError(t, err, "Unexpected Error") assert.Equal(t, "2-466a1fab90a810dc0a63565b70680e4e", retrievedXattr["rev"]) diff --git a/rest/importtest/import_test.go b/rest/importtest/import_test.go index 54268284d3..5dfe099275 100644 --- a/rest/importtest/import_test.go +++ b/rest/importtest/import_test.go @@ -654,9 +654,8 @@ func TestXattrImportMultipleActorOnDemandGet(t *testing.T) { // Modify the document via the SDK to add a new, non-mobile xattr xattrVal := make(map[string]interface{}) xattrVal["actor"] = "not mobile" - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - assert.True(t, ok, "Unable to cast bucket to gocb bucket") - _, mutateErr := subdocXattrStore.SubdocUpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) + + _, mutateErr := dataStore.UpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) assert.NoError(t, mutateErr, "Error updating non-mobile xattr for multi-actor document") @@ -710,9 +709,7 @@ func TestXattrImportMultipleActorOnDemandPut(t *testing.T) { // Modify the document via the SDK to add a new, non-mobile xattr xattrVal := make(map[string]interface{}) xattrVal["actor"] = "not mobile" - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - assert.True(t, ok, "Unable to cast bucket to gocb bucket") - _, mutateErr := subdocXattrStore.SubdocUpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) + _, mutateErr := dataStore.UpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) assert.NoError(t, mutateErr, "Error updating non-mobile xattr for multi-actor document") // Attempt to update the document again via Sync Gateway. Should not trigger import, PUT should be successful, @@ -772,9 +769,7 @@ func TestXattrImportMultipleActorOnDemandFeed(t *testing.T) { // Modify the document via the SDK to add a new, non-mobile xattr xattrVal := make(map[string]interface{}) xattrVal["actor"] = "not mobile" - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - assert.True(t, ok, "Unable to cast bucket to gocb bucket") - _, mutateErr := subdocXattrStore.SubdocUpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) + _, mutateErr := dataStore.UpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) assert.NoError(t, mutateErr, "Error updating non-mobile xattr for multi-actor document") // Wait until crc match count changes @@ -2288,9 +2283,7 @@ func TestUnexpectedBodyOnTombstone(t *testing.T) { // Modify the document via the SDK to add the body back xattrVal := make(map[string]interface{}) xattrVal["actor"] = "not mobile" - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - assert.True(t, ok, "Unable to cast bucket to gocb bucket") - _, mutateErr := subdocXattrStore.SubdocUpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) + _, mutateErr := dataStore.UpdateXattr(mobileKey, "_nonmobile", uint32(0), cas, xattrVal) assert.NoError(t, mutateErr, "Error updating non-mobile xattr for multi-actor document") // Attempt to get the document again via Sync Gateway. Should not trigger import. @@ -2735,9 +2728,7 @@ func TestUserXattrAutoImport(t *testing.T) { // Get Xattr and ensure channel value set correctly var syncData db.SyncData - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - require.True(t, ok) - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData) assert.NoError(t, err) assert.Equal(t, []string{channelName}, syncData.Channels.KeySet()) @@ -2752,7 +2743,7 @@ func TestUserXattrAutoImport(t *testing.T) { assert.NoError(t, err) var syncData2 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData2) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData2) assert.NoError(t, err) assert.Equal(t, syncData.Crc32c, syncData2.Crc32c) @@ -2770,7 +2761,7 @@ func TestUserXattrAutoImport(t *testing.T) { assert.NoError(t, err) var syncData3 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData3) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData3) assert.NoError(t, err) assert.Equal(t, syncData2.Crc32c, syncData3.Crc32c) @@ -2791,7 +2782,7 @@ func TestUserXattrAutoImport(t *testing.T) { assert.Equal(t, int64(3), rt.GetDatabase().DbStats.Database().SyncFunctionCount.Value()) var syncData4 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData4) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData4) assert.NoError(t, err) assert.Equal(t, base.Crc32cHashString(updateVal), syncData4.Crc32c) @@ -2835,8 +2826,6 @@ func TestUserXattrOnDemandImportGET(t *testing.T) { if !ok { t.Skip("Test requires Couchbase Bucket") } - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - require.True(t, ok) // Add doc with SDK err := dataStore.Set(docKey, 0, nil, []byte(`{}`)) @@ -2874,7 +2863,7 @@ func TestUserXattrOnDemandImportGET(t *testing.T) { // Get sync data for doc and ensure user xattr has been used correctly to set channel var syncData db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData) assert.NoError(t, err) assert.Equal(t, []string{channelName}, syncData.Channels.KeySet()) @@ -2888,7 +2877,7 @@ func TestUserXattrOnDemandImportGET(t *testing.T) { rest.RequireStatus(t, resp, http.StatusOK) var syncData2 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData2) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData2) assert.NoError(t, err) assert.Equal(t, syncData.Crc32c, syncData2.Crc32c) @@ -2972,10 +2961,8 @@ func TestUserXattrOnDemandImportWrite(t *testing.T) { // Ensure sync function has ran on import assert.Equal(t, int64(3), rt.GetDatabase().DbStats.Database().SyncFunctionCount.Value()) - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - require.True(t, ok) var syncData db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData) assert.NoError(t, err) assert.Equal(t, []string{channelName}, syncData.Channels.KeySet()) diff --git a/rest/user_api_test.go b/rest/user_api_test.go index 3e94e95d40..55212c7b7e 100644 --- a/rest/user_api_test.go +++ b/rest/user_api_test.go @@ -1218,9 +1218,9 @@ func TestRemovingUserXattr(t *testing.T) { // Get sync data for doc and ensure user xattr has been used correctly to set channel var syncData db.SyncData - subdocStore, ok := rt.GetSingleDataStore().(base.SubdocXattrStore) + dataStore := rt.GetSingleDataStore() require.True(t, ok) - _, err = subdocStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData) assert.NoError(t, err) assert.Equal(t, []string{channelName}, syncData.Channels.KeySet()) @@ -1238,7 +1238,7 @@ func TestRemovingUserXattr(t *testing.T) { // Ensure old channel set with user xattr has been removed var syncData2 db.SyncData - _, err = subdocStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData2) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData2) assert.NoError(t, err) assert.Equal(t, uint64(3), syncData2.Channels[channelName].Seq) @@ -1469,9 +1469,6 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { t.Skip("Test requires Couchbase Bucket") } - subdocXattrStore, ok := dataStore.(base.SubdocXattrStore) - require.True(t, ok) - // Initial PUT resp := rt.SendAdminRequest("PUT", "/{{.keyspace}}/"+docKey, `{}`) RequireStatus(t, resp, http.StatusCreated) @@ -1480,7 +1477,7 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { // Get current sync data var syncData db.SyncData - _, err := subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData) + _, err := dataStore.GetXattr(docKey, base.SyncXattrName, &syncData) assert.NoError(t, err) docRev, err := rt.GetSingleTestDatabaseCollection().GetRevisionCacheForTest().Get(base.TestCtx(t), docKey, syncData.CurrentRev, true, false) @@ -1500,7 +1497,7 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { // Ensure import worked and sequence incremented but that sequence did not var syncData2 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData2) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData2) assert.NoError(t, err) docRev2, err := rt.GetSingleTestDatabaseCollection().GetRevisionCacheForTest().Get(base.TestCtx(t), docKey, syncData.CurrentRev, true, false) @@ -1522,7 +1519,7 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { assert.NoError(t, err) var syncData3 db.SyncData - _, err = subdocXattrStore.SubdocGetXattr(docKey, base.SyncXattrName, &syncData2) + _, err = dataStore.GetXattr(docKey, base.SyncXattrName, &syncData2) assert.NoError(t, err) assert.NotEqual(t, syncData2.CurrentRev, syncData3.CurrentRev) From 31691ddcecd6128332265d179cd4dd06f84d0abf Mon Sep 17 00:00:00 2001 From: adamcfraser Date: Tue, 12 Sep 2023 11:38:46 -0700 Subject: [PATCH 2/3] PR fixes --- base/bucket_gocb_test.go | 3 --- rest/user_api_test.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/base/bucket_gocb_test.go b/base/bucket_gocb_test.go index 983124c3b2..506ee224a1 100644 --- a/base/bucket_gocb_test.go +++ b/base/bucket_gocb_test.go @@ -1274,11 +1274,9 @@ func TestXattrTombstoneDocAndUpdateXattr(t *testing.T) { // First attempt to update with a bad cas value, and ensure we're getting the expected error _, errCasMismatch := dataStore.WriteWithXattr(key, xattrName, 0, uint64(1234), nil, nil, xattrValBytes, true, shouldDeleteBody[i]) - //_, errCasMismatch := UpdateTombstoneXattr(dataStore, key, xattrName, 0, uint64(1234), &updatedXattrVal, shouldDeleteBody[i]) assert.True(t, IsCasMismatch(errCasMismatch), fmt.Sprintf("Expected cas mismatch for %s", key)) _, errDelete := dataStore.WriteWithXattr(key, xattrName, 0, uint64(casValues[i]), nil, nil, xattrValBytes, true, shouldDeleteBody[i]) - //_, errDelete := UpdateTombstoneXattr(dataStore, key, xattrName, 0, uint64(casValues[i]), &updatedXattrVal, shouldDeleteBody[i]) log.Printf("Delete error: %v", errDelete) assert.NoError(t, errDelete, fmt.Sprintf("Unexpected error deleting %s", key)) @@ -1289,7 +1287,6 @@ func TestXattrTombstoneDocAndUpdateXattr(t *testing.T) { log.Printf("Deleting key: %v", key4) _, errDelete := dataStore.WriteWithXattr(key4, xattrName, 0, uint64(0), nil, nil, xattrValBytes, true, false) - //_, errDelete := UpdateTombstoneXattr(dataStore, key4, xattrName, 0, uint64(0), &updatedXattrVal, false) assert.NoError(t, errDelete, "Unexpected error tombstoning non-existent doc") assert.True(t, verifyDocDeletedXattrExists(dataStore, key4, xattrName), "Expected doc to be deleted, but xattrs to exist") diff --git a/rest/user_api_test.go b/rest/user_api_test.go index 55212c7b7e..3a0594f2d1 100644 --- a/rest/user_api_test.go +++ b/rest/user_api_test.go @@ -1248,9 +1248,6 @@ func TestRemovingUserXattr(t *testing.T) { func TestUserXattrRevCache(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) - if base.UnitTestUrlIsWalrus() { - t.Skip("This test only works against Couchbase Server") - } if !base.TestUseXattrs() { t.Skip("This test only works with XATTRS enabled") From 498f4da7dc7991738d907c03048a253756b772eb Mon Sep 17 00:00:00 2001 From: adamcfraser Date: Tue, 12 Sep 2023 14:08:23 -0700 Subject: [PATCH 3/3] go.mod bump for merged dependencies --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5cade5b44b..4fd264e3c5 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/couchbase/gocbcore/v10 v10.2.4-0.20230511103754-8dd1a95f5f33 github.com/couchbase/gomemcached v0.2.1 github.com/couchbase/goutils v0.1.2 - github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c + github.com/couchbase/sg-bucket v0.0.0-20230912183916-dfc81a029d3b github.com/couchbaselabs/go-fleecedelta v0.0.0-20200408160354-2ed3f45fde8f - github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0 + github.com/couchbaselabs/rosmar v0.0.0-20230912185358-c5847ff7b0be github.com/elastic/gosigar v0.14.2 github.com/felixge/fgprof v0.9.2 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index de1df9c459..c1bda197f0 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/couchbase/sg-bucket v0.0.0-20230824211334-789eb9635f76 h1:SkanGCttA5x github.com/couchbase/sg-bucket v0.0.0-20230824211334-789eb9635f76/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c h1:gs5NrvLopWkZOoI7FdYHWjAuJ3KYQtey/++CTTfJBow= github.com/couchbase/sg-bucket v0.0.0-20230908223124-2f1078e2282c/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= +github.com/couchbase/sg-bucket v0.0.0-20230912183916-dfc81a029d3b h1:kTGWkJ30HJQ39SIgtkGjFIOcE3p1L0RwN5sBiVakVAs= +github.com/couchbase/sg-bucket v0.0.0-20230912183916-dfc81a029d3b/go.mod h1:sZO/b/EAyLhPJWOSiaaYRQ3rnveSumRh2wUEfNkK/UM= github.com/couchbase/tools-common v0.0.0-20220810163003-4c3c185822d4 h1:Ub0gZWccoNL+ag59v8S23tbHDknfN1l3CVl7Kb+hFO0= github.com/couchbase/tools-common v0.0.0-20220810163003-4c3c185822d4/go.mod h1:lW0Em1mo5xqQ0mRMzJlg013/6ruyISupTfW7nbLaa1E= github.com/couchbaselabs/go-fleecedelta v0.0.0-20200408160354-2ed3f45fde8f h1:al5DxXEBAUmINnP5dR950gL47424WzncuRpNdg0TWR0= @@ -102,6 +104,8 @@ github.com/couchbaselabs/rosmar v0.0.0-20230907142031-ad1bbc5eb635 h1:us+fVo+22D github.com/couchbaselabs/rosmar v0.0.0-20230907142031-ad1bbc5eb635/go.mod h1:abjH2Mv7BHf/OkvRJDtL3cNpF9tjXnpyKYaDIj7YGf4= github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0 h1:iicxlq0vE6wfYDsQ4ziIfSLqpt/LMfDStv/gKPnAiRw= github.com/couchbaselabs/rosmar v0.0.0-20230911231259-0e6c8bec54b0/go.mod h1:TZR8hpREKvIClLqJY22IbfX6PugLx9Z/HyY2iawyXP0= +github.com/couchbaselabs/rosmar v0.0.0-20230912185358-c5847ff7b0be h1:isy668YrOJq4HooAf0Rxeo1bK3S72Os8QUu8ww3sUGg= +github.com/couchbaselabs/rosmar v0.0.0-20230912185358-c5847ff7b0be/go.mod h1:zLWlNnNF3ro+KtAeOFmghmhonmD0ttXlx314CtFWwb4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=