From 1fba317e2c71aac359aab528c0aad5e36add74ab Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 23 Apr 2024 18:32:25 +0300 Subject: [PATCH 1/5] Migrate use of two deprecated APIs. --- tiledb/libtiledb.pxd | 9 +++++-- tiledb/libtiledb.pyx | 57 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/tiledb/libtiledb.pxd b/tiledb/libtiledb.pxd index c15b4941cc..3b88ecf121 100644 --- a/tiledb/libtiledb.pxd +++ b/tiledb/libtiledb.pxd @@ -651,9 +651,8 @@ cdef extern from "tiledb/tiledb.h": uint32_t key_length, tiledb_config_t* config) nogil - int tiledb_array_delete_array( + int tiledb_array_delete( tiledb_ctx_t* ctx, - tiledb_array_t* array, const char* uri) nogil # Query @@ -879,6 +878,12 @@ cdef extern from "tiledb/tiledb.h": uint64_t timestamp_start, uint64_t timestamp_end) + int tiledb_array_delete_fragments_v2( + tiledb_ctx_t* ctx, + const char* uri, + uint64_t timestamp_start, + uint64_t timestamp_end) + int tiledb_array_get_schema( tiledb_ctx_t* ctx, tiledb_array_t* array, diff --git a/tiledb/libtiledb.pyx b/tiledb/libtiledb.pyx index 0781a03b5f..1250649ec7 100644 --- a/tiledb/libtiledb.pyx +++ b/tiledb/libtiledb.pyx @@ -1278,6 +1278,10 @@ cdef class Array(object): array([0., 0., 0., 0.]) """ + warnings.warn( + "The `tiledb.Array.delete_fragments` instance method is deprecated. Use the static method with the same name instead.", + DeprecationWarning, + ) cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(self.ctx) cdef tiledb_array_t* array_ptr = self.ptr cdef tiledb_query_t* query_ptr = NULL @@ -1295,6 +1299,55 @@ cdef class Array(object): if rc != TILEDB_OK: _raise_ctx_err(ctx_ptr, rc) + @staticmethod + def delete_fragments(uri, timestamp_start, timestamp_end, ctx=None): + """ + Delete a range of fragments from timestamp_start to timestamp_end. + The array needs to be opened in 'm' mode as shown in the example below. + + :param timestamp_start: the first fragment to delete in the range + :type timestamp_start: int + :param timestamp_end: the last fragment to delete in the range + :type timestamp_end: int + + **Example:** + + >>> import tiledb, tempfile, numpy as np + >>> path = tempfile.mkdtemp() + + >>> with tiledb.from_numpy(path, np.zeros(4), timestamp=1) as A: + ... pass + >>> with tiledb.open(path, 'w', timestamp=2) as A: + ... A[:] = np.ones(4, dtype=np.int64) + + >>> with tiledb.open(path, 'r') as A: + ... A[:] + array([1., 1., 1., 1.]) + + >>> tiledb.Array.delete_fragments(path, 2, 2) + + >>> with tiledb.open(path, 'r') as A: + ... A[:] + array([0., 0., 0., 0.]) + + """ + if not ctx: + ctx = default_ctx() + + cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(ctx) + cdef bytes buri = uri.encode('UTF-8') + + cdef int rc = TILEDB_OK + + rc = tiledb_array_delete_fragments_v2( + ctx_ptr, + buri, + timestamp_start, + timestamp_end + ) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + @staticmethod def delete_array(uri, ctx=None): """ @@ -1324,12 +1377,10 @@ cdef class Array(object): cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(ctx) cdef bytes buri = uri.encode('UTF-8') - cdef ArrayPtr preload_ptr = preload_array(uri, 'm', None, None) - cdef tiledb_array_t* array_ptr = preload_ptr.ptr cdef int rc = TILEDB_OK - rc = tiledb_array_delete_array(ctx_ptr, array_ptr, buri) + rc = tiledb_array_delete(ctx_ptr, buri) if rc != TILEDB_OK: _raise_ctx_err(ctx_ptr, rc) From d3fd5831c41305ff70f5058f15b1dc802bfe80e7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 23 Apr 2024 18:50:09 +0300 Subject: [PATCH 2/5] Update tests to use the new delete_fragments API. --- tiledb/tests/test_fragments.py | 26 ++++++++++++-------------- tiledb/tests/test_libtiledb.py | 6 ++---- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tiledb/tests/test_fragments.py b/tiledb/tests/test_fragments.py index 05439d7824..87dccb7524 100644 --- a/tiledb/tests/test_fragments.py +++ b/tiledb/tests/test_fragments.py @@ -704,13 +704,12 @@ def write_fragments(target_path, dshape, num_writes): if use_timestamps: assert frags.timestamp_range == ts - with tiledb.open(path, "m") as A: - if use_timestamps: - A.delete_fragments(3, 6) - else: - A.delete_fragments( - frags.timestamp_range[2][0], frags.timestamp_range[5][1] - ) + if use_timestamps: + tiledb.Array.delete_fragments(path, 3, 6) + else: + tiledb.Array.delete_fragments( + path, frags.timestamp_range[2][0], frags.timestamp_range[5][1] + ) frags = tiledb.array_fragments(path) assert len(frags) == 6 @@ -755,13 +754,12 @@ def test_delete_fragments_with_schema_evolution(self, use_timestamps): assert_array_equal(A[:]["a1"], ts2_data) assert_array_equal(A[:]["a2"], ts2_data) - with tiledb.open(path, "m") as A: - if use_timestamps: - A.delete_fragments(2, 2) - else: - A.delete_fragments( - frags.timestamp_range[1][0], frags.timestamp_range[1][1] - ) + if use_timestamps: + tiledb.Array.delete_fragments(path, 2, 2) + else: + tiledb.Array.delete_fragments( + path, frags.timestamp_range[1][0], frags.timestamp_range[1][1] + ) frags = tiledb.array_fragments(path) assert len(frags) == 1 diff --git a/tiledb/tests/test_libtiledb.py b/tiledb/tests/test_libtiledb.py index b9797aadaa..05fe9bf59a 100644 --- a/tiledb/tests/test_libtiledb.py +++ b/tiledb/tests/test_libtiledb.py @@ -367,12 +367,10 @@ def write_fragments(target_path, dshape, num_writes): assert frags.timestamp_range == ts if use_timestamps: - with tiledb.open(path, "m") as arr: - arr.delete_fragments(3, 6) + tiledb.Array.delete_fragments(path, 3, 6) else: timestamps = [t[0] for t in tiledb.array_fragments(path).timestamp_range] - with tiledb.open(path, "m") as arr: - arr.delete_fragments(timestamps[2], timestamps[5]) + tiledb.Array.delete_fragments(path, timestamps[2], timestamps[5]) frags = tiledb.array_fragments(path) assert len(frags) == 6 From 2f7ca979d365aaf28e409ba3766a27175d35891c Mon Sep 17 00:00:00 2001 From: Agis Kounelis Date: Fri, 26 Apr 2024 11:31:06 +0300 Subject: [PATCH 3/5] Fix instance/static method conflict --- tiledb/libtiledb.pyx | 61 ++++++-------------------------------------- 1 file changed, 8 insertions(+), 53 deletions(-) diff --git a/tiledb/libtiledb.pyx b/tiledb/libtiledb.pyx index 1250649ec7..b86f4a24b5 100644 --- a/tiledb/libtiledb.pyx +++ b/tiledb/libtiledb.pyx @@ -1246,59 +1246,6 @@ cdef class Array(object): _raise_ctx_err(ctx_ptr, rc) return Enumeration.from_capsule(self.ctx, PyCapsule_New(enum_ptr, "enum", NULL)) - def delete_fragments(self, timestamp_start, timestamp_end): - """ - Delete a range of fragments from timestamp_start to timestamp_end. - The array needs to be opened in 'm' mode as shown in the example below. - - :param timestamp_start: the first fragment to delete in the range - :type timestamp_start: int - :param timestamp_end: the last fragment to delete in the range - :type timestamp_end: int - - **Example:** - - >>> import tiledb, tempfile, numpy as np - >>> path = tempfile.mkdtemp() - - >>> with tiledb.from_numpy(path, np.zeros(4), timestamp=1) as A: - ... pass - >>> with tiledb.open(path, 'w', timestamp=2) as A: - ... A[:] = np.ones(4, dtype=np.int64) - - >>> with tiledb.open(path, 'r') as A: - ... A[:] - array([1., 1., 1., 1.]) - - >>> with tiledb.open(path, 'm') as A: - ... A.delete_fragments(2, 2) - - >>> with tiledb.open(path, 'r') as A: - ... A[:] - array([0., 0., 0., 0.]) - - """ - warnings.warn( - "The `tiledb.Array.delete_fragments` instance method is deprecated. Use the static method with the same name instead.", - DeprecationWarning, - ) - cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(self.ctx) - cdef tiledb_array_t* array_ptr = self.ptr - cdef tiledb_query_t* query_ptr = NULL - cdef bytes buri = self.uri.encode('UTF-8') - - cdef int rc = TILEDB_OK - - rc = tiledb_array_delete_fragments( - ctx_ptr, - array_ptr, - buri, - timestamp_start, - timestamp_end - ) - if rc != TILEDB_OK: - _raise_ctx_err(ctx_ptr, rc) - @staticmethod def delete_fragments(uri, timestamp_start, timestamp_end, ctx=None): """ @@ -1331,6 +1278,14 @@ cdef class Array(object): array([0., 0., 0., 0.]) """ + # If uri is an instance of Array (user calls the old instance method), issue a warning + if isinstance(uri, Array): + warnings.warn( + "The `tiledb.Array.delete_fragments` instance method is deprecated. Use the static method with the same name instead.", + DeprecationWarning, + ) + uri = uri.uri + if not ctx: ctx = default_ctx() From 178318e304823588155fb32c4cff74f40bb981f8 Mon Sep 17 00:00:00 2001 From: Agis Kounelis Date: Fri, 26 Apr 2024 15:21:21 +0300 Subject: [PATCH 4/5] Migrate use of tiledb_array_create_with_key deprecated API. --- tiledb/libtiledb.pxd | 8 -------- tiledb/libtiledb.pyx | 23 ++++++++++++++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tiledb/libtiledb.pxd b/tiledb/libtiledb.pxd index 3b88ecf121..c251517382 100644 --- a/tiledb/libtiledb.pxd +++ b/tiledb/libtiledb.pxd @@ -845,14 +845,6 @@ cdef extern from "tiledb/tiledb.h": const char* uri, const tiledb_array_schema_t* array_schema) nogil - int tiledb_array_create_with_key( - tiledb_ctx_t* ctx, - const char* uri, - const tiledb_array_schema_t* array_schema, - tiledb_encryption_type_t key_type, - const void* key, - unsigned int key_len) nogil - int tiledb_array_is_open( tiledb_ctx_t* ctx, tiledb_array_t* array, diff --git a/tiledb/libtiledb.pyx b/tiledb/libtiledb.pyx index b86f4a24b5..196d3c6f1a 100644 --- a/tiledb/libtiledb.pyx +++ b/tiledb/libtiledb.pyx @@ -940,9 +940,11 @@ cdef class Array(object): cdef bytes bkey cdef tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION - cdef void* key_ptr = NULL + cdef const char* key_ptr = NULL cdef unsigned int key_len = 0 + cdef tiledb_config_t* config_ptr = NULL + cdef tiledb_error_t* err_ptr = NULL cdef int rc = TILEDB_OK if key is not None: @@ -951,10 +953,25 @@ cdef class Array(object): else: bkey = bytes(key) key_type = TILEDB_AES_256_GCM - key_ptr = PyBytes_AS_STRING(bkey) + key_ptr = PyBytes_AS_STRING(bkey) #TODO: unsafe cast here ssize_t -> uint64_t key_len = PyBytes_GET_SIZE(bkey) + rc = tiledb_config_alloc(&config_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + rc = tiledb_ctx_alloc(config_ptr, &ctx_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + if overwrite: if object_type(uri) == "array": if uri.startswith("file://") or "://" not in uri: @@ -971,7 +988,7 @@ cdef class Array(object): "object to argument ctx") ctx_ptr = safe_ctx_ptr(ctx) with nogil: - rc = tiledb_array_create_with_key(ctx_ptr, uri_ptr, schema_ptr, key_type, key_ptr, key_len) + rc = tiledb_array_create(ctx_ptr, uri_ptr, schema_ptr) if rc != TILEDB_OK: _raise_ctx_err(ctx_ptr, rc) return From 04879b5161be018d315f78f965e78b3601ea205a Mon Sep 17 00:00:00 2001 From: Agis Kounelis Date: Fri, 26 Apr 2024 15:31:20 +0300 Subject: [PATCH 5/5] Migrate use of tiledb_array_consolidate_with_key deprecated API. --- tiledb/libmetadata.pyx | 22 ++++++++++++++++------ tiledb/libtiledb.pxd | 8 -------- tiledb/libtiledb.pyx | 21 +++++++++++++++++---- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tiledb/libmetadata.pyx b/tiledb/libmetadata.pyx index 9b85e00b60..9985786122 100644 --- a/tiledb/libmetadata.pyx +++ b/tiledb/libmetadata.pyx @@ -400,11 +400,12 @@ cdef class Metadata: ctx.__capsule__(), "ctx") tiledb_config_t* config_ptr = NULL tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION - void* key_ptr = NULL + const char* key_ptr = NULL uint32_t key_len = 0 bytes bkey bytes buri = unicode_path(self.array.uri) str key = (self.array).key + tiledb_error_t* err_ptr = NULL if config: config_ptr = PyCapsule_GetPointer( @@ -416,19 +417,28 @@ cdef class Metadata: else: bkey = bytes(self.array.key) key_type = TILEDB_AES_256_GCM - key_ptr = PyBytes_AS_STRING(bkey) + key_ptr = PyBytes_AS_STRING(bkey) #TODO: unsafe cast here ssize_t -> uint64_t key_len = PyBytes_GET_SIZE(bkey) + rc = tiledb_config_alloc(&config_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + cdef const char* buri_ptr = buri with nogil: - rc = tiledb_array_consolidate_with_key( + rc = tiledb_array_consolidate( ctx_ptr, buri_ptr, - key_type, - key_ptr, - key_len, config_ptr) if rc != TILEDB_OK: _raise_ctx_err(ctx_ptr, rc) diff --git a/tiledb/libtiledb.pxd b/tiledb/libtiledb.pxd index c251517382..d05f4ae7a5 100644 --- a/tiledb/libtiledb.pxd +++ b/tiledb/libtiledb.pxd @@ -855,14 +855,6 @@ cdef extern from "tiledb/tiledb.h": const char* array_path, tiledb_config_t* config) nogil - int tiledb_array_consolidate_with_key( - tiledb_ctx_t* ctx, - const char* uri, - tiledb_encryption_type_t key_type, - const void* key_ptr, - unsigned int key_len, - tiledb_config_t* config) nogil - int tiledb_array_delete_fragments( tiledb_ctx_t* ctx, tiledb_array_t* array, diff --git a/tiledb/libtiledb.pyx b/tiledb/libtiledb.pyx index 196d3c6f1a..d996bf8304 100644 --- a/tiledb/libtiledb.pyx +++ b/tiledb/libtiledb.pyx @@ -3647,8 +3647,9 @@ def _consolidate_timestamp(uri, key=None, config=None, ctx=None, timestamp=None) cdef: bytes bkey tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION - void* key_ptr = NULL + const char* key_ptr = NULL unsigned int key_len = 0 + tiledb_error_t* err_ptr = NULL if key is not None: if isinstance(key, str): @@ -3656,13 +3657,25 @@ def _consolidate_timestamp(uri, key=None, config=None, ctx=None, timestamp=None) else: bkey = bytes(key) key_type = TILEDB_AES_256_GCM - key_ptr = PyBytes_AS_STRING(bkey) + key_ptr = PyBytes_AS_STRING(bkey) #TODO: unsafe cast here ssize_t -> uint64_t key_len = PyBytes_GET_SIZE(bkey) + + rc = tiledb_config_alloc(&config_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) + + rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr) + if rc != TILEDB_OK: + _raise_ctx_err(ctx_ptr, rc) with nogil: - rc = tiledb_array_consolidate_with_key( - ctx_ptr, array_uri_ptr, key_type, key_ptr, key_len, config_ptr) + rc = tiledb_array_consolidate( + ctx_ptr, array_uri_ptr, config_ptr) if rc != TILEDB_OK: _raise_ctx_err(ctx_ptr, rc) return uri