Skip to content

Commit

Permalink
Eliminate storage_manager argument from post_array_from_rest. (#4964)
Browse files Browse the repository at this point in the history 
As a larger part of eliminating class `StorageManager`, all usages of
member variable `storage_manager_` must also be eliminated from class
`Array`. To that effect, this PR removes `storage_manager` as an
argument in `post_array_to_rest`, subsequently also removing it from
`array_from_capnp` and `array_deserialize`.

---

[sc-47009]
---
TYPE: NO_HISTORY
DESC: Eliminate the storage_manager argument from post_array_from_rest.
  • Loading branch information
@bekadavis9
bekadavis9 authored May 11, 2024
1 parent 7387605 commit 748b80b
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 173 deletions.
5 changes: 2 additions & 3 deletions test/src/unit-capi-array.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2605,13 +2605,12 @@ TEST_CASE_METHOD(
1,
&buff);
REQUIRE(rc == TILEDB_OK);
auto st = tiledb::sm::serialization::array_deserialize(
tiledb::sm::serialization::array_deserialize(
array->array_.get(),
tiledb::sm::SerializationType::CAPNP,
buff->buffer(),
ctx_->storage_manager(),
ctx_->context().resources(),
memory_tracker_);
REQUIRE(st.ok());

// 6. Server: Close array and clean up
rc = tiledb_array_close(ctx_, deserialized_array_server);
Expand Down
6 changes: 3 additions & 3 deletions test/src/unit-enumerations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2023 TileDB Inc.
* @copyright Copyright (c) 2023-2024 TileDB Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -2950,8 +2950,8 @@ void EnumerationFx::ser_des_array(
SerializationType stype) {
Buffer buf;
throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side));
throw_if_not_ok(serialization::array_deserialize(
out, stype, buf, ctx.storage_manager(), memory_tracker_));
serialization::array_deserialize(
out, stype, buf, ctx.resources(), memory_tracker_);
}

#else // No TILEDB_SERIALIZATION
Expand Down
12 changes: 2 additions & 10 deletions tiledb/sm/array/array.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,11 +173,7 @@ Status Array::open_without_fragments(
}
set_array_schema_latest(array_schema_latest.value());
} else {
auto st = rest_client->post_array_from_rest(
array_uri_, storage_manager_, this);
if (!st.ok()) {
throw StatusException(st);
}
rest_client->post_array_from_rest(array_uri_, resources_, this);
}
} else {
{
Expand Down Expand Up @@ -338,11 +334,7 @@ Status Array::open(
throw_if_not_ok(st);
set_array_schema_latest(array_schema_latest.value());
} else {
auto st = rest_client->post_array_from_rest(
array_uri_, storage_manager_, this);
if (!st.ok()) {
throw StatusException(st);
}
rest_client->post_array_from_rest(array_uri_, resources_, this);
}
} else if (query_type == QueryType::READ) {
{
Expand Down
16 changes: 8 additions & 8 deletions tiledb/sm/c_api/tiledb.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3431,14 +3431,14 @@ int32_t tiledb_deserialize_array(

auto memory_tracker = ctx->context().resources().create_memory_tracker();
memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD);
if (SAVE_ERROR_CATCH(
ctx,
tiledb::sm::serialization::array_deserialize(
(*array)->array_.get(),
(tiledb::sm::SerializationType)serialize_type,
buffer->buffer(),
ctx->storage_manager(),
memory_tracker))) {
try {
tiledb::sm::serialization::array_deserialize(
(*array)->array_.get(),
(tiledb::sm::SerializationType)serialize_type,
buffer->buffer(),
ctx->context().resources(),
memory_tracker);
} catch (StatusException& e) {
delete *array;
*array = nullptr;
return TILEDB_ERR;
Expand Down
144 changes: 63 additions & 81 deletions tiledb/sm/rest/rest_client.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2018-2023 TileDB, Inc.
* @copyright Copyright (c) 2018-2024 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -74,8 +74,22 @@

using namespace tiledb::common;

namespace tiledb {
namespace sm {
namespace tiledb::sm {

class RestClientException : public StatusException {
public:
explicit RestClientException(const std::string& message)
: StatusException("RestClient", message) {
}
};

class RestClientDisabledException : public RestClientException {
public:
explicit RestClientDisabledException()
: RestClientException(
"Cannot use rest client; serialization not enabled.") {
}
};

#ifdef TILEDB_SERIALIZATION

Expand Down Expand Up @@ -347,26 +361,25 @@ Status RestClient::post_array_schema_to_rest(
return sc;
}

Status RestClient::post_array_from_rest(
const URI& uri, StorageManager* storage_manager, Array* array) {
void RestClient::post_array_from_rest(
const URI& uri, ContextResources& resources, Array* array) {
if (array == nullptr) {
return LOG_STATUS(Status_SerializationError(
"Error getting remote array; array is null."));
throw RestClientException("Error getting remote array; array is null.");
}

Buffer buff;
RETURN_NOT_OK(
throw_if_not_ok(
serialization::array_open_serialize(*array, serialization_type_, &buff));
// Wrap in a list
BufferList serialized;
RETURN_NOT_OK(serialized.add_buffer(std::move(buff)));
throw_if_not_ok(serialized.add_buffer(std::move(buff)));

// Init curl and form the URL
Curl curlc(logger_);
std::string array_ns, array_uri;
RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri));
throw_if_not_ok(uri.get_rest_components(&array_ns, &array_uri));
const std::string cache_key = array_ns + ":" + array_uri;
RETURN_NOT_OK(
throw_if_not_ok(
curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_));
std::string url = redirect_uri(cache_key) + "/v2/arrays/" + array_ns + "/" +
curlc.url_escape(array_uri) + "/?";
Expand All @@ -377,25 +390,22 @@ Status RestClient::post_array_from_rest(

// Get the data
Buffer returned_data;
RETURN_NOT_OK(curlc.post_data(
throw_if_not_ok(curlc.post_data(
stats_,
url,
serialization_type_,
&serialized,
&returned_data,
cache_key));
if (returned_data.data() == nullptr || returned_data.size() == 0)
return LOG_STATUS(Status_RestError(
"Error getting array from REST; server returned no data."));
if (returned_data.data() == nullptr || returned_data.size() == 0) {
throw RestClientException(
"Error getting array from REST; server returned no data.");
}

// Ensure data has a null delimiter for cap'n proto if using JSON
RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data));
return serialization::array_deserialize(
array,
serialization_type_,
returned_data,
storage_manager,
memory_tracker_);
throw_if_not_ok(ensure_json_null_delimited_string(&returned_data));
serialization::array_deserialize(
array, serialization_type_, returned_data, resources, memory_tracker_);
}

void RestClient::delete_array_from_rest(const URI& uri) {
Expand Down Expand Up @@ -1654,13 +1664,11 @@ Status RestClient::init(
ThreadPool*,
const std::shared_ptr<Logger>&,
ContextResources&) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::set_header(const std::string&, const std::string&) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

tuple<Status, optional<shared_ptr<ArraySchema>>>
Expand All @@ -1672,61 +1680,51 @@ RestClient::get_array_schema_from_rest(const URI&) {
}

Status RestClient::post_array_schema_to_rest(const URI&, const ArraySchema&) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_array_from_rest(const URI&, StorageManager*, Array*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
void RestClient::post_array_from_rest(const URI&, ContextResources&, Array*) {
throw RestClientDisabledException();
}

void RestClient::delete_array_from_rest(const URI&) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

void RestClient::post_delete_fragments_to_rest(
const URI&, Array*, uint64_t, uint64_t) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

void RestClient::post_delete_fragments_list_to_rest(
const URI&, Array*, const std::vector<URI>&) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::deregister_array_from_rest(const URI&) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::get_array_non_empty_domain(Array*, uint64_t, uint64_t) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::get_array_max_buffer_sizes(
const URI&,
const ArraySchema&,
const void*,
std::unordered_map<std::string, std::pair<uint64_t, uint64_t>>*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::get_array_metadata_from_rest(
const URI&, uint64_t, uint64_t, Array*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_array_metadata_to_rest(
const URI&, uint64_t, uint64_t, Array*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

std::vector<shared_ptr<const Enumeration>>
Expand All @@ -1737,37 +1735,32 @@ RestClient::post_enumerations_from_rest(
Array*,
const std::vector<std::string>&,
shared_ptr<MemoryTracker>) {
throw Status_RestError("Cannot use rest client; serialization not enabled.");
throw RestClientDisabledException();
}

void RestClient::post_query_plan_from_rest(const URI&, Query&, QueryPlan&) {
throw Status_RestError("Cannot use rest client; serialization not enabled.");
throw RestClientDisabledException();
}

Status RestClient::submit_query_to_rest(const URI&, Query*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::finalize_query_to_rest(const URI&, Query*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::submit_and_finalize_query_to_rest(const URI&, Query*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::get_query_est_result_sizes(const URI&, Query*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_array_schema_evolution_to_rest(
const URI&, ArraySchemaEvolution*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

tuple<Status, std::optional<bool>> RestClient::check_array_exists_from_rest(
Expand All @@ -1787,58 +1780,47 @@ tuple<Status, std::optional<bool>> RestClient::check_group_exists_from_rest(
}

Status RestClient::post_group_metadata_from_rest(const URI&, Group*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_fragment_info_from_rest(const URI&, FragmentInfo*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::put_group_metadata_to_rest(const URI&, Group*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_group_create_to_rest(const URI&, Group*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_group_from_rest(const URI&, Group*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::patch_group_to_rest(const URI&, Group*) {
return LOG_STATUS(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

void RestClient::delete_group_from_rest(const URI&, bool) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_consolidation_to_rest(const URI&, const Config&) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

Status RestClient::post_vacuum_to_rest(const URI&, const Config&) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

std::vector<std::vector<std::string>>
RestClient::post_consolidation_plan_from_rest(
const URI&, const Config&, uint64_t) {
throw StatusException(
Status_RestError("Cannot use rest client; serialization not enabled."));
throw RestClientDisabledException();
}

#endif // TILEDB_SERIALIZATION

} // namespace sm
} // namespace tiledb
} // namespace tiledb::sm
Loading

0 comments on commit 748b80b

Please sign in to comment.