diff --git a/src/http/http_call_registry.h b/src/http/http_call_registry.h index fef3b0c467..2e944463ce 100644 --- a/src/http/http_call_registry.h +++ b/src/http/http_call_registry.h @@ -35,11 +35,11 @@ class http_call_registry : public utils::singleton std::shared_ptr find(const std::string &path) const { std::lock_guard guard(_mu); - auto it = _call_map.find(path); - if (it == _call_map.end()) { + const auto &iter = _call_map.find(path); + if (iter == _call_map.end()) { return nullptr; } - return it->second; + return iter->second; } void remove(const std::string &path) @@ -48,14 +48,19 @@ class http_call_registry : public utils::singleton _call_map.erase(path); } - void add(std::unique_ptr call_uptr) + void add(const std::shared_ptr &call) { - auto call = std::shared_ptr(call_uptr.release()); std::lock_guard guard(_mu); - CHECK_EQ_MSG(_call_map.count(call->path), 0, call->path); + CHECK_EQ_MSG(_call_map.count(call->path), 0, "{} has been added", call->path); _call_map[call->path] = call; } + void add(std::unique_ptr call_uptr) + { + auto call = std::shared_ptr(call_uptr.release()); + add(call); + } + std::vector> list_all_calls() const { std::lock_guard guard(_mu); diff --git a/src/http/http_client.cpp b/src/http/http_client.cpp new file mode 100644 index 0000000000..32ae5eaea1 --- /dev/null +++ b/src/http/http_client.cpp @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/http_client.h" + +#include +#include +#include + +#include "curl/curl.h" +#include "utils/error_code.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +DSN_DEFINE_uint32(http, + curl_timeout_ms, + 10000, + "The maximum time in milliseconds that you allow the libcurl transfer operation " + "to complete"); + +http_client::http_client() + : _curl(nullptr), + _method(http_method::GET), + _recv_callback(nullptr), + _header_changed(true), + _header_list(nullptr) +{ + // Since `kErrorBufferBytes` is private, `static_assert` have to be put in constructor. + static_assert(http_client::kErrorBufferBytes >= CURL_ERROR_SIZE, + "The error buffer used by libcurl must be at least CURL_ERROR_SIZE bytes big"); + + clear_error_buf(); +} + +http_client::~http_client() +{ + if (_curl != nullptr) { + curl_easy_cleanup(_curl); + _curl = nullptr; + } + + free_header_list(); +} + +namespace { + +inline dsn::error_code to_error_code(CURLcode code) +{ + switch (code) { + case CURLE_OK: + return dsn::ERR_OK; + case CURLE_OPERATION_TIMEDOUT: + return dsn::ERR_TIMEOUT; + default: + return dsn::ERR_CURL_FAILED; + } +} + +} // anonymous namespace + +#define RETURN_IF_CURL_NOT_OK(expr, ...) \ + do { \ + const auto code = (expr); \ + if (dsn_unlikely(code != CURLE_OK)) { \ + std::string msg(fmt::format("{}: {}", fmt::format(__VA_ARGS__), to_error_msg(code))); \ + return dsn::error_s::make(to_error_code(code), msg); \ + } \ + } while (0) + +#define RETURN_IF_SETOPT_NOT_OK(opt, input) \ + RETURN_IF_CURL_NOT_OK(curl_easy_setopt(_curl, opt, input), \ + "failed to set " #opt " to" \ + " " #input) + +#define RETURN_IF_GETINFO_NOT_OK(info, output) \ + RETURN_IF_CURL_NOT_OK(curl_easy_getinfo(_curl, info, output), "failed to get from " #info) + +#define RETURN_IF_EXEC_METHOD_NOT_OK() \ + RETURN_IF_CURL_NOT_OK(curl_easy_perform(_curl), \ + "failed to perform http request(method={}, url={})", \ + enum_to_string(_method), \ + _url) + +dsn::error_s http_client::init() +{ + if (_curl == nullptr) { + _curl = curl_easy_init(); + if (_curl == nullptr) { + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "fail to initialize curl"); + } + } else { + curl_easy_reset(_curl); + } + + clear_header_fields(); + free_header_list(); + + // Additional messages for errors are needed. + clear_error_buf(); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_ERRORBUFFER, _error_buf); + + // Set with NOSIGNAL since we are multi-threaded. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_NOSIGNAL, 1L); + + // Redirects are supported. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_FOLLOWLOCATION, 1L); + + // Before 8.3.0, CURLOPT_MAXREDIRS was unlimited. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_MAXREDIRS, 20); + + // Set common timeout for transfer operation. Users could also change it with their + // custom values by `set_timeout`. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, static_cast(FLAGS_curl_timeout_ms)); + + // A lambda can only be converted to a function pointer if it does not capture: + // https://stackoverflow.com/questions/28746744/passing-capturing-lambda-as-function-pointer + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf + curl_write_callback callback = [](char *buffer, size_t size, size_t nmemb, void *param) { + http_client *client = reinterpret_cast(param); + return client->on_response_data(buffer, size * nmemb); + }; + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEFUNCTION, callback); + + // This http_client object itself is passed to the callback function. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEDATA, reinterpret_cast(this)); + + return dsn::error_s::ok(); +} + +void http_client::clear_error_buf() { _error_buf[0] = 0; } + +bool http_client::is_error_buf_empty() const { return _error_buf[0] == 0; } + +std::string http_client::to_error_msg(CURLcode code) const +{ + std::string err_msg = + fmt::format("code={}, desc=\"{}\"", static_cast(code), curl_easy_strerror(code)); + if (is_error_buf_empty()) { + return err_msg; + } + + err_msg += fmt::format(", msg=\"{}\"", _error_buf); + return err_msg; +} + +// `data` passed to this function is NOT null-terminated. +// `length` might be zero. +size_t http_client::on_response_data(const void *data, size_t length) +{ + if (_recv_callback == nullptr) { + return length; + } + + if (!(*_recv_callback)) { + // callback function is empty. + return length; + } + + // According to libcurl, callback should return the number of bytes actually taken care of. + // If that amount differs from the amount passed to callback function, it would signals an + // error condition. This causes the transfer to get aborted and the libcurl function used + // returns CURLE_WRITE_ERROR. Therefore, here we just return the max limit of size_t for + // failure. + // + // See https://curl.se/libcurl/c/CURLOPT_WRITEFUNCTION.html for details. + return (*_recv_callback)(data, length) ? length : std::numeric_limits::max(); +} + +dsn::error_s http_client::set_url(const std::string &url) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_URL, url.c_str()); + + _url = url; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_post_method(const std::string &data) +{ + // No need to enable CURLOPT_POST by `RETURN_IF_SETOPT_NOT_OK(CURLOPT_POST, 1L)`, since using + // either of CURLOPT_POSTFIELDS or CURLOPT_COPYPOSTFIELDS implies setting CURLOPT_POST to 1. + // See https://curl.se/libcurl/c/CURLOPT_POSTFIELDS.html for details. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_POSTFIELDSIZE, static_cast(data.size())); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_COPYPOSTFIELDS, data.data()); + _method = http_method::POST; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_get_method() { return set_method(http_method::GET); } + +dsn::error_s http_client::set_method(http_method method) +{ + // No need to process the case of http_method::POST, since it should be enabled by + // `with_post_method`. + switch (method) { + case http_method::GET: + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPGET, 1L); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + _method = method; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::set_timeout(long timeout_ms) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, timeout_ms); + return dsn::error_s::ok(); +} + +void http_client::clear_header_fields() +{ + _header_fields.clear(); + + _header_changed = true; +} + +void http_client::free_header_list() +{ + if (_header_list == nullptr) { + return; + } + + curl_slist_free_all(_header_list); + _header_list = nullptr; +} + +void http_client::set_header_field(dsn::string_view key, dsn::string_view val) +{ + _header_fields[std::string(key)] = std::string(val); + _header_changed = true; +} + +void http_client::set_accept(dsn::string_view val) { set_header_field("Accept", val); } + +void http_client::set_content_type(dsn::string_view val) { set_header_field("Content-Type", val); } + +dsn::error_s http_client::process_header() +{ + if (!_header_changed) { + return dsn::error_s::ok(); + } + + free_header_list(); + + for (const auto &field : _header_fields) { + auto str = fmt::format("{}: {}", field.first, field.second); + + // A null pointer is returned if anything went wrong, otherwise the new list pointer is + // returned. To avoid overwriting an existing non-empty list on failure, the new list + // should be returned to a temporary variable which can be tested for NULL before updating + // the original list pointer. (https://curl.se/libcurl/c/curl_slist_append.html) + struct curl_slist *temp = curl_slist_append(_header_list, str.c_str()); + if (temp == nullptr) { + free_header_list(); + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "curl_slist_append failed"); + } + _header_list = temp; + } + + // This would work well even if `_header_list` is NULL pointer. Pass a NULL to this option + // to reset back to no custom headers. (https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPHEADER, _header_list); + + // New header has been built successfully, thus mark it unchanged. + _header_changed = false; + + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(const http_client::recv_callback &callback) +{ + // `curl_easy_perform` would run synchronously, thus it is safe to use the pointer to + // `callback`. + _recv_callback = &callback; + + RETURN_NOT_OK(process_header()); + + RETURN_IF_EXEC_METHOD_NOT_OK(); + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(std::string *response) +{ + if (response == nullptr) { + return exec_method(); + } + + auto callback = [response](const void *data, size_t length) { + response->append(reinterpret_cast(data), length); + return true; + }; + + return exec_method(callback); +} + +dsn::error_s http_client::get_http_status(long &http_status) const +{ + RETURN_IF_GETINFO_NOT_OK(CURLINFO_RESPONSE_CODE, &http_status); + return dsn::error_s::ok(); +} + +#undef RETURN_IF_EXEC_METHOD_NOT_OK +#undef RETURN_IF_SETOPT_NOT_OK +#undef RETURN_IF_CURL_NOT_OK + +} // namespace dsn diff --git a/src/http/http_client.h b/src/http/http_client.h new file mode 100644 index 0000000000..190af11f2b --- /dev/null +++ b/src/http/http_client.h @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "utils/errors.h" +#include "utils/ports.h" +#include "utils/string_view.h" + +namespace dsn { + +// A library for http client that provides convenient APIs to access http services, implemented +// based on libcurl (https://curl.se/libcurl/c/). +// +// This class is not thread-safe. Thus maintain one instance for each thread. +// +// Example of submitting GET request to remote http service +// -------------------------------------------------------- +// Create an instance of http_client: +// http_client client; +// +// It's necessary to initialize the new instance before coming into use: +// auto err = client.init(); +// +// Specify the target url that you would request for: +// err = client.set_url(method); +// +// If you would use GET method, call `with_get_method`: +// err = client.with_get_method(); +// +// If you would use POST method, call `with_post_method` with post data: +// err = client.with_post_method(post_data); +// +// Submit the request to remote http service: +// err = client.exec_method(); +// +// If response data should be processed, use callback function: +// auto callback = [...](const void *data, size_t length) { +// ...... +// return true; +// }; +// err = client.exec_method(callback); +// +// Or just provide a string pointer: +// std::string response; +// err = client.exec_method(&response); +// +// Get the http status code after requesting: +// long http_status; +// err = client.get_http_status(http_status); +class http_client +{ +public: + using recv_callback = std::function; + + http_client(); + ~http_client(); + + // Before coming into use, init() must be called to initialize http client. It could also be + // called to reset the http clients that have been initialized previously. + dsn::error_s init(); + + // Specify the target url that the request would be sent for. + dsn::error_s set_url(const std::string &url); + + // Using post method, with `data` as the payload for post body. + dsn::error_s with_post_method(const std::string &data); + + // Using get method. + dsn::error_s with_get_method(); + + // Specify the maximum time in milliseconds that a request is allowed to complete. + dsn::error_s set_timeout(long timeout_ms); + + // Operations for the header fields. + void clear_header_fields(); + void set_accept(dsn::string_view val); + void set_content_type(dsn::string_view val); + + // Submit request to remote http service, with response processed by callback function. + // + // `callback` function gets called by libcurl as soon as there is data received that needs + // to be saved. For most transfers, this callback gets called many times and each invoke + // delivers another chunk of data. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(const recv_callback &callback = {}); + + // Submit request to remote http service, with response data returned in a string. + // + // This function would run synchronously, which means it would wait until the response was + // returned and processed appropriately. + dsn::error_s exec_method(std::string *response); + + // Get the last http status code after requesting. + dsn::error_s get_http_status(long &http_status) const; + +private: + using header_field_map = std::unordered_map; + + void clear_error_buf(); + bool is_error_buf_empty() const; + std::string to_error_msg(CURLcode code) const; + + size_t on_response_data(const void *data, size_t length); + + // Specify which http method would be used, such as GET. Enabling POST should not use this + // function (use `with_post_method` instead). + dsn::error_s set_method(http_method method); + + void free_header_list(); + void set_header_field(dsn::string_view key, dsn::string_view val); + dsn::error_s process_header(); + + // The size of a buffer that is used by libcurl to store human readable + // error messages on failures or problems. + static const constexpr size_t kErrorBufferBytes = CURL_ERROR_SIZE; + + CURL *_curl; + http_method _method; + std::string _url; + const recv_callback *_recv_callback; + char _error_buf[kErrorBufferBytes]; + + bool _header_changed; + header_field_map _header_fields; + struct curl_slist *_header_list; + + DISALLOW_COPY_AND_ASSIGN(http_client); +}; + +} // namespace dsn diff --git a/src/http/http_message_parser.cpp b/src/http/http_message_parser.cpp index ce2e0054ce..df873eea2e 100644 --- a/src/http/http_message_parser.cpp +++ b/src/http/http_message_parser.cpp @@ -26,12 +26,13 @@ #include "http_message_parser.h" +#include // IWYU pragma: no_include #include #include #include -#include "http_server.h" +#include "http/http_method.h" #include "nodejs/http_parser.h" #include "runtime/rpc/rpc_message.h" #include "utils/blob.h" @@ -132,10 +133,10 @@ http_message_parser::http_message_parser() message_header *header = msg->header; if (parser->type == HTTP_REQUEST && parser->method == HTTP_GET) { - header->hdr_type = http_method::HTTP_METHOD_GET; + header->hdr_type = static_cast(http_method::GET); header->context.u.is_request = 1; } else if (parser->type == HTTP_REQUEST && parser->method == HTTP_POST) { - header->hdr_type = http_method::HTTP_METHOD_POST; + header->hdr_type = static_cast(http_method::POST); header->context.u.is_request = 1; } else { // Bit fields don't work with "perfect" forwarding, see diff --git a/src/http/http_method.h b/src/http/http_method.h new file mode 100644 index 0000000000..f337d24e78 --- /dev/null +++ b/src/http/http_method.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "utils/enum_helper.h" + +namespace dsn { + +enum class http_method +{ + GET = 1, + POST = 2, + INVALID = 100, +}; + +ENUM_BEGIN(http_method, http_method::INVALID) +ENUM_REG2(http_method, GET) +ENUM_REG2(http_method, POST) +ENUM_END(http_method) + +} // namespace dsn diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index cbc179c815..4c2f704016 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -25,6 +25,7 @@ #include "builtin_http_calls.h" #include "fmt/core.h" +#include "http/http_method.h" #include "http_call_registry.h" #include "http_message_parser.h" #include "http_server_impl.h" @@ -146,8 +147,8 @@ void http_server::serve(message_ex *msg) resp.body = fmt::format("failed to parse request: {}", res.get_error()); } else { const http_request &req = res.get_value(); - std::shared_ptr call = http_call_registry::instance().find(req.path); - if (call != nullptr) { + auto call = http_call_registry::instance().find(req.path); + if (call) { call->callback(req, resp); } else { resp.status_code = http_status_code::not_found; diff --git a/src/http/http_server.h b/src/http/http_server.h index b182947e41..594ec70f80 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -25,6 +25,7 @@ #include #include +#include "http_method.h" #include "runtime/task/task_code.h" #include "utils/blob.h" #include "utils/errors.h" @@ -38,12 +39,6 @@ DSN_DECLARE_bool(enable_http_server); /// The rpc code for all the HTTP RPCs. DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT); -enum http_method -{ - HTTP_METHOD_GET = 1, - HTTP_METHOD_POST = 2, -}; - class message_ex; struct http_request diff --git a/src/http/pprof_http_service.cpp b/src/http/pprof_http_service.cpp index f57c57b193..f5aeb8b505 100644 --- a/src/http/pprof_http_service.cpp +++ b/src/http/pprof_http_service.cpp @@ -38,6 +38,7 @@ #include #include +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/api_layer1.h" #include "utils/blob.h" @@ -308,7 +309,7 @@ void pprof_http_service::symbol_handler(const http_request &req, http_response & // Load /proc/self/maps pthread_once(&s_load_symbolmap_once, load_symbols); - if (req.method != http_method::HTTP_METHOD_POST) { + if (req.method != http_method::POST) { char buf[64]; snprintf(buf, sizeof(buf), "num_symbols: %lu\n", symbol_map.size()); resp.body = buf; diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index d4da7e5b3a..85f2c3798a 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -24,14 +24,16 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_http dsn_runtime + curl gtest - gtest_main - rocksdb) + rocksdb + ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/run.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/config-test.ini" ) dsn_add_test() diff --git a/src/http/test/config-test.ini b/src/http/test/config-test.ini new file mode 100644 index 0000000000..744d8d412a --- /dev/null +++ b/src/http/test/config-test.ini @@ -0,0 +1,75 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[apps..default] +run = true +count = 1 +network.client.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.client.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 +network.server.0.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider, 65536 +network.server.0.RPC_CHANNEL_UDP = dsn::tools::asio_udp_provider, 65536 + +[apps.test] +type = test +arguments = +run = true +ports = 20001 +count = 1 +pools = THREAD_POOL_DEFAULT + +[core] +tool = nativerun + +toollets = tracer, profiler +pause_on_start = false + +logging_start_level = LOG_LEVEL_DEBUG +logging_factory_name = dsn::tools::simple_logger + +[tools.simple_logger] +fast_flush = true +short_header = false +stderr_start_level = LOG_LEVEL_DEBUG + +[network] +; how many network threads for network library (used by asio) +io_service_worker_count = 2 + +[task..default] +is_trace = true +is_profile = true +allow_inline = false +rpc_call_channel = RPC_CHANNEL_TCP +rpc_message_header_format = dsn +rpc_timeout_milliseconds = 1000 + +[task.LPC_RPC_TIMEOUT] +is_trace = false +is_profile = false + +[task.RPC_TEST_UDP] +rpc_call_channel = RPC_CHANNEL_UDP +rpc_message_crc_required = true + +; specification for each thread pool +[threadpool..default] +worker_count = 2 + +[threadpool.THREAD_POOL_DEFAULT] +partitioned = false +worker_priority = THREAD_xPRIORITY_NORMAL + diff --git a/src/http/test/http_client_test.cpp b/src/http/test/http_client_test.cpp new file mode 100644 index 0000000000..d15cb7bdd4 --- /dev/null +++ b/src/http/test/http_client_test.cpp @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include +#include +#include +#include + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get")); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + std::cout << "failed to connect: "; + + // "code=7" means CURLE_COULDNT_CONNECT, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed to connect to 127.0.0.1 port 20000: Connection refused" + // * msg="Failed to connect to 127.0.0.1 port 20000 after 0 ms: Connection refused" + // Thus we don't check if `msg` fields are consistent. + const std::string expected_description_prefix( + "ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20000/test/get): code=7, " + "desc=\"Couldn't connect to server\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +TEST(HttpClientTest, Callback) +{ + http_client client; + ASSERT_TRUE(client.init()); + + ASSERT_TRUE(client.set_url("http://127.0.0.1:20001/test/get")); + ASSERT_TRUE(client.with_get_method()); + + auto callback = [](const void *, size_t) { return false; }; + + const auto &err = client.exec_method(callback); + ASSERT_EQ(dsn::ERR_CURL_FAILED, err.code()); + + long actual_http_status; + ASSERT_TRUE(client.get_http_status(actual_http_status)); + EXPECT_EQ(200, actual_http_status); + + std::cout << "failed for callback: "; + + // "code=23" means CURLE_WRITE_ERROR, see https://curl.se/libcurl/c/libcurl-errors.html + // for details. + // + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed writing body (18446744073709551615 != 24)" + // * msg="Failure writing output to destination" + // Thus we don't check if `msg` fields are consistent. + const auto expected_description_prefix = + fmt::format("ERR_CURL_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20001/test/get): code=23, " + "desc=\"Failed writing received data to disk/application\""); + NO_FATALS(check_expected_description_prefix(expected_description_prefix, err)); +} + +using http_client_method_case = + std::tuple; + +class HttpClientMethodTest : public testing::TestWithParam +{ +public: + void SetUp() override { ASSERT_TRUE(_client.init()); } + + void test_method_with_response_string(const long expected_http_status, + const std::string &expected_response) + { + std::string actual_response; + ASSERT_TRUE(_client.exec_method(&actual_response)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + + EXPECT_EQ(expected_http_status, actual_http_status); + EXPECT_EQ(expected_response, actual_response); + } + + void test_method_with_response_callback(const long expected_http_status, + const std::string &expected_response) + { + auto callback = [&expected_response](const void *data, size_t length) { + auto compare = [](const char *expected_data, + size_t expected_length, + const void *actual_data, + size_t actual_length) { + if (expected_length != actual_length) { + return false; + } + return std::memcmp(expected_data, actual_data, actual_length) == 0; + }; + EXPECT_PRED4(compare, expected_response.data(), expected_response.size(), data, length); + return true; + }; + ASSERT_TRUE(_client.exec_method(callback)); + + long actual_http_status; + ASSERT_TRUE(_client.get_http_status(actual_http_status)); + EXPECT_EQ(expected_http_status, actual_http_status); + } + + void test_mothod(const std::string &url, + const http_method method, + const std::string &post_data, + const long expected_http_status, + const std::string &expected_response) + { + _client.set_url(url); + + switch (method) { + case http_method::GET: + ASSERT_TRUE(_client.with_get_method()); + break; + case http_method::POST: + ASSERT_TRUE(_client.with_post_method(post_data)); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + test_method_with_response_string(expected_http_status, expected_response); + test_method_with_response_callback(expected_http_status, expected_response); + } + +private: + http_client _client; +}; + +TEST_P(HttpClientMethodTest, ExecMethod) +{ + const char *url; + http_method method; + const char *post_data; + long expected_http_status; + const char *expected_response; + std::tie(url, method, post_data, expected_http_status, expected_response) = GetParam(); + + http_client _client; + test_mothod(url, method, post_data, expected_http_status, expected_response); +} + +const std::vector http_client_method_tests = { + {"http://127.0.0.1:20001/test/get", + http_method::POST, + "with POST DATA", + 400, + "please use GET method"}, + {"http://127.0.0.1:20001/test/get", http_method::GET, "", 200, "you are using GET method"}, + {"http://127.0.0.1:20001/test/post", + http_method::POST, + "with POST DATA", + 200, + "you are using POST method with POST DATA"}, + {"http://127.0.0.1:20001/test/post", http_method::GET, "", 400, "please use POST method"}, +}; + +INSTANTIATE_TEST_CASE_P(HttpClientTest, + HttpClientMethodTest, + testing::ValuesIn(http_client_method_tests)); + +} // namespace dsn diff --git a/src/http/test/http_server_test.cpp b/src/http/test/http_server_test.cpp index 9aa9f17c39..0c6b9ec6d3 100644 --- a/src/http/test/http_server_test.cpp +++ b/src/http/test/http_server_test.cpp @@ -18,6 +18,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include "http/builtin_http_calls.h" #include "http/http_call_registry.h" #include "http/http_message_parser.h" +#include "http/http_method.h" #include "http/http_server.h" #include "runtime/rpc/message_parser.h" #include "runtime/rpc/rpc_message.h" @@ -87,7 +89,12 @@ TEST(bultin_http_calls_test, meta_query) TEST(bultin_http_calls_test, get_help) { + // Used to save current http calls as backup. + std::vector> backup_calls; + + // Remove all http calls. for (const auto &call : http_call_registry::instance().list_all_calls()) { + backup_calls.push_back(call); http_call_registry::instance().remove(call->path); } @@ -111,9 +118,15 @@ TEST(bultin_http_calls_test, get_help) get_help_handler(req, resp); ASSERT_EQ(resp.body, "{\"/\":\"ip:port/\",\"/recentStartTime\":\"ip:port/recentStartTime\"}\n"); + // Remove all http calls, especially `recentStartTime`. for (const auto &call : http_call_registry::instance().list_all_calls()) { http_call_registry::instance().remove(call->path); } + + // Recover http calls from backup. + for (const auto &call : backup_calls) { + http_call_registry::instance().add(call); + } } class http_message_parser_test : public testing::Test @@ -174,7 +187,7 @@ class http_message_parser_test : public testing::Test message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 1); // url @@ -215,7 +228,7 @@ TEST_F(http_message_parser_test, parse_request) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_POST); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::POST)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), "Message Body sdfsdf"); // body @@ -266,7 +279,7 @@ TEST_F(http_message_parser_test, eof) ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[1].to_string(), ""); // body @@ -297,7 +310,7 @@ TEST_F(http_message_parser_test, parse_long_url) message_ptr msg = parser.get_message_on_receive(&reader, read_next); ASSERT_NE(msg, nullptr); ASSERT_EQ(msg->hdr_format, NET_HDR_HTTP); - ASSERT_EQ(msg->header->hdr_type, http_method::HTTP_METHOD_GET); + ASSERT_EQ(msg->header->hdr_type, static_cast(http_method::GET)); ASSERT_EQ(msg->header->context.u.is_request, 1); ASSERT_EQ(msg->buffers.size(), HTTP_MSG_BUFFERS_NUM); ASSERT_EQ(msg->buffers[2].size(), 4097); // url diff --git a/src/http/test/main.cpp b/src/http/test/main.cpp new file mode 100644 index 0000000000..1eeb3d04ef --- /dev/null +++ b/src/http/test/main.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "http/http_method.h" +#include "http/http_server.h" +#include "runtime/app_model.h" +#include "runtime/service_app.h" +#include "utils/blob.h" +#include "utils/error_code.h" +#include "utils/ports.h" + +int gtest_flags = 0; +int gtest_ret = 0; + +class test_http_service : public dsn::http_server_base +{ +public: + test_http_service() + { + register_handler("get", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::GET, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/get"); + register_handler("post", + std::bind(&test_http_service::method_handler, + this, + dsn::http_method::POST, + std::placeholders::_1, + std::placeholders::_2), + "ip:port/test/post"); + } + + ~test_http_service() = default; + + std::string path() const override { return "test"; } + +private: + void method_handler(dsn::http_method target_method, + const dsn::http_request &req, + dsn::http_response &resp) + { + if (req.method != target_method) { + resp.body = fmt::format("please use {} method", enum_to_string(target_method)); + resp.status_code = dsn::http_status_code::bad_request; + return; + } + + std::string postfix; + if (target_method == dsn::http_method::POST) { + postfix = " "; + postfix += req.body.to_string(); + } + + resp.body = + fmt::format("you are using {} method{}", enum_to_string(target_method), postfix); + resp.status_code = dsn::http_status_code::ok; + } + + DISALLOW_COPY_AND_ASSIGN(test_http_service); +}; + +class test_service_app : public dsn::service_app +{ +public: + test_service_app(const dsn::service_app_info *info) : dsn::service_app(info) + { + dsn::register_http_service(new test_http_service); + dsn::start_http_server(); + } + + dsn::error_code start(const std::vector &args) override + { + gtest_ret = RUN_ALL_TESTS(); + gtest_flags = 1; + return dsn::ERR_OK; + } +}; + +GTEST_API_ int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + + // Register test service. + dsn::service_app::register_factory("test"); + + dsn_run_config("config-test.ini", false); + while (gtest_flags == 0) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + +#ifndef ENABLE_GCOV + dsn_exit(gtest_ret); +#endif + return gtest_ret; +} diff --git a/src/utils/error_code.h b/src/utils/error_code.h index 45a6e793cc..04df97947a 100644 --- a/src/utils/error_code.h +++ b/src/utils/error_code.h @@ -180,6 +180,8 @@ DEFINE_ERR_CODE(ERR_RANGER_POLICIES_NO_NEED_UPDATE) DEFINE_ERR_CODE(ERR_RDB_CORRUPTION) DEFINE_ERR_CODE(ERR_DISK_IO_ERROR) + +DEFINE_ERR_CODE(ERR_CURL_FAILED) } // namespace dsn USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_code); diff --git a/src/utils/errors.h b/src/utils/errors.h index 9cbf70e092..ce36befca9 100644 --- a/src/utils/errors.h +++ b/src/utils/errors.h @@ -97,6 +97,8 @@ class error_s return true; } + explicit operator bool() const noexcept { return is_ok(); } + std::string description() const { if (!_info) { @@ -227,7 +229,7 @@ USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_s); #define RETURN_NOT_OK(s) \ do { \ const ::dsn::error_s &_s = (s); \ - if (dsn_unlikely(!_s.is_ok())) { \ + if (dsn_unlikely(!_s)) { \ return _s; \ } \ } while (false); diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 40caace7a4..b5800d3cd1 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -23,6 +23,7 @@ #include #include +#include "http/http_method.h" #include "runtime/api_layer1.h" #include "utils/flags.h" #include "utils/rand.h" @@ -275,7 +276,7 @@ const dsn::metric_filters::metric_fields_type kBriefMetricFields = get_brief_met void metrics_http_service::get_metrics_handler(const http_request &req, http_response &resp) { - if (req.method != http_method::HTTP_METHOD_GET) { + if (req.method != http_method::GET) { resp.body = encode_error_as_json("please use 'GET' method while querying for metrics"); resp.status_code = http_status_code::bad_request; return; diff --git a/src/utils/test/metrics_test.cpp b/src/utils/test/metrics_test.cpp index b2118f03ec..8d65c65dae 100644 --- a/src/utils/test/metrics_test.cpp +++ b/src/utils/test/metrics_test.cpp @@ -2513,7 +2513,7 @@ void test_http_get_metrics(const std::string &request_string, ASSERT_TRUE(req_res.is_ok()); const auto &req = req_res.get_value(); - std::cout << "method: " << req.method << std::endl; + std::cout << "method: " << enum_to_string(req.method) << std::endl; http_response resp; test_get_metrics_handler(req, resp); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 4558aa94f7..f6fd11d4ba 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -276,11 +276,7 @@ ExternalProject_Add(civetweb ExternalProject_Get_property(civetweb SOURCE_DIR) set(civetweb_SRC ${SOURCE_DIR}) -ExternalProject_Add(curl - URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz - http://curl.haxx.se/download/curl-7.47.0.tar.gz - URL_MD5 5109d1232d208dfd712c0272b8360393 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} +set(CURL_OPTIONS --disable-dict --disable-file --disable-ftp @@ -301,6 +297,19 @@ ExternalProject_Add(curl --without-libssh2 --without-ssl --without-libidn + ) +if (APPLE) + set(CURL_OPTIONS + ${CURL_OPTIONS} + --without-nghttp2 + ) +endif () +ExternalProject_Add(curl + URL ${OSS_URL_PREFIX}/curl-7.47.0.tar.gz + http://curl.haxx.se/download/curl-7.47.0.tar.gz + URL_MD5 5109d1232d208dfd712c0272b8360393 + CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} + ${CURL_OPTIONS} BUILD_IN_SOURCE 1 )