Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Restrict the maximum number of open HTTP RPC requests #9431

Merged
merged 5 commits into from
Aug 25, 2020
Merged

Restrict the maximum number of open HTTP RPC requests #9431

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d97bced
introduce an option to restrict the maximum number of open HTTP RPC r…
vzqzhang Aug 20, 2020
b38f7c4
code optimization for new http_plugin option http-max-in-flight-requests
vzqzhang Aug 24, 2020
c104598
fix the issue that std::move does nothing for a constant string
vzqzhang Aug 24, 2020
3a00270
Per the PR comments, improve the code quality
vzqzhang Aug 24, 2020
960beec
correct the error code if bytes in flight or request in flight exceed…
vzqzhang Aug 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 68 additions & 22 deletions plugins/http_plugin/http_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,10 @@ namespace eosio {
struct abstract_conn {
virtual ~abstract_conn() {}
virtual bool verify_max_bytes_in_flight() = 0;
virtual bool verify_max_requests_in_flight() = 0;
virtual void handle_exception() = 0;

virtual void update_connection(const std::string & body, int code) = 0;
b1bart marked this conversation as resolved.
Show resolved Hide resolved
};

using abstract_conn_ptr = std::shared_ptr<abstract_conn>;
Expand Down Expand Up @@ -195,7 +198,9 @@ namespace eosio {
uint16_t thread_pool_size = 2;
optional<eosio::chain::named_thread_pool> thread_pool;
std::atomic<size_t> bytes_in_flight{0};
std::atomic<int32_t> requests_in_flight{0};
size_t max_bytes_in_flight = 0;
int32_t max_requests_in_flight = -1;
fc::microseconds max_response_time{30*1000};

optional<tcp::endpoint> https_listen_endpoint;
Expand Down Expand Up @@ -322,25 +327,46 @@ namespace eosio {
return true;
}

template<typename T>
void report_429_error( const T& con, const std::string & what) {
error_results::error_info ei;
ei.code = websocketpp::http::status_code::too_many_requests;
ei.name = "Busy";
ei.what = what;
error_results results{websocketpp::http::status_code::too_many_requests, "Busy", ei};
con->set_body( fc::json::to_string( results, fc::time_point::maximum() ));
con->set_status( websocketpp::http::status_code::too_many_requests );
con->send_http_response();
}

template<typename T>
bool verify_max_bytes_in_flight( const T& con ) {
auto bytes_in_flight_size = bytes_in_flight.load();
if( bytes_in_flight_size > max_bytes_in_flight ) {
fc_dlog( logger, "429 - too many bytes in flight: ${bytes}", ("bytes", bytes_in_flight_size) );
error_results::error_info ei;
ei.code = websocketpp::http::status_code::too_many_requests;
ei.name = "Busy";
ei.what = "Too many bytes in flight: " + std::to_string( bytes_in_flight_size );
error_results results{websocketpp::http::status_code::too_many_requests, "Busy", ei};
con->set_body( fc::json::to_string( results, fc::time_point::maximum() ));
con->set_status( websocketpp::http::status_code::too_many_requests );
con->send_http_response();
string what = "Too many bytes in flight: " + std::to_string( bytes_in_flight_size ) + ". Try again later.";;
report_429_error(con, what);
return false;
}

return true;
}

template<typename T>
bool verify_max_requests_in_flight( const T& con ) {
if (max_requests_in_flight < 0)
return true;

auto requests_in_flight_num = requests_in_flight.load();
if( requests_in_flight_num > max_requests_in_flight ) {
fc_dlog( logger, "429 - too many requests in flight: ${requests}", ("requests", requests_in_flight_num) );
string what = "Too many requests in flight: " + std::to_string( requests_in_flight_num ) + ". Try again later.";
report_429_error(con, what);
return false;
}
return true;
}

/**
* child struct, implementing abstract connection for various underlying connection types
* that ties it to an http_plugin_impl
Expand All @@ -352,20 +378,38 @@ namespace eosio {
abstract_conn_impl(detail::connection_ptr<T> conn, http_plugin_impl& impl)
:_conn(std::move(conn))
,_impl(impl)
{}
{
_impl.requests_in_flight += 1;
}

~abstract_conn_impl() {
_impl.requests_in_flight -= 1;
}

// No copy constructor and no move
abstract_conn_impl(abstract_conn_impl&) = delete;
abstract_conn_impl(abstract_conn_impl&&) = delete;

~abstract_conn_impl() = default;
abstract_conn_impl(abstract_conn_impl&&) = default;
abstract_conn_impl& operator=(abstract_conn_impl&&) noexcept = default;

bool verify_max_bytes_in_flight() override {
return _impl.verify_max_bytes_in_flight(_conn);
}

bool verify_max_requests_in_flight() override {
return _impl.verify_max_requests_in_flight(_conn);
}

void handle_exception()override {
http_plugin_impl::handle_exception<T>(_conn);
}

void update_connection(const std::string & body, int code) override {
_conn->set_body(body);
_conn->set_status( websocketpp::http::status_code::value( code ) );
_conn->send_http_response();
}

detail::connection_ptr<T> _conn;
http_plugin_impl &_impl;
};
Expand Down Expand Up @@ -449,7 +493,7 @@ namespace eosio {
*/
template<typename T>
static auto make_in_flight(T&& object, http_plugin_impl& impl) {
return std::make_shared<in_flight<T>>(in_flight<T>(std::forward<T>(object), impl));
return std::make_shared<in_flight<T>>(std::forward<T>(object), impl);
}

/**
Expand Down Expand Up @@ -512,23 +556,21 @@ namespace eosio {
* @return lambda suitable for url_response_callback
*/
template<typename T>
auto make_http_response_handler( detail::connection_ptr<T> con ) {
return [this, con]( int code, fc::variant response ) {
auto make_http_response_handler( detail::abstract_conn_ptr abstract_conn_ptr ) {
return [this, abstract_conn_ptr]( int code, fc::variant response ) {
auto tracked_response = make_in_flight(std::move(response), *this);
if (!verify_max_bytes_in_flight(con)) {
if (!abstract_conn_ptr->verify_max_bytes_in_flight()) {
return;
}

// post back to an HTTP thread to to allow the response handler to be called from any thread
boost::asio::post( thread_pool->get_executor(), [this, con, code, tracked_response=std::move(tracked_response)]() {
boost::asio::post( thread_pool->get_executor(), [this, abstract_conn_ptr, code, tracked_response=std::move(tracked_response)]() {
try {
std::string json = fc::json::to_string( *(*tracked_response), fc::time_point::now() + max_response_time );
auto tracked_json = make_in_flight(std::move(json), *this);
con->set_body( std::move( *(*tracked_json) ) );
con->set_status( websocketpp::http::status_code::value( code ) );
con->send_http_response();
abstract_conn_ptr->update_connection(std::move(*(*tracked_json)), code);
} catch( ... ) {
handle_exception<T>( con );
abstract_conn_ptr->handle_exception();
}
});
};
Expand Down Expand Up @@ -563,13 +605,14 @@ namespace eosio {
con->append_header( "Content-type", "application/json" );
con->defer_http_response();

if( !verify_max_bytes_in_flight( con ) ) return;
auto abstract_conn_ptr = make_abstract_conn_ptr<T>(con, *this);
if( !verify_max_bytes_in_flight( con ) || !verify_max_requests_in_flight( con ) ) return;

std::string resource = con->get_uri()->get_resource();
auto handler_itr = url_handlers.find( resource );
if( handler_itr != url_handlers.end()) {
std::string body = con->get_request_body();
handler_itr->second( make_abstract_conn_ptr<T>(con, *this), std::move( resource ), std::move( body ), make_http_response_handler<T>(con) );
handler_itr->second( abstract_conn_ptr, std::move( resource ), std::move( body ), make_http_response_handler<T>(abstract_conn_ptr) );
} else {
fc_dlog( logger, "404 - not found: ${ep}", ("ep", resource) );
error_results results{websocketpp::http::status_code::not_found,
Expand Down Expand Up @@ -689,6 +732,8 @@ namespace eosio {
"The maximum body size in bytes allowed for incoming RPC requests")
("http-max-bytes-in-flight-mb", bpo::value<uint32_t>()->default_value(500),
"Maximum size in megabytes http_plugin should use for processing http requests. 503 error response when exceeded." )
("http-max-in-flight-requests", bpo::value<int32_t>()->default_value(-1),
"Maximum number of requests http_plugin should use for processing http requests. 503 error response when exceeded." )
vzqzhang marked this conversation as resolved.
Show resolved Hide resolved
("http-max-response-time-ms", bpo::value<uint32_t>()->default_value(30),
"Maximum time for processing a request.")
("verbose-http-errors", bpo::bool_switch()->default_value(false),
Expand Down Expand Up @@ -778,6 +823,7 @@ namespace eosio {
"http-threads ${num} must be greater than 0", ("num", my->thread_pool_size));

my->max_bytes_in_flight = options.at( "http-max-bytes-in-flight-mb" ).as<uint32_t>() * 1024 * 1024;
my->max_requests_in_flight = options.at( "http-max-in-flight-requests" ).as<int32_t>();
my->max_response_time = fc::microseconds( options.at("http-max-response-time-ms").as<uint32_t>() * 1000 );

//watch out for the returns above when adding new code here
Expand Down