rpcdaemon: add HTTP gzip compression (no streaming) (#1909)

conanfile.py

@@ -25,6 +25,7 @@ def requirements(self):
         self.requires('ms-gsl/4.0.0')
         self.requires('nlohmann_json/3.11.3')
         self.requires('tl-expected/1.1.0')
+        self.requires('zlib/1.2.13')
         if self.settings.arch == 'wasm':
             return
 

silkworm/rpc/http/connection.cpp

@@ -16,6 +16,8 @@
 
 #include "connection.hpp"
 
+#include <zlib.h>
+
 #include <array>
 #include <exception>
 #include <string_view>
@@ -190,10 +192,11 @@ Task<void> Connection::handle_actual_request(const boost::beast::http::request<b
     vary_ = req[boost::beast::http::field::vary];
     origin_ = req[boost::beast::http::field::origin];
     method_ = req.method();
+    auto encoding = req[boost::beast::http::field::accept_encoding];
 
     auto rsp_content = co_await request_handler_.handle(req.body());
     if (rsp_content) {
-        co_await do_write(rsp_content->append("\n"), boost::beast::http::status::ok);
+        co_await do_write(rsp_content->append("\n"), boost::beast::http::status::ok, !encoding.empty());
     }
 }
 
@@ -251,7 +254,7 @@ Task<std::size_t> Connection::write(std::string_view content, bool /*last*/) {
     co_return bytes_transferred;
 }
 
-Task<void> Connection::do_write(const std::string& content, boost::beast::http::status http_status) {
+Task<void> Connection::do_write(const std::string& content, boost::beast::http::status http_status, bool compress) {
     try {
         SILK_TRACE << "Connection::do_write response: " << http_status << " content: " << content;
         boost::beast::http::response<boost::beast::http::string_body> res{http_status, request_http_version_};
@@ -265,8 +268,22 @@ Task<void> Connection::do_write(const std::string& content, boost::beast::http::
         res.set(boost::beast::http::field::date, get_date_time());
         res.erase(boost::beast::http::field::host);
         res.keep_alive(request_keep_alive_);
-        res.content_length(content.size());
-        res.body() = content;
+        if (compress) {
+            const std::string compression_type = "gzip";
+            res.set(boost::beast::http::field::content_encoding, compression_type);
+            std::string compressed_data;
+            try {
+                compress_data(content, compressed_data);
+            } catch (const std::exception& e) {
+                SILK_ERROR << "Connection::compress_data exception: " << e.what();
+                throw;
+            }
+            res.content_length(compressed_data.length());
+            res.body() = std::move(compressed_data);
+        } else {
+            res.content_length(content.size());
+            res.body() = std::move(content);
+        }
 
         set_cors<boost::beast::http::string_body>(res);
 
@@ -284,6 +301,37 @@ Task<void> Connection::do_write(const std::string& content, boost::beast::http::
     co_return;
 }
 
+void Connection::compress_data(const std::string& clear_data, std::string& compressed_data) {
+    z_stream strm;
+
+    memset(&strm, 0, sizeof(strm));
+    int ret = Z_OK;
+    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY);
+    if (ret != Z_OK) {
+        throw std::runtime_error("deflateInit2 fail");
+    }
+    strm.avail_in = static_cast<unsigned int>(clear_data.size());
+    auto ptr_clear = const_cast<char*>(clear_data.c_str());
+    strm.next_in = reinterpret_cast<Bytef*>(ptr_clear);
+
+    do {
+        strm.next_out = reinterpret_cast<Bytef*>(temp_compressed_buffer_);
+        strm.avail_out = sizeof(temp_compressed_buffer_);
+
+        ret = deflate(&strm, Z_FINISH);
+        if (ret < 0) {
+            deflateEnd(&strm);
+            throw std::runtime_error("deflate fail");
+        }
+        if (compressed_data.size() < strm.total_out) {
+            // append the block to the output string
+            compressed_data.append(temp_compressed_buffer_, strm.total_out - compressed_data.size());
+        }
+    } while (ret != Z_STREAM_END);
+
+    deflateEnd(&strm);
+}
+
 Connection::AuthorizationResult Connection::is_request_authorized(const boost::beast::http::request<boost::beast::http::string_body>& req) {
     if (!jwt_secret_.has_value() || (*jwt_secret_).empty()) {
         return {};

silkworm/rpc/http/connection.hpp

@@ -86,10 +86,12 @@ class Connection : public StreamWriter {
     Task<bool> do_read();
 
     //! Perform an asynchronous write operation.
-    Task<void> do_write(const std::string& content, boost::beast::http::status http_status);
+    Task<void> do_write(const std::string& content, boost::beast::http::status http_status, bool compress = false);
 
     static std::string get_date_time();
 
+    void compress_data(const std::string& clear_data, std::string& compressed_data);
+
     //! Socket for the connection.
     boost::asio::ip::tcp::socket socket_;
 
@@ -114,6 +116,8 @@ class Connection : public StreamWriter {
     std::string vary_;
     std::string origin_;
     boost::beast::http::verb method_{boost::beast::http::verb::unknown};
+
+    char temp_compressed_buffer_[10 * 1024 * 1024];
 };
 
 }  // namespace silkworm::rpc::http