From 5acb1d0f8f3de08859b2734706016488e0632b2e Mon Sep 17 00:00:00 2001 From: Isaac Polinsky <7132878+ikepolinsky@users.noreply.github.com> Date: Fri, 13 Aug 2021 17:33:17 -0400 Subject: [PATCH] ext_proc filter: Increase fuzz coverage (#17636) Fuzz test sends generated downstream requests and replies to ext_proc ProcessingRequest messages with generated ProcessingResponse messages. Currently uses the autonomous upstream for upstream responses. Risk Level: low Testing: Adds fuzz test Signed-off-by: Isaac Polinsky --- test/extensions/filters/http/ext_proc/BUILD | 42 ++ .../http/ext_proc/ext_proc_grpc_corpus/test | 1 + .../http/ext_proc/ext_proc_grpc_fuzz.cc | 320 ++++++++++++ .../ext_proc/ext_proc_grpc_fuzz_helper.cc | 459 ++++++++++++++++++ .../http/ext_proc/ext_proc_grpc_fuzz_helper.h | 121 +++++ 5 files changed, 943 insertions(+) create mode 100644 test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/test create mode 100644 test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.cc create mode 100644 test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc create mode 100644 test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index 827731d8066f..ffb0bf6d46c8 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_fuzz_test", "envoy_package", ) load( @@ -162,3 +163,44 @@ envoy_extension_cc_test_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_test_library( + name = "ext_proc_grpc_fuzz_lib", + srcs = ["ext_proc_grpc_fuzz_helper.cc"], + hdrs = ["ext_proc_grpc_fuzz_helper.h"], + extension_names = ["envoy.filters.http.ext_proc"], + deps = [ + "//source/common/common:thread_lib", + "//source/common/grpc:common_lib", + "//test/common/http:common_lib", + "//test/fuzz:fuzz_runner_lib", + "//test/fuzz:utility_lib", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@com_github_grpc_grpc//:grpc++", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/http/ext_proc/v3alpha:pkg_cc_proto", + "@envoy_api//envoy/service/ext_proc/v3alpha:pkg_cc_proto", + "@envoy_api//envoy/type/v3:pkg_cc_proto", + ], +) + +envoy_cc_fuzz_test( + name = "ext_proc_grpc_fuzz_test", + srcs = ["ext_proc_grpc_fuzz.cc"], + corpus = "ext_proc_grpc_corpus", + deps = [ + ":ext_proc_grpc_fuzz_lib", + ":test_processor_lib", + "//source/common/network:address_lib", + "//source/extensions/filters/http/ext_proc:config", + "//test/common/http:common_lib", + "//test/fuzz:utility_lib", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/http/ext_proc/v3alpha:pkg_cc_proto", + "@envoy_api//envoy/service/ext_proc/v3alpha:pkg_cc_proto", + "@envoy_api//envoy/type/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/test b/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/test new file mode 100644 index 000000000000..aa15709f3ba5 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/test @@ -0,0 +1 @@ +FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.cc b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.cc new file mode 100644 index 000000000000..956a0f2a08a5 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.cc @@ -0,0 +1,320 @@ +// TODO(ikepolinsky): Major action items to improve this fuzzer +// 1. Move external process from separate thread to have test all in one thread +// - Explore using fake gRPC client for this +// 2. Implement sending trailers from downstream and mutating headers/trailers +// in the external process. +// 3. Use an upstream that sends varying responses (also with trailers) +// 4. Explore performance optimizations: +// - Threads and fake gRPC client above might help +// - Local testing had almost 800k inline 8 bit counters resulting in ~3 +// exec/s. How far can we reduce the number of counters? +// - At the loss of reproducibility use a persistent envoy +// 5. Protobuf fuzzing would greatly increase crash test case readability +// - How will this impact speed? +// - Can it be done on single thread as well? +// 6. Restructure to inherit common functions between ExtProcIntegrationTest +// and this class. This involves adding a new ExtProcIntegrationBase class +// common to both. +// 7. Remove locks after crash is addressed by separate issue + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.pb.h" +#include "envoy/service/ext_proc/v3alpha/external_processor.pb.h" +#include "envoy/type/v3/http_status.pb.h" + +#include "source/common/network/address_impl.h" + +#include "test/common/http/common.h" +#include "test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h" +#include "test/extensions/filters/http/ext_proc/test_processor.h" +#include "test/fuzz/fuzz_runner.h" +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +using envoy::extensions::filters::http::ext_proc::v3alpha::ProcessingMode; +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; + +// The buffer size for the listeners +static const uint32_t BufferSize = 100000; + +// These tests exercise the ext_proc filter through Envoy's integration test +// environment by configuring an instance of the Envoy server and driving it +// through the mock network stack. + +class ExtProcIntegrationFuzz : public HttpIntegrationTest, + public Grpc::BaseGrpcClientIntegrationParamTest { +public: + ExtProcIntegrationFuzz(Network::Address::IpVersion ip_version, Grpc::ClientType client_type) + : HttpIntegrationTest(Http::CodecType::HTTP2, ip_version) { + ip_version_ = ip_version; + client_type_ = client_type; + } + + void tearDown() { + cleanupUpstreamAndDownstream(); + test_processor_.shutdown(); + } + + Network::Address::IpVersion ipVersion() const override { return ip_version_; } + Grpc::ClientType clientType() const override { return client_type_; } + + void initializeFuzzer(bool autonomous_upstream) { + autonomous_upstream_ = autonomous_upstream; + autonomous_allow_incomplete_streams_ = true; + initializeConfig(); + HttpIntegrationTest::initialize(); + } + + void initializeConfig() { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + // Create a cluster for our gRPC server pointing to the address that is running the gRPC + // server. + auto* processor_cluster = bootstrap.mutable_static_resources()->add_clusters(); + processor_cluster->set_name("ext_proc_server"); + processor_cluster->mutable_load_assignment()->set_cluster_name("ext_proc_server"); + auto* address = processor_cluster->mutable_load_assignment() + ->add_endpoints() + ->add_lb_endpoints() + ->mutable_endpoint() + ->mutable_address() + ->mutable_socket_address(); + address->set_address(Network::Test::getLoopbackAddressString(ipVersion())); + address->set_port_value(test_processor_.port()); + + // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC. + ConfigHelper::setHttp2( + *(bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0))); + ConfigHelper::setHttp2(*processor_cluster); + + // Make sure both flavors of gRPC client use the right address. + if (ipVersion() == Network::Address::IpVersion::v4) { + const auto addr = std::make_shared( + Network::Test::getLoopbackAddressString(ipVersion()), test_processor_.port()); + setGrpcService(*proto_config_.mutable_grpc_service(), "ext_proc_server", addr); + } else { + const auto addr = std::make_shared( + Network::Test::getLoopbackAddressString(ipVersion()), test_processor_.port()); + setGrpcService(*proto_config_.mutable_grpc_service(), "ext_proc_server", addr); + } + + // Merge the filter. + envoy::config::listener::v3::Filter ext_proc_filter; + ext_proc_filter.set_name("envoy.filters.http.ext_proc"); + ext_proc_filter.mutable_typed_config()->PackFrom(proto_config_); + config_helper_.addFilter(MessageUtil::getJsonStringFromMessageOrDie(ext_proc_filter)); + }); + + // Make sure that we have control over when buffers will fill up. + config_helper_.setBufferLimits(BufferSize, BufferSize); + + setUpstreamProtocol(Http::CodecType::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + } + + IntegrationStreamDecoderPtr sendDownstreamRequest( + absl::optional> modify_headers, + absl::string_view http_method = "GET") { + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers{{":method", std::string(http_method)}}; + if (modify_headers) { + (*modify_headers)(headers); + } + HttpTestUtility::addDefaultHeaders(headers, false); + return codec_client_->makeHeaderOnlyRequest(headers); + } + + IntegrationStreamDecoderPtr sendDownstreamRequestWithBody( + absl::string_view body, + absl::optional> modify_headers, + absl::string_view http_method = "POST") { + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers{{":method", std::string(http_method)}}; + HttpTestUtility::addDefaultHeaders(headers, false); + if (modify_headers) { + (*modify_headers)(headers); + } + return codec_client_->makeRequestWithBody(headers, std::string(body)); + } + + IntegrationStreamDecoderPtr sendDownstreamRequestWithChunks( + FuzzedDataProvider* fdp, ExtProcFuzzHelper* fh, + absl::optional> modify_headers, + absl::string_view http_method = "POST") { + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers{{":method", std::string(http_method)}}; + HttpTestUtility::addDefaultHeaders(headers, false); + if (modify_headers) { + (*modify_headers)(headers); + } + auto encoder_decoder = codec_client_->startRequest(headers); + IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); + auto& encoder = encoder_decoder.first; + + const uint32_t num_chunks = + fdp->ConsumeIntegralInRange(0, ExtProcFuzzMaxStreamChunks); + for (uint32_t i = 0; i < num_chunks; i++) { + // TODO(ikepolinsky): open issue for this crash and remove locks once + // fixed. + // If proxy closes connection before body is fully sent it causes a + // crash. To address this, the external processor sets a flag to + // signal when it has generated an immediate response which will close + // the connection in the future. We check this flag, which is protected + // by a lock, before sending a chunk. If the flag is set, we don't attempt + // to send more data, regardless of whether or not the + // codec_client connection is still open. There are no locks protecting + // the codec_client connection and cannot trust that it's safe to send + // another chunk + fh->immediate_resp_lock_.lock(); + if (fh->immediate_resp_sent_) { + ENVOY_LOG_MISC(trace, "Proxy closed connection, returning early"); + fh->immediate_resp_lock_.unlock(); + return response; + } + const uint32_t data_size = fdp->ConsumeIntegralInRange(0, ExtProcFuzzMaxDataSize); + ENVOY_LOG_MISC(trace, "Sending chunk of {} bytes", data_size); + codec_client_->sendData(encoder, data_size, false); + fh->immediate_resp_lock_.unlock(); + } + + // See comment above + fh->immediate_resp_lock_.lock(); + if (!fh->immediate_resp_sent_) { + ENVOY_LOG_MISC(trace, "Sending empty chunk to close stream"); + Buffer::OwnedImpl empty_chunk; + codec_client_->sendData(encoder, empty_chunk, true); + } + fh->immediate_resp_lock_.unlock(); + return response; + } + + IntegrationStreamDecoderPtr randomDownstreamRequest(FuzzedDataProvider* fdp, + ExtProcFuzzHelper* fh) { + // From the external processor's view each of these requests + // are handled the same way. They only differ in what the server should + // send back to the client. + // TODO(ikepolinsky): add random flag for sending trailers with a request + // using HttpIntegration::sendTrailers() + switch (fdp->ConsumeEnum()) { + case HttpMethod::GET: + ENVOY_LOG_MISC(trace, "Sending GET request"); + return sendDownstreamRequest(absl::nullopt); + case HttpMethod::POST: + if (fdp->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "Sending POST request with body"); + const uint32_t data_size = fdp->ConsumeIntegralInRange(0, ExtProcFuzzMaxDataSize); + const std::string data = std::string(data_size, 'a'); + return sendDownstreamRequestWithBody(data, absl::nullopt); + } else { + ENVOY_LOG_MISC(trace, "Sending POST request with chunked body"); + return sendDownstreamRequestWithChunks(fdp, fh, absl::nullopt); + } + default: + RELEASE_ASSERT(false, "Unhandled HttpMethod"); + } + } + + envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor proto_config_{}; + TestProcessor test_processor_; + Network::Address::IpVersion ip_version_; + Grpc::ClientType client_type_; +}; + +DEFINE_FUZZER(const uint8_t* buf, size_t len) { + // Split the buffer into two buffers with at least 1 byte + if (len < 2) { + return; + } + + // External Process and downstream are on different threads so they should + // have separate data providers + const size_t downstream_buf_len = len / 2; + const size_t ext_proc_buf_len = len - downstream_buf_len; + + // downstream buf starts at 0, ext_prob buf starts at buf[downstream_buf_len] + FuzzedDataProvider downstream_provider(buf, downstream_buf_len); + FuzzedDataProvider ext_proc_provider(&buf[downstream_buf_len], ext_proc_buf_len); + + // Get IP and gRPC version from environment + ExtProcIntegrationFuzz fuzzer(TestEnvironment::getIpVersionsForTest()[0], + TestEnvironment::getsGrpcVersionsForTest()[0]); + ExtProcFuzzHelper fuzz_helper(&ext_proc_provider); + + // This starts an external processor in a separate thread. This allows for the + // external process to consume messages in a loop without blocking the fuzz + // target from receiving the response. + fuzzer.test_processor_.start( + [&fuzz_helper](grpc::ServerReaderWriter* stream) { + while (true) { + ProcessingRequest req; + if (!stream->Read(&req)) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "expected message"); + } + + fuzz_helper.logRequest(&req); + + // The following blocks generate random data for the 9 fields of the + // ProcessingResponse gRPC message + + // 1 - 7. Randomize response + // If true, immediately close the connection with a random Grpc Status. + // Otherwise randomize the response + ProcessingResponse resp; + if (fuzz_helper.provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "Immediately Closing gRPC connection"); + return fuzz_helper.randomGrpcStatusWithMessage(); + } else { + ENVOY_LOG_MISC(trace, "Generating Random ProcessingResponse"); + fuzz_helper.randomizeResponse(&resp, &req); + } + + // 8. Randomize dynamic_metadata + // TODO(ikepolinsky): ext_proc does not support dynamic_metadata + + // 9. Randomize mode_override + if (fuzz_helper.provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "Generating Random ProcessingMode Override"); + ProcessingMode* msg = resp.mutable_mode_override(); + fuzz_helper.randomizeOverrideResponse(msg); + } + + ENVOY_LOG_MISC(trace, "Response generated, writing to stream."); + stream->Write(resp); + } + + return grpc::Status::OK; + }); + + ENVOY_LOG_MISC(trace, "External Process started."); + + fuzzer.initializeFuzzer(true); + ENVOY_LOG_MISC(trace, "Fuzzer initialized"); + + const auto response = fuzzer.randomDownstreamRequest(&downstream_provider, &fuzz_helper); + + // For fuzz testing we don't care about the response code, only that + // the stream ended in some graceful manner + ENVOY_LOG_MISC(trace, "Waiting for response."); + if (response->waitForEndStream(std::chrono::milliseconds(200))) { + ENVOY_LOG_MISC(trace, "Response received."); + } else { + // TODO(ikepolinsky): investigate if there is anyway around this. + // Waiting too long for a fuzz case to fail will drastically + // reduce executions/second. + ENVOY_LOG_MISC(trace, "Response timed out."); + } + fuzzer.tearDown(); +} + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc new file mode 100644 index 000000000000..b5d5e112dffe --- /dev/null +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc @@ -0,0 +1,459 @@ +#include "test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h" + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.pb.h" +#include "envoy/service/ext_proc/v3alpha/external_processor.pb.h" +#include "envoy/type/v3/http_status.pb.h" + +#include "source/common/common/thread.h" + +#include "test/common/http/common.h" +#include "test/fuzz/fuzz_runner.h" +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +using envoy::extensions::filters::http::ext_proc::v3alpha::ProcessingMode; +using envoy::service::ext_proc::v3alpha::CommonResponse; +using envoy::service::ext_proc::v3alpha::HeaderMutation; +using envoy::service::ext_proc::v3alpha::ImmediateResponse; +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; +using envoy::type::v3::StatusCode; + +const StatusCode HttpStatusCodes[] = { + StatusCode::Continue, + StatusCode::OK, + StatusCode::Created, + StatusCode::Accepted, + StatusCode::NonAuthoritativeInformation, + StatusCode::NoContent, + StatusCode::ResetContent, + StatusCode::PartialContent, + StatusCode::MultiStatus, + StatusCode::AlreadyReported, + StatusCode::IMUsed, + StatusCode::MultipleChoices, + StatusCode::MovedPermanently, + StatusCode::Found, + StatusCode::SeeOther, + StatusCode::NotModified, + StatusCode::UseProxy, + StatusCode::TemporaryRedirect, + StatusCode::PermanentRedirect, + StatusCode::BadRequest, + StatusCode::Unauthorized, + StatusCode::PaymentRequired, + StatusCode::Forbidden, + StatusCode::NotFound, + StatusCode::MethodNotAllowed, + StatusCode::NotAcceptable, + StatusCode::ProxyAuthenticationRequired, + StatusCode::RequestTimeout, + StatusCode::Conflict, + StatusCode::Gone, + StatusCode::LengthRequired, + StatusCode::PreconditionFailed, + StatusCode::PayloadTooLarge, + StatusCode::URITooLong, + StatusCode::UnsupportedMediaType, + StatusCode::RangeNotSatisfiable, + StatusCode::ExpectationFailed, + StatusCode::MisdirectedRequest, + StatusCode::UnprocessableEntity, + StatusCode::Locked, + StatusCode::FailedDependency, + StatusCode::UpgradeRequired, + StatusCode::PreconditionRequired, + StatusCode::TooManyRequests, + StatusCode::RequestHeaderFieldsTooLarge, + StatusCode::InternalServerError, + StatusCode::NotImplemented, + StatusCode::BadGateway, + StatusCode::ServiceUnavailable, + StatusCode::GatewayTimeout, + StatusCode::HTTPVersionNotSupported, + StatusCode::VariantAlsoNegotiates, + StatusCode::InsufficientStorage, + StatusCode::LoopDetected, + StatusCode::NotExtended, + StatusCode::NetworkAuthenticationRequired, +}; + +const grpc::StatusCode GrpcStatusCodes[] = { + grpc::StatusCode::OK, + grpc::StatusCode::CANCELLED, + grpc::StatusCode::UNKNOWN, + grpc::StatusCode::INVALID_ARGUMENT, + grpc::StatusCode::DEADLINE_EXCEEDED, + grpc::StatusCode::NOT_FOUND, + grpc::StatusCode::ALREADY_EXISTS, + grpc::StatusCode::PERMISSION_DENIED, + grpc::StatusCode::RESOURCE_EXHAUSTED, + grpc::StatusCode::FAILED_PRECONDITION, + grpc::StatusCode::ABORTED, + grpc::StatusCode::OUT_OF_RANGE, + grpc::StatusCode::UNIMPLEMENTED, + grpc::StatusCode::INTERNAL, + grpc::StatusCode::UNAVAILABLE, + grpc::StatusCode::DATA_LOSS, + grpc::StatusCode::UNAUTHENTICATED, +}; + +ExtProcFuzzHelper::ExtProcFuzzHelper(FuzzedDataProvider* provider) { + provider_ = provider; + immediate_resp_sent_ = false; +} + +std::string ExtProcFuzzHelper::consumeRepeatedString() { + const uint32_t str_len = provider_->ConsumeIntegralInRange(0, ExtProcFuzzMaxDataSize); + return std::string(str_len, 'b'); +} + +// Since FuzzedDataProvider requires enums to define a kMaxValue, we cannot +// use the envoy::type::v3::StatusCode enum directly. +StatusCode ExtProcFuzzHelper::randomHttpStatus() { + const StatusCode rv = provider_->PickValueInArray(HttpStatusCodes); + ENVOY_LOG_MISC(trace, "Selected HTTP StatusCode {}", rv); + return rv; +} + +// Since FuzzedDataProvider requires enums to define a kMaxValue, we cannot +// use the grpc::StatusCode enum directly. +grpc::StatusCode ExtProcFuzzHelper::randomGrpcStatusCode() { + const grpc::StatusCode rv = provider_->PickValueInArray(GrpcStatusCodes); + ENVOY_LOG_MISC(trace, "Selected gRPC StatusCode {}", rv); + return rv; +} + +void ExtProcFuzzHelper::logRequest(const ProcessingRequest* req) { + if (req->has_request_headers()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest request_headers"); + } else if (req->has_response_headers()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest response_headers"); + } else if (req->has_request_body()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest request_body"); + } else if (req->has_response_body()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest response_body"); + } else if (req->has_request_trailers()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest request_trailers"); + } else if (req->has_response_trailers()) { + ENVOY_LOG_MISC(trace, "Received ProcessingRequest response_trailers"); + } else { + ENVOY_LOG_MISC(trace, "Received unexpected ProcessingRequest"); + } +} + +grpc::Status ExtProcFuzzHelper::randomGrpcStatusWithMessage() { + const grpc::StatusCode code = randomGrpcStatusCode(); + ENVOY_LOG_MISC(trace, "Closing stream with StatusCode {}", code); + return grpc::Status(code, consumeRepeatedString()); +} + +// TODO(ikepolinsky): implement this function +// Randomizes a HeaderMutation taken as input. Header/Trailer values of the +// request are available in req which allows for more guided manipulation of the +// headers. The bool value should be false to manipulate headers and +// true to manipulate trailers (which are also a header map) +void ExtProcFuzzHelper::randomizeHeaderMutation(HeaderMutation*, const ProcessingRequest*, + const bool) { + // Each of the following blocks generates random data for the 2 fields + // of a HeaderMutation gRPC message + + // 1. Randomize set_headers + // TODO(ikepolinsky): randomly add headers + + // 2. Randomize remove headers + // TODO(ikepolinsky): Randomly remove headers +} + +void ExtProcFuzzHelper::randomizeCommonResponse(CommonResponse* msg, const ProcessingRequest* req) { + // Each of the following blocks generates random data for the 5 fields + // of CommonResponse gRPC message + // 1. Randomize status + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case CommonResponseStatus::Continue: + ENVOY_LOG_MISC(trace, "CommonResponse status CONTINUE"); + msg->set_status(CommonResponse::CONTINUE); + break; + case CommonResponseStatus::ContinueAndReplace: + ENVOY_LOG_MISC(trace, "CommonResponse status CONTINUE_AND_REPLACE"); + msg->set_status(CommonResponse::CONTINUE_AND_REPLACE); + break; + default: + RELEASE_ASSERT(false, "Unhandled case in random CommonResponse Status"); + } + } + + // 2. Randomize header_mutation + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "CommonResponse setting header_mutation"); + randomizeHeaderMutation(msg->mutable_header_mutation(), req, false); + } + + // 3. Randomize body_mutation + if (provider_->ConsumeBool()) { + auto* body_mutation = msg->mutable_body_mutation(); + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "CommonResponse setting body_mutation, replacing body with bytes"); + body_mutation->set_body(consumeRepeatedString()); + } else { + ENVOY_LOG_MISC(trace, "CommonResponse setting body_mutation, clearing body"); + body_mutation->set_clear_body(provider_->ConsumeBool()); + } + } + + // 4. Randomize trailers + // TODO(ikepolinsky) ext_proc currently does not support this field + + // 5. Randomize clear_route_cache + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "CommonResponse clearing route cache"); + msg->set_clear_route_cache(true); + } +} + +void ExtProcFuzzHelper::randomizeImmediateResponse(ImmediateResponse* msg, + const ProcessingRequest* req) { + // Each of the following blocks generates random data for the 5 fields + // of an ImmediateResponse gRPC message + // 1. Randomize HTTP status (required) + ENVOY_LOG_MISC(trace, "ImmediateResponse setting status"); + msg->mutable_status()->set_code(randomHttpStatus()); + + // 2. Randomize headers + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "ImmediateResponse setting headers"); + randomizeHeaderMutation(msg->mutable_headers(), req, false); + } + + // 3. Randomize body + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "ImmediateResponse setting body"); + msg->set_body(consumeRepeatedString()); + } + + // 4. Randomize grpc_status + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "ImmediateResponse setting grpc_status"); + msg->mutable_grpc_status()->set_status(randomGrpcStatusCode()); + } + + // 5. Randomize details + if (provider_->ConsumeBool()) { + ENVOY_LOG_MISC(trace, "ImmediateResponse setting details"); + msg->set_details(consumeRepeatedString()); + } +} + +void ExtProcFuzzHelper::randomizeOverrideResponse(ProcessingMode* msg) { + // Each of the following blocks generates random data for the 6 fields + // of a ProcessingMode gRPC message + // 1. Randomize request_header_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case HeaderSendSetting::Default: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_header_mode DEFAULT"); + msg->set_request_header_mode(ProcessingMode::DEFAULT); + break; + case HeaderSendSetting::Send: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_header_mode SEND"); + msg->set_request_header_mode(ProcessingMode::SEND); + break; + case HeaderSendSetting::Skip: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_header_mode SKIP"); + msg->set_request_header_mode(ProcessingMode::SKIP); + break; + default: + RELEASE_ASSERT(false, "HeaderSendSetting not handled"); + } + } + + // 2. Randomize response_header_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case HeaderSendSetting::Default: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_header_mode DEFAULT"); + msg->set_response_header_mode(ProcessingMode::DEFAULT); + break; + case HeaderSendSetting::Send: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_header_mode SEND"); + msg->set_response_header_mode(ProcessingMode::SEND); + break; + case HeaderSendSetting::Skip: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_header_mode SKIP"); + msg->set_response_header_mode(ProcessingMode::SKIP); + break; + default: + RELEASE_ASSERT(false, "HeaderSendSetting not handled"); + } + } + + // 3. Randomize request_body_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case BodySendSetting::None: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_body_mode NONE"); + msg->set_request_body_mode(ProcessingMode::NONE); + break; + case BodySendSetting::Streamed: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_body_mode STREAMED"); + msg->set_request_body_mode(ProcessingMode::STREAMED); + break; + case BodySendSetting::Buffered: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_body_mode BUFFERED"); + msg->set_request_body_mode(ProcessingMode::BUFFERED); + break; + case BodySendSetting::BufferedPartial: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_body_mode " + "BUFFERED_PARTIAL"); + msg->set_request_body_mode(ProcessingMode::BUFFERED_PARTIAL); + break; + default: + RELEASE_ASSERT(false, "BodySendSetting not handled"); + } + } + + // 4. Randomize response_body_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case BodySendSetting::None: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_body_mode NONE"); + msg->set_response_body_mode(ProcessingMode::NONE); + break; + case BodySendSetting::Streamed: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_body_mode STREAMED"); + msg->set_response_body_mode(ProcessingMode::STREAMED); + break; + case BodySendSetting::Buffered: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_body_mode BUFFERED"); + msg->set_response_body_mode(ProcessingMode::BUFFERED); + break; + case BodySendSetting::BufferedPartial: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_body_mode " + "BUFFERED_PARTIAL"); + msg->set_response_body_mode(ProcessingMode::BUFFERED_PARTIAL); + break; + default: + RELEASE_ASSERT(false, "BodySendSetting not handled"); + } + } + + // 5. Randomize request_trailer_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case HeaderSendSetting::Default: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_trailer_mode DEFAULT"); + msg->set_request_trailer_mode(ProcessingMode::DEFAULT); + break; + case HeaderSendSetting::Send: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_trailer_mode SEND"); + msg->set_request_trailer_mode(ProcessingMode::SEND); + break; + case HeaderSendSetting::Skip: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting request_trailer_mode SKIP"); + msg->set_request_trailer_mode(ProcessingMode::SKIP); + break; + default: + RELEASE_ASSERT(false, "HeaderSendSetting not handled"); + } + } + + // 6. Randomize response_trailer_mode + if (provider_->ConsumeBool()) { + switch (provider_->ConsumeEnum()) { + case HeaderSendSetting::Default: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_trailer_mode DEFAULT"); + msg->set_response_trailer_mode(ProcessingMode::DEFAULT); + break; + case HeaderSendSetting::Send: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_trailer_mode SEND"); + msg->set_response_trailer_mode(ProcessingMode::SEND); + break; + case HeaderSendSetting::Skip: + ENVOY_LOG_MISC(trace, "Override ProcessingMode: setting response_trailer_mode SKIP"); + msg->set_response_trailer_mode(ProcessingMode::SKIP); + break; + default: + RELEASE_ASSERT(false, "HeaderSendSetting not handled"); + } + } +} + +void ExtProcFuzzHelper::randomizeResponse(ProcessingResponse* resp, const ProcessingRequest* req) { + // Each of the following switch cases generate random data for 1 of the 7 + // ProcessingResponse.response fields + switch (provider_->ConsumeEnum()) { + // 1. Randomize request_headers message + case ResponseType::RequestHeaders: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting request_headers response"); + CommonResponse* msg = resp->mutable_request_headers()->mutable_response(); + randomizeCommonResponse(msg, req); + break; + } + // 2. Randomize response_headers message + case ResponseType::ResponseHeaders: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting response_headers response"); + CommonResponse* msg = resp->mutable_response_headers()->mutable_response(); + randomizeCommonResponse(msg, req); + break; + } + // 3. Randomize request_body message + case ResponseType::RequestBody: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting request_body response"); + CommonResponse* msg = resp->mutable_request_body()->mutable_response(); + randomizeCommonResponse(msg, req); + break; + } + // 4. Randomize response_body message + case ResponseType::ResponseBody: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting response_body response"); + CommonResponse* msg = resp->mutable_response_body()->mutable_response(); + randomizeCommonResponse(msg, req); + break; + } + // 5. Randomize request_trailers message + case ResponseType::RequestTrailers: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting request_trailers response"); + HeaderMutation* header_mutation = resp->mutable_request_trailers()->mutable_header_mutation(); + randomizeHeaderMutation(header_mutation, req, true); + break; + } + // 6. Randomize response_trailers message + case ResponseType::ResponseTrailers: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting response_trailers response"); + HeaderMutation* header_mutation = resp->mutable_request_trailers()->mutable_header_mutation(); + randomizeHeaderMutation(header_mutation, req, true); + break; + } + // 7. Randomize immediate_response message + case ResponseType::ImmediateResponse: { + ENVOY_LOG_MISC(trace, "ProcessingResponse setting immediate_response response"); + ImmediateResponse* msg = resp->mutable_immediate_response(); + randomizeImmediateResponse(msg, req); + + // Since we are sending an immediate response, envoy will close the + // mock connection with the downstream. As a result, the + // codec_client_connection will be deleted and if the upstream is still + // sending data chunks (e.g., streaming mode) it will cause a crash + // Note: At this point provider_lock_ is not held so deadlock is not + // possible + + immediate_resp_lock_.lock(); + immediate_resp_sent_ = true; + immediate_resp_lock_.unlock(); + break; + } + default: + RELEASE_ASSERT(false, "ProcessingResponse Action not handled"); + } +} + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h new file mode 100644 index 000000000000..1b5a6359dea4 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.h @@ -0,0 +1,121 @@ +#pragma once + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.pb.h" +#include "envoy/service/ext_proc/v3alpha/external_processor.pb.h" +#include "envoy/type/v3/http_status.pb.h" + +#include "source/common/common/thread.h" +#include "source/common/grpc/common.h" + +#include "test/common/http/common.h" +#include "test/fuzz/fuzz_runner.h" +#include "test/test_common/utility.h" + +#include "grpc++/server_builder.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +using envoy::extensions::filters::http::ext_proc::v3alpha::ProcessingMode; +using envoy::service::ext_proc::v3alpha::CommonResponse; +using envoy::service::ext_proc::v3alpha::HeaderMutation; +using envoy::service::ext_proc::v3alpha::ImmediateResponse; +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; +using envoy::type::v3::StatusCode; + +const uint32_t ExtProcFuzzMaxDataSize = 1024; +const uint32_t ExtProcFuzzMaxStreamChunks = 50; + +// TODO(ikepolinsky): integrate an upstream that can be controlled by the fuzzer +// and responds appropriately to HTTP requests. +// Currently using autonomous upstream which sends 10 bytes in response to any +// HTTP message. This is an invalid response to TRACE, HEAD, and PUT requests +// so they are currently not supported. DELETE, PATCH, CONNECT, and OPTIONS +// use the same two send functions as GET and POST but with a different method value +// (e.g., they just use sendDownstreamRequest and sendDownstreamRequestWithBody) +// for simplicity I have excluded anything other than GET and POST for now. +// As more HTTP methods are added, update kMaxValue as appropriate to include +// the new enum as a fuzz choice +enum class HttpMethod { + GET, + POST, + DELETE, + PATCH, + CONNECT, + OPTIONS, + TRACE, + HEAD, + PUT, + kMaxValue = POST // NOLINT: FuzzedDataProvider requires lowercase k +}; + +enum class ResponseType { + RequestHeaders, + ResponseHeaders, + RequestBody, + ResponseBody, + ImmediateResponse, + RequestTrailers, + ResponseTrailers, + kMaxValue = ResponseTrailers // NOLINT: FuzzedDataProvider requires lowercase k +}; + +enum class HeaderSendSetting { + Default, + Send, + Skip, + kMaxValue = Skip // NOLINT: FuzzedDataProvider requires lowercase k +}; + +enum class BodySendSetting { + None, + Buffered, + Streamed, + BufferedPartial, + kMaxValue = BufferedPartial // NOLINT: FuzzedDataProvider requires lowercase k +}; + +enum class CommonResponseStatus { + Continue, + ContinueAndReplace, + kMaxValue = ContinueAndReplace // NOLINT: FuzzedDataProvider requires lowercase k +}; + +// Helper class for fuzzing the ext_proc filter. +// This class exposes functions for randomizing fields of ProcessingResponse +// messages and sub-messages. Further, this class exposes wrappers for +// FuzzedDataProvider functions enabling it to be used safely across multiple +// threads (e.g., in the fuzzer thread and the external processor thread). +class ExtProcFuzzHelper { +public: + ExtProcFuzzHelper(FuzzedDataProvider* provider); + + StatusCode randomHttpStatus(); + std::string consumeRepeatedString(); + grpc::StatusCode randomGrpcStatusCode(); + grpc::Status randomGrpcStatusWithMessage(); + + void logRequest(const ProcessingRequest* req); + void randomizeHeaderMutation(HeaderMutation* headers, const ProcessingRequest* req, + const bool trailers); + void randomizeCommonResponse(CommonResponse* msg, const ProcessingRequest* req); + void randomizeImmediateResponse(ImmediateResponse* msg, const ProcessingRequest* req); + void randomizeOverrideResponse(ProcessingMode* msg); + void randomizeResponse(ProcessingResponse* resp, const ProcessingRequest* req); + + FuzzedDataProvider* provider_; + + // Protects immediate_resp_sent_ + Thread::MutexBasicLockable immediate_resp_lock_; + // Flags if an immediate response was generated and sent + bool immediate_resp_sent_; +}; + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy