Merge branch 'master' into mpuncel/http2-hc-goaway

* master: (70 commits)
  upstream: avoid reset after end_stream in TCP HTTP upstream (envoyproxy#14106)
  bazelci: add fuzz coverage (envoyproxy#14179)
  dependencies: allowlist CVE-2020-8277 to prevent false positives. (envoyproxy#14228)
  cleanup: replace ad-hoc [0, 1] value types with UnitFloat (envoyproxy#14081)
  Update docs for skywalking tracer (envoyproxy#14210)
  Fix some errors in the switch statement when decode dubbo response (envoyproxy#14207)
  Windows: enable tests and envoy-static.exe pdb file (envoyproxy#13688)
  http: add Kill Request HTTP filter (envoyproxy#14170)
  dependencies: fix release_dates error behavior. (envoyproxy#14216)
  thrift filter: support skip decoding data after metadata in the thrift message (envoyproxy#13592)
  update cares (envoyproxy#14213)
  docs: clarify behavior of hedge_on_per_try_timeout (envoyproxy#12983)
  repokitteh: add support for randomized auto-assign. (envoyproxy#14185)
  [grpc] validate grpc config for illegal characters (envoyproxy#14129)
  server: Return nullopt when process_context is nullptr (envoyproxy#14181)
  [Windows] Fix thrift proxy tests (envoyproxy#13220)
  kafka: add missing unit tests (envoyproxy#14195)
  doc: mention gperftools explicitly in PPROF.md (envoyproxy#14199)
  Removed `--use-fake-symbol-table` option. (envoyproxy#14178)
  filter contract: clarification around local replies (envoyproxy#14193)
  ...

Signed-off-by: Michael Puncel <[email protected]>

.azure-pipelines/pipelines.yml

@@ -393,6 +393,12 @@ stages:
               BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
               BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
               GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          - task: PublishTestResults@2
+            inputs:
+              testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
+              testRunTitle: "windows"
+              searchFolder: $(Build.StagingDirectory)/tmp
+            condition: always()
           - task: PublishBuildArtifacts@1
             inputs:
               pathtoPublish: "$(Build.StagingDirectory)/envoy"

.bazelci/presubmit.yml

@@ -24,3 +24,14 @@ tasks:
     test_flags:
       - "--config=coverage"
       - "--config=clang"
+  fuzz_coverage:
+    name: "Fuzz-Coverage"
+    platform: ubuntu1804
+    shell_commands:
+      - "bazel/setup_clang.sh /usr/lib/llvm-10"
+    test_targets:
+      - "//test/server:server_fuzz_test"
+    test_flags:
+      - "--config=fuzz-coverage"
+      - "--config=coverage"
+      - "--config=clang"

.bazelrc

@@ -144,9 +144,9 @@ build:coverage --experimental_use_llvm_covmap
 build:coverage --collect_code_coverage
 build:coverage --test_tag_filters=-nocoverage
 build:coverage --instrumentation_filter="//source(?!/common/chromium_url|/extensions/quic_listeners/quiche/platform)[/:],//include[/:]"
-coverage:test-coverage --test_arg="-l trace"
-coverage:fuzz-coverage --config=plain-fuzzer
-coverage:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
+build:test-coverage --test_arg="-l trace"
+build:fuzz-coverage --config=plain-fuzzer
+build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
 
 # Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
 build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
@@ -277,22 +277,18 @@ build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
 
 # Fuzz builds
-# -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is passed in in the bazel build target
-# rules for fuzz tests. Passing it in the CLI will cause dependencies to be build
-# with the macro. Causing issues in RouteMatcherTest.TestRoutes that expect prod
-# behavior from RE2 library.
-build:asan-fuzzer --config=asan
-build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
-build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
-build:asan-fuzzer --copt=-fno-omit-frame-pointer
-# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
-build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
-
 # Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts.
 build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:plain-fuzzer --define ENVOY_CONFIG_ASAN=1
 build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link
 build:plain-fuzzer --linkopt=-fsanitize=fuzzer-no-link
+build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+
+build:asan-fuzzer --config=plain-fuzzer
+build:asan-fuzzer --config=asan
+build:asan-fuzzer --copt=-fno-omit-frame-pointer
+# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
+build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 
 # Compile database generation config
 build:compdb --build_tag_filters=-nocompdb
@@ -326,6 +322,12 @@ build:clang-cl --define clang_cl=1
 # Override determinism flags (DATE etc) is valid on clang-cl compiler
 build:clang-cl --copt="-Wno-macro-redefined"
 build:clang-cl --copt="-Wno-builtin-macro-redefined"
+# Workaround problematic missing override declarations of mocks
+# TODO: resolve this class of problematic mocks, e.g.
+# ./test/mocks/http/stream.h(16,21): error: 'addCallbacks'
+#     overrides a member function but is not marked 'override'
+#   MOCK_METHOD(void, addCallbacks, (StreamCallbacks & callbacks));
+build:clang-cl --copt="-Wno-inconsistent-missing-override"
 build:clang-cl --action_env=USE_CLANG_CL=1
 
 # Defaults to 'auto' - Off for windows, so override to linux behavior

.dockerignore

@@ -1,4 +1,5 @@
 /*
+!/build_envoy
 !/ci
 !/configs/google-vrp
 !/configs/*yaml

CODEOWNERS

@@ -154,3 +154,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 # HTTP Local Rate Limit
 /*/extensions/filters/http/local_ratelimit @rgs1 @mattklein123
 /*/extensions/filters/common/local_ratelimit @mattklein123 @rgs1
+# HTTP Kill Request
+/*/extensions/filters/http/kill_request @qqustc @htuch

PULL_REQUESTS.md

@@ -60,7 +60,7 @@ changes may be in [docs/root](docs/root) and/or inline with the API protos. Plea
 N/A if there were no documentation changes.
 
 Any PRs with structural changes to the dataplane should also update the [Life of a
-Request](docs/root/intro/life_of_a_request.md) documentation as appropriate.
+Request](https://www.envoyproxy.io/docs/envoy/latest/intro/life_of_a_request) documentation as appropriate.
 
 ### <a name="relnotes"></a>Release notes
 

REPO_LAYOUT.md

@@ -63,9 +63,8 @@ Not every directory within test is described below, but a few highlights:
 ## [source/extensions](source/extensions/) layout
 
 We maintain a very specific code and namespace layout for extensions. This aids in discovering
-code/extensions, and also will allow us in the future to more easily scale out our extension
-maintainers by having OWNERS files specific to certain extensions. (As of this writing, this is not
-currently implemented but that is the plan moving forward.)
+code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNERS).
+
 
 * All extensions are either registered in [all_extensions.bzl](source/extensions/all_extensions.bzl)
   or [extensions_build_config.bzl](source/extensions/extensions_build_config.bzl). The former is
@@ -76,6 +75,14 @@ currently implemented but that is the plan moving forward.)
 * These are the top level extension directories and associated namespaces:
   * [access_loggers/](/source/extensions/access_loggers): Access log implementations which use
     the `Envoy::Extensions::AccessLoggers` namespace.
+  * [bootstrap](/source/extensions/bootstrap): Bootstrap extensions which use
+    the `Envoy::Extensions::Bootstrap` namespace.
+  * [clusters](/source/extensions/clusters): Cluster extensions which use the
+    `Envoy::Extensions::Clusters` namespace.
+  * [compression](/source/extensions/compression): Compression extensions
+    which use `Envoy::Extensions::Compression` namespace.
+  * [fatal_actions](/source/extensions/fatal_actions): Fatal Action extensions
+    which use the `Envoy::Extensions::FatalActions` namespace.
   * [filters/http/](/source/extensions/filters/http): HTTP L7 filters which use the
     `Envoy::Extensions::HttpFilters` namespace.
   * [filters/listener/](/source/extensions/filters/listener): Listener filters which use the
@@ -86,14 +93,24 @@ currently implemented but that is the plan moving forward.)
     `Envoy::Extensions::GrpcCredentials` namespace.
   * [health_checker/](/source/extensions/health_checker): Custom health checkers which use the
     `Envoy::Extensions::HealthCheckers` namespace.
-  * [resolvers/](/source/extensions/resolvers): Network address resolvers which use the
-    `Envoy::Extensions::Resolvers` namespace.
+  * [internal_redirect](/source/extensions/internal_redirect): Internal Redirect
+    extensions which use the `Envoy::Extensions::InternalRedirect` namespace.
+  * [quic_listeners](/source/extensions/quic_listeners): QUIC extensions which
+    use the `Envoy::Quic` namespace.
+  * [resource_monitors](/source/extensions/resource_monitors): Resource monitor
+    extensions which use the `Envoy::Extensions::ResourceMonitors` namespace.
+  * [retry](/source/extensions/retry): Retry extensions which use the
+    `Envoy::Extensions::Retry` namespace.
   * [stat_sinks/](/source/extensions/stat_sinks): Stat sink implementations which use the
     `Envoy::Extensions::StatSinks` namespace.
   * [tracers/](/source/extensions/tracers): Tracers which use the
     `Envoy::Extensions::Tracers` namespace.
   * [transport_sockets/](/source/extensions/transport_sockets): Transport socket implementations
     which use the `Envoy::Extensions::TransportSockets` namespace.
+  * [upstreams](/source/extensions/upstreams): Upstream extensions use the
+    `Envoy::Extensions::Upstreams` namespace.
+  * [watchdog](/source/extensions/watchdog): Watchdog extensions use the
+    `Envoy::Extensions::Watchdog` namespace.
 * Each extension is contained wholly in its own namespace. E.g.,
   `Envoy::Extensions::NetworkFilters::Echo`.
 * Common code that is used by multiple extensions should be in a `common/` directory as close to

STYLE.md

@@ -19,7 +19,7 @@
 # Documentation
 
 * If you are modifying the data plane structually, please keep the [Life of a
-  Request](docs/root/intro/life_of_a_request.md) documentation up-to-date.
+  Request](https://www.envoyproxy.io/docs/envoy/latest/intro/life_of_a_request) documentation up-to-date.
 
 # Deviations from Google C++ style guidelines
 

api/BUILD

@@ -190,6 +190,7 @@ proto_library(
         "//envoy/extensions/filters/http/health_check/v3:pkg",
         "//envoy/extensions/filters/http/ip_tagging/v3:pkg",
         "//envoy/extensions/filters/http/jwt_authn/v3:pkg",
+        "//envoy/extensions/filters/http/kill_request/v3:pkg",
         "//envoy/extensions/filters/http/local_ratelimit/v3:pkg",
         "//envoy/extensions/filters/http/lua/v3:pkg",
         "//envoy/extensions/filters/http/oauth2/v3alpha:pkg",

api/bazel/repository_locations.bzl

@@ -22,14 +22,15 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         use_category = ["api"],
     ),
     com_github_cncf_udpa = dict(
-        project_name = "Universal Data Plane API",
-        project_desc = "Universal Data Plane API Working Group (UDPA-WG)",
+        project_name = "xDS API",
+        project_desc = "xDS API Working Group (xDS-WG)",
         project_url = "https://github.com/cncf/udpa",
-        version = "0.0.1",
-        sha256 = "83a7dcc316d741031f34c0409021432b74a39c4811845a177133f02f948fe2d8",
+        # During the UDPA -> xDS migration, we aren't working with releases.
+        version = "5459f2c994033b0afed7e4a70ac7e90c90c1ffee",
+        sha256 = "c1f5c2438cf725b5f66aa4210dbc4bb691020c5ed4f64d2bc6638b06a11482f1",
         strip_prefix = "udpa-{version}",
-        urls = ["https://github.com/cncf/udpa/archive/v{version}.tar.gz"],
-        release_date = "2020-06-29",
+        urls = ["https://github.com/cncf/udpa/archive/{version}.tar.gz"],
+        release_date = "2020-11-20",
         use_category = ["api"],
     ),
     com_github_openzipkin_zipkinapi = dict(

api/envoy/api/v2/route/route_components.proto

@@ -1133,13 +1133,18 @@ message HedgePolicy {
   // [#not-implemented-hide:]
   type.FractionalPercent additional_request_chance = 2;
 
-  // Indicates that a hedged request should be sent when the per-try timeout
-  // is hit. This will only occur if the retry policy also indicates that a
-  // timed out request should be retried.
-  // Once a timed out request is retried due to per try timeout, the router
-  // filter will ensure that it is not retried again even if the returned
-  // response headers would otherwise be retried according the specified
-  // :ref:`RetryPolicy <envoy_api_msg_route.RetryPolicy>`.
+  // Indicates that a hedged request should be sent when the per-try timeout is hit.
+  // This means that a retry will be issued without resetting the original request, leaving multiple upstream requests in flight.
+  // The first request to complete successfully will be the one returned to the caller.
+  //
+  // * At any time, a successful response (i.e. not triggering any of the retry-on conditions) would be returned to the client.
+  // * Before per-try timeout, an error response (per retry-on conditions) would be retried immediately or returned ot the client
+  //   if there are no more retries left.
+  // * After per-try timeout, an error response would be discarded, as a retry in the form of a hedged request is already in progress.
+  //
+  // Note: For this to have effect, you must have a :ref:`RetryPolicy <envoy_api_msg_route.RetryPolicy>` that retries at least
+  // one error code and specifies a maximum number of retries.
+  //
   // Defaults to false.
   bool hedge_on_per_try_timeout = 3;
 }

api/envoy/config/bootstrap/v3/BUILD

@@ -17,6 +17,6 @@ api_proto_package(
         "//envoy/extensions/transport_sockets/tls/v3:pkg",
         "//envoy/type/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
-        "@com_github_cncf_udpa//udpa/core/v1:pkg",
+        "@com_github_cncf_udpa//xds/core/v3:pkg",
     ],
 )

api/envoy/config/bootstrap/v3/bootstrap.proto

@@ -20,7 +20,7 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
-import "udpa/core/v1/resource_locator.proto";
+import "xds/core/v3/resource_locator.proto";
 
 import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/migrate.proto";
@@ -40,7 +40,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // <config_overview_bootstrap>` for more detail.
 
 // Bootstrap :ref:`configuration overview <config_overview_bootstrap>`.
-// [#next-free-field: 28]
+// [#next-free-field: 29]
 message Bootstrap {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.bootstrap.v2.Bootstrap";
@@ -79,7 +79,7 @@ message Bootstrap {
 
     // Resource locator for listener collection.
     // [#not-implemented-hide:]
-    udpa.core.v1.ResourceLocator lds_resources_locator = 5;
+    xds.core.v3.ResourceLocator lds_resources_locator = 5;
 
     // All post-bootstrap :ref:`Cluster <envoy_api_msg_config.cluster.v3.Cluster>` definitions are
     // provided by a single :ref:`CDS <arch_overview_dynamic_config_cds>`
@@ -88,7 +88,7 @@ message Bootstrap {
 
     // Resource locator for cluster collection.
     // [#not-implemented-hide:]
-    udpa.core.v1.ResourceLocator cds_resources_locator = 6;
+    xds.core.v3.ResourceLocator cds_resources_locator = 6;
 
     // A single :ref:`ADS <config_overview_ads>` source may be optionally
     // specified. This must have :ref:`api_type
@@ -110,7 +110,7 @@ message Bootstrap {
 
   // A list of :ref:`Node <envoy_v3_api_msg_config.core.v3.Node>` field names
   // that will be included in the context parameters of the effective
-  // *UdpaResourceLocator* that is sent in a discovery request when resource
+  // *XdsResourceLocator* that is sent in a discovery request when resource
   // locators are used for LDS/CDS. Any non-string field will have its JSON
   // encoding set as the context parameter value, with the exception of
   // metadata, which will be flattened (see example below). The supported field
@@ -243,10 +243,14 @@ message Bootstrap {
   // Each item contains extension specific configuration.
   repeated core.v3.TypedExtensionConfig bootstrap_extensions = 21;
 
+  // Specifies optional extensions instantiated at startup time and
+  // invoked during crash time on the request that caused the crash.
+  repeated FatalAction fatal_actions = 28;
+
   // Configuration sources that will participate in
-  // *udpa.core.v1.ResourceLocator* authority resolution. The algorithm is as
+  // *xds.core.v3.ResourceLocator* authority resolution. The algorithm is as
   // follows:
-  // 1. The authority field is taken from the *udpa.core.v1.ResourceLocator*, call
+  // 1. The authority field is taken from the *xds.core.v3.ResourceLocator*, call
   //    this *resource_authority*.
   // 2. *resource_authority* is compared against the authorities in any peer
   //    *ConfigSource*. The peer *ConfigSource* is the configuration source
@@ -262,7 +266,7 @@ message Bootstrap {
   // [#not-implemented-hide:]
   repeated core.v3.ConfigSource config_sources = 22;
 
-  // Default configuration source for *udpa.core.v1.ResourceLocator* if all
+  // Default configuration source for *xds.core.v3.ResourceLocator* if all
   // other resolution fails.
   // [#not-implemented-hide:]
   core.v3.ConfigSource default_config_source = 23;
@@ -420,6 +424,20 @@ message Watchdog {
   type.v3.Percent multikill_threshold = 5;
 }
 
+// Fatal actions to run while crashing. Actions can be safe (meaning they are
+// async-signal safe) or unsafe. We run all safe actions before we run unsafe actions.
+// If using an unsafe action that could get stuck or deadlock, it important to
+// have an out of band system to terminate the process.
+//
+// The interface for the extension is ``Envoy::Server::Configuration::FatalAction``.
+// *FatalAction* extensions live in the ``envoy.extensions.fatal_actions`` API
+// namespace.
+message FatalAction {
+  // Extension specific configuration for the action. It's expected to conform
+  // to the ``Envoy::Server::Configuration::FatalAction`` interface.
+  core.v3.TypedExtensionConfig config = 1;
+}
+
 // Runtime :ref:`configuration overview <config_runtime>` (deprecated).
 message Runtime {
   option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Runtime";
@@ -497,7 +515,7 @@ message RuntimeLayer {
 
     // Resource locator for RTDS layer. This is mutually exclusive to *name*.
     // [#not-implemented-hide:]
-    udpa.core.v1.ResourceLocator rtds_resource_locator = 3
+    xds.core.v3.ResourceLocator rtds_resource_locator = 3
         [(udpa.annotations.field_migrate).oneof_promotion = "name_specifier"];
 
     // RTDS configuration source.