Merge branch 'main' into feat/otel-resource-detectors-env

Signed-off-by: Joao Grassi <[email protected]>

.github/workflows/mobile-release.yml

@@ -119,6 +119,7 @@ jobs:
         version="0.5.0.$(date '+%Y%m%d')"
         python mobile/ci/sonatype_nexus_upload.py \
           --profile_id=$SONATYPE_PROFILE_ID \
+          --artifact_id=envoy \
           --version=$version \
           --files \
             envoy.aar \
@@ -161,14 +162,16 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       working-directory: mobile
       run: |
+        # --define=google_grpc=enabled must be defined last because
+        # --config=mobile-release-android has --define=google_grpc=disabled
         version="0.5.0.$(date '+%Y%m%d')"
         ./bazelw build \
           --config=mobile-remote-release-clang \
           --remote_header="Authorization=Bearer $GITHUB_TOKEN" \
           --fat_apk_cpu=x86,x86_64,armeabi-v7a,arm64-v8a \
           --define=pom_version="$version" \
-          --define=google_grpc=enabled \
           --config=mobile-release-android \
+          --define=google_grpc=enabled \
           --linkopt=-fuse-ld=lld \
           //:android_xds_dist
     - name: 'Tar artifacts'
@@ -235,6 +238,7 @@ jobs:
         version="0.5.0.$(date '+%Y%m%d')"
         python mobile/ci/sonatype_nexus_upload.py \
           --profile_id=$SONATYPE_PROFILE_ID \
+          --artifact_id=envoy-xds \
           --version=$version \
           --files \
             envoy_xds.aar \

api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto

@@ -22,7 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Local Rate limit :ref:`configuration overview <config_http_filters_local_rate_limit>`.
 // [#extension: envoy.filters.http.local_ratelimit]
 
-// [#next-free-field: 15]
+// [#next-free-field: 16]
 message LocalRateLimit {
   // The human readable prefix to use when emitting stats.
   string stat_prefix = 1 [(validate.rules).string = {min_len: 1}];
@@ -125,4 +125,9 @@ message LocalRateLimit {
   // no matching descriptor. If set to true, default token bucket will always
   // be consumed. Default is true.
   google.protobuf.BoolValue always_consume_default_token_bucket = 14;
+
+  // Specifies whether a ``RESOURCE_EXHAUSTED`` gRPC code must be returned instead
+  // of the default ``UNAVAILABLE`` gRPC code for a rate limited gRPC call. The
+  // HTTP code will be 200 for a gRPC response.
+  bool rate_limited_as_resource_exhausted = 15;
 }

bazel/aspect.patch

@@ -0,0 +1,20 @@
+diff --git a/lib/private/yq.bzl b/lib/private/yq.bzl
+index 29ca3d7..c8cd5eb 100644
+--- a/lib/private/yq.bzl
++++ b/lib/private/yq.bzl
+@@ -71,10 +71,13 @@ def _yq_impl(ctx):
+
+     # For split operations, yq outputs files in the same directory so we
+     # must cd to the correct output dir before executing it
+-    bin_dir = "/".join([ctx.bin_dir.path, ctx.label.package]) if ctx.label.package else ctx.bin_dir.path
++    bin_dir = ctx.bin_dir.path
++    if ctx.label.workspace_name:
++        bin_dir = "%s/external/%s" % (bin_dir, ctx.label.workspace_name)
++    bin_dir = "/".join([bin_dir, ctx.label.package]) if ctx.label.package else bin_dir
+     escape_bin_dir = _escape_path(bin_dir)
+     cmd = "cd {bin_dir} && {yq} {args} {eval_cmd} {expression} {sources} {maybe_out}".format(
+-        bin_dir = ctx.bin_dir.path + "/" + ctx.label.package,
++        bin_dir = bin_dir,
+         yq = escape_bin_dir + yq_bin.path,
+         eval_cmd = "eval" if len(inputs) <= 1 else "eval-all",
+         args = " ".join(args),

bazel/repositories.bzl

@@ -339,7 +339,12 @@ def envoy_dependencies(skip_targets = []):
         patches = ["@envoy//bazel:rules_pkg.patch"],
     )
     external_http_archive("com_github_aignas_rules_shellcheck")
-    external_http_archive("aspect_bazel_lib")
+    external_http_archive(
+        "aspect_bazel_lib",
+        patch_args = ["-p1"],
+        patches = ["@envoy//bazel:aspect.patch"],
+    )
+
     _com_github_fdio_vpp_vcl()
 
     # Unconditional, since we use this only for compiler-agnostic fuzzing utils.

changelogs/current.yaml

@@ -5,6 +5,11 @@ behavior_changes:
 
 minor_behavior_changes:
 # *Changes that may cause incompatibilities for some users, but should not for most*
+- area: local_rate_limit
+  change: |
+    Added new configuration field :ref:`rate_limited_as_resource_exhausted
+    <envoy_v3_api_field_extensions.filters.http.local_ratelimit.v3.LocalRateLimit.rate_limited_as_resource_exhausted>`
+    to allow for setting if rate limit grpc response should be RESOURCE_EXHAUSTED instead of the default UNAVAILABLE.
 
 bug_fixes:
 # *Changes expected to improve the state of the world and are unlikely to have negative effects*
@@ -21,17 +26,33 @@ bug_fixes:
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
+- area: http
+  change: |
+    Removed ``envoy.reloadable_features.expand_agnostic_stream_lifetime`` and legacy code paths.
+- area: http
+  change: |
+    removed ``envoy.reloadable_features.correctly_validate_alpn`` and legacy code paths.
+- area: maglev
+  change: |
+    Removed ``envoy.reloadable_features.allow_compact_maglev`` and legacy code paths.
 - area: router
   change: |
     Removed the deprecated ``envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent``
     runtime flag and legacy code path.
-
 - area: upstream
   change: |
     Removed the deprecated ``envoy.reloadable_features.validate_detailed_override_host_statuses``
     runtime flag and legacy code path.
+- area: grpc
+  change: |
+    Removed the deprecated ``envoy.reloadable_features.service_sanitize_non_utf8_strings``
+    runtime flag and legacy code path.
 
 new_features:
+- area: jwt
+  change: |
+    The jwt filter can now serialize non-primitive custom claims when maping claims to headers.
+    These claims will be serialized as JSON and encoded as Base64.
 - area: tracing
   change: |
     Added support for configuring resource detectors on the OpenTelemetry tracer.

docs/root/configuration/http/http_filters/jwt_authn_filter.rst

@@ -274,10 +274,13 @@ The field :ref:`claim_to_headers <envoy_v3_api_field_extensions.filters.http.jwt
         claim_name: sub
       - header_name: x-jwt-claim-nested-key
         claim_name: nested.claim.key
+      - header_name: x-jwt-tenants
+        claim_name: tenants
 
-JWT claim ("sub" and "nested.claim.key") will be added to HTTP headers as following format:
+In this example the `tenants` claim is an object, therefore the JWT claim ("sub", "nested.claim.key" and "tenants") will be added to HTTP headers as following format:
 
 .. code-block::
 
     x-jwt-claim-sub: <JWT Claim>
     x-jwt-claim-nested-key: <JWT Claim>
+    x-jwt-tenants: <Base64 encoded JSON JWT Claim>

docs/root/intro/arch_overview/http/http3.rst

@@ -8,8 +8,8 @@ HTTP/3 overview
   While HTTP/3 **downstream support is deemed ready for production use**, improvements are ongoing,
   tracked in the `area-quic <https://github.com/envoyproxy/envoy/labels/area%2Fquic>`_ tag.
 
-  HTTP/3 **upstream support is fine for locally controlled networks**, but is not ready for
-  general internet use, and is missing some key latency features. See details below.
+  HTTP/3 **upstream support is fine for locally controlled networks**, but is alpha for
+  general internet use - key features are implemented but have not been tested at scale.
 
 .. _arch_overview_http3_downstream:
 

mobile/bazel/pom_template.xml

@@ -5,11 +5,12 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>io.envoyproxy.envoymobile</groupId>
-    <artifactId>pom_artifact_id</artifactId>
+    <artifactId>{pom_artifact_id}</artifactId>
     <version>{pom_version}</version>
     <packaging>aar</packaging>
     <dependencies>
         {generated_bzl_deps}
+        {pom_extra_dependencies}
     </dependencies>
 
     <name>Envoy Mobile</name>

mobile/ci/sonatype_nexus_upload.py

@@ -21,7 +21,6 @@
 
 _ARTIFACT_HOST_URL = "https://oss.sonatype.org/service/local/staging"
 _GROUP_ID = "io.envoyproxy.envoymobile"
-_ARTIFACT_ID = "envoy"
 _LOCAL_INSTALL_PATH = os.path.expanduser(
     "~/.m2/repository/{directory}/envoy".format(directory=_GROUP_ID.replace(".", "/")))
 
@@ -52,7 +51,7 @@ def _resolve_name(file):
             return "", extension
 
 
-def _install_locally(version, files):
+def _install_locally(artifact_id, version, files):
     path = "{}/{}".format(_LOCAL_INSTALL_PATH, version)
 
     if os.path.exists(path):
@@ -63,7 +62,7 @@ def _install_locally(version, files):
     for file in files:
         suffix, file_extension = _resolve_name(file)
         basename = "{name}-{version}{suffix}.{extension}".format(
-            name=_ARTIFACT_ID, version=version, suffix=suffix, extension=file_extension)
+            name=artifact_id, version=version, suffix=suffix, extension=file_extension)
 
         shutil.copyfile(file, os.path.join(path, basename))
         print("{file_name}\n{sha}\n".format(file_name=file, sha=_sha256(file)))
@@ -116,7 +115,7 @@ def _create_staging_repository(profile_id):
         raise e
 
 
-def _upload_files(staging_id, version, files, ascs, sha256):
+def _upload_files(staging_id, artifact_id, version, files, ascs, sha256):
     uploaded_file_count = 0
 
     # aggregate all the files for uploading
@@ -126,11 +125,11 @@ def _upload_files(staging_id, version, files, ascs, sha256):
         print("Uploading file {}".format(file))
         suffix, file_extension = _resolve_name(file)
         basename = "{name}-{version}{suffix}.{extension}".format(
-            name=_ARTIFACT_ID, version=version, suffix=suffix, extension=file_extension)
+            name=artifact_id, version=version, suffix=suffix, extension=file_extension)
 
         artifact_url = os.path.join(
             _ARTIFACT_HOST_URL, "deployByRepositoryId/{}".format(staging_id),
-            _GROUP_ID.replace('.', "/"), _ARTIFACT_ID, version, basename)
+            _GROUP_ID.replace('.', "/"), artifact_id, version, basename)
 
         try:
             with open(file, "rb") as f:
@@ -233,6 +232,12 @@ def _build_parser():
                         curl -u {usr}:{psswrd} -H "Accept: application/json"
                         https://oss.sonatype.org//nexus/service/local/staging/profile_repositories
                         """)
+    parser.add_argument(
+        "--artifact_id",
+        required=True,
+        help="""
+                        The artifact ID to be published.
+                        """)
     parser.add_argument(
         "--version",
         default="LOCAL-SNAPSHOT",
@@ -285,7 +290,7 @@ def _build_parser():
 
     version = args.version
     if args.local:
-        _install_locally(version, args.files)
+        _install_locally(args.artifact_id, version, args.files)
     else:
         staging_id = ""
 
@@ -301,7 +306,7 @@ def _build_parser():
             print("Uploading files...")
             sha256_files = _create_sha256_files(args.files)
             uploaded_file_count = _upload_files(
-                staging_id, version, args.files, args.signed_files, sha256_files)
+                staging_id, args.artifact_id, version, args.files, args.signed_files, sha256_files)
             if uploaded_file_count > 0:
                 print("Uploading files complete!")
                 print("Closing staging repository...")

mobile/library/cc/engine_builder.cc

@@ -877,10 +877,10 @@ std::unique_ptr<envoy::config::bootstrap::v3::Bootstrap> EngineBuilder::generate
     node->mutable_locality()->set_zone(node_locality_->zone);
     node->mutable_locality()->set_sub_zone(node_locality_->sub_zone);
   }
-  ProtobufWkt::Struct& metadata = *node->mutable_metadata();
   if (node_metadata_.has_value()) {
     *node->mutable_metadata() = *node_metadata_;
   }
+  ProtobufWkt::Struct& metadata = *node->mutable_metadata();
   (*metadata.mutable_fields())["app_id"].set_string_value(app_id_);
   (*metadata.mutable_fields())["app_version"].set_string_value(app_version_);
   (*metadata.mutable_fields())["device_os"].set_string_value(device_os_);

mobile/library/common/jni/jni_utility.cc

@@ -388,7 +388,8 @@ std::vector<MatcherData> javaObjectArrayToMatcherData(JNIEnv* env, jobjectArray
 void javaByteArrayToProto(JNIEnv* env, jbyteArray source, Envoy::Protobuf::MessageLite* dest) {
   jbyte* bytes = env->GetByteArrayElements(source, /* isCopy= */ nullptr);
   jsize size = env->GetArrayLength(source);
-  ASSERT(dest->ParseFromArray(bytes, size));
+  bool success = dest->ParseFromArray(bytes, size);
+  RELEASE_ASSERT(success, "Failed to parse protobuf message.");
   env->ReleaseByteArrayElements(source, bytes, 0);
 }
 

mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD

@@ -17,7 +17,8 @@ android_artifacts(
     }),
     proguard_rules = "//library:proguard_rules",
     substitutions = {
-        "pom_artifact_id": "envoy",
+        "{pom_artifact_id}": "envoy",
+        "{pom_extra_dependencies}": "",
     },
     visibility = ["//visibility:public"],
 )
@@ -33,7 +34,13 @@ android_artifacts(
     }),
     proguard_rules = "//library:proguard_rules",
     substitutions = {
-        "pom_artifact_id": "envoy-xds",
+        "{pom_artifact_id}": "envoy-xds",
+        "{pom_extra_dependencies}": """
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-javalite</artifactId>
+            <version>3.24.4</version>
+        </dependency>""",
     },
     visibility = ["//visibility:public"],
 )

mobile/test/common/integration/BUILD

@@ -188,6 +188,7 @@ envoy_cc_test_library(
     repository = "@envoy",
     deps = [
         ":base_client_integration_test_lib",
+        "@envoy//source/common/event:libevent_lib",
         "@envoy//source/common/grpc:google_grpc_creds_lib",
         "@envoy//source/exe:process_wide_lib",
         "@envoy//test/integration:autonomous_upstream_lib",

mobile/test/common/integration/xds_test_server.cc

@@ -2,6 +2,7 @@
 
 #include <utility>
 
+#include "source/common/event/libevent.h"
 #include "source/common/grpc/google_grpc_creds_impl.h"
 #include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h"
 #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h"
@@ -17,10 +18,14 @@ namespace Envoy {
 XdsTestServer::XdsTestServer()
     : api_(Api::createApiForTest(stats_store_, time_system_)),
       version_(Network::Address::IpVersion::v4),
-      mock_buffer_factory_(new NiceMock<MockBufferFactory>),
-      dispatcher_(api_->allocateDispatcher("test_thread",
-                                           Buffer::WatermarkFactoryPtr{mock_buffer_factory_})),
-      upstream_config_(time_system_) {
+      mock_buffer_factory_(new NiceMock<MockBufferFactory>), upstream_config_(time_system_) {
+  if (!Envoy::Event::Libevent::Global::initialized()) {
+    // Required by the Dispatcher.
+    Envoy::Event::Libevent::Global::initialize();
+  }
+  dispatcher_ =
+      api_->allocateDispatcher("test_thread", Buffer::WatermarkFactoryPtr{mock_buffer_factory_});
+
   ON_CALL(*mock_buffer_factory_, createBuffer_(_, _, _))
       .WillByDefault(Invoke([](std::function<void()> below_low, std::function<void()> above_high,
                                std::function<void()> above_overflow) -> Buffer::Instance* {

mobile/test/common/jni/test_jni_interface.cc

@@ -73,6 +73,8 @@ Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeSendDiscoveryRespons
                                                                                   jstring yaml) {
   jni_log("[XTS]", "sending DiscoveryResponse from the xDS server");
   const char* yaml_chars = env->GetStringUTFChars(yaml, /* isCopy= */ nullptr);
+  // The yaml utilities have non-relevant thread asserts.
+  Envoy::Thread::SkipAsserts skip;
   envoy::service::discovery::v3::DiscoveryResponse response;
   Envoy::TestUtility::loadFromYaml(yaml_chars, response);
   sendDiscoveryResponse(response);

mobile/test/kotlin/integration/XdsTest.kt

@@ -28,7 +28,6 @@ class XdsTest {
 
   @Before
   fun setUp() {
-    TestJni.startHttp2TestServer()
     TestJni.initXdsTestServer()
     val latch = CountDownLatch(1)
     engine =
@@ -50,7 +49,6 @@ class XdsTest {
   @After
   fun tearDown() {
     engine.terminate()
-    TestJni.shutdownTestServer()
     TestJni.shutdownXdsTestServer()
   }
 
@@ -66,15 +64,6 @@ class XdsTest {
         name: my_cluster
         type: STATIC
         connect_timeout: 5s
-        load_assignment:
-          cluster_name: xds_cluster
-          endpoints:
-            - lb_endpoints:
-                - endpoint:
-                    address:
-                      socket_address:
-                        address: ${TestJni.getServerHost()}
-                        port_value: ${TestJni.getServerPort()}
         typed_extension_protocol_options:
           envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
             "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
@@ -86,7 +75,7 @@ class XdsTest {
     """
         .trimIndent()
     TestJni.sendDiscoveryResponse(cdsResponse)
-    // There are now 3 clusters: base, base_cluster, and xds_cluster.
+    // There are now 3 clusters: base, base_cluster, and my_cluster.
     engine.waitForStatGe("cluster_manager.cluster_added", 3)
   }
 }