From f0ec1baf58422f443a32c7374c1fce6109705547 Mon Sep 17 00:00:00 2001 From: "yu-keun, OH" <37852701+eottabom@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:46:51 +0900 Subject: [PATCH] Add gRPC reverse proxy server example (#5722) Motivation: Add gRPC reverse proxy server example Modifications: Add gRPC reverse proxy server example with testcontainers. Result: - Closes #2353 . Co-authored-by: jrhee17 --- examples/grpc-envoy/build.gradle | 19 ++++++ .../armeria/grpc/envoy/EnvoyContainer.java | 62 +++++++++++++++++++ .../armeria/grpc/envoy/HelloService.java | 29 +++++++++ .../java/example/armeria/grpc/envoy/Main.java | 52 ++++++++++++++++ .../grpc-envoy/src/main/proto/hello.proto | 19 ++++++ .../grpc-envoy/src/main/resources/.gitkeep | 0 .../src/main/resources/envoy/envoy.yaml | 52 ++++++++++++++++ .../src/main/resources/envoy/launch_envoy.sh | 16 +++++ .../grpc/envoy/GrpcEnvoyProxyTest.java | 54 ++++++++++++++++ settings.gradle | 1 + 10 files changed, 304 insertions(+) create mode 100644 examples/grpc-envoy/build.gradle create mode 100644 examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/EnvoyContainer.java create mode 100644 examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/HelloService.java create mode 100644 examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/Main.java create mode 100644 examples/grpc-envoy/src/main/proto/hello.proto create mode 100644 examples/grpc-envoy/src/main/resources/.gitkeep create mode 100644 examples/grpc-envoy/src/main/resources/envoy/envoy.yaml create mode 100755 examples/grpc-envoy/src/main/resources/envoy/launch_envoy.sh create mode 100644 examples/grpc-envoy/src/test/java/example/armeria/grpc/envoy/GrpcEnvoyProxyTest.java diff --git a/examples/grpc-envoy/build.gradle b/examples/grpc-envoy/build.gradle new file mode 100644 index 00000000000..1203c0986af --- /dev/null +++ b/examples/grpc-envoy/build.gradle @@ -0,0 +1,19 @@ +plugins { + id 'application' +} + +dependencies { + implementation project(':core') + implementation project(':grpc') + implementation libs.testcontainers.junit.jupiter + compileOnly libs.javax.annotation + runtimeOnly libs.slf4j.simple + + testImplementation project(':junit5') + testImplementation libs.assertj + testImplementation libs.junit5.jupiter.api +} + +application { + mainClass.set('example.armeria.grpc.envoy.Main') +} diff --git a/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/EnvoyContainer.java b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/EnvoyContainer.java new file mode 100644 index 00000000000..d62d55767fb --- /dev/null +++ b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/EnvoyContainer.java @@ -0,0 +1,62 @@ +package example.armeria.grpc.envoy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +import com.github.dockerjava.api.command.InspectContainerResponse; + +import com.linecorp.armeria.common.annotation.Nullable; + +// https://github.com/envoyproxy/java-control-plane/blob/eaca1a4380e53b4b6339db4e9ffe0ada5e0b7f8f/server/src/test/java/io/envoyproxy/controlplane/server/EnvoyContainer.java +class EnvoyContainer extends GenericContainer { + + private static final Logger LOGGER = LoggerFactory.getLogger(EnvoyContainer.class); + + private static final String CONFIG_DEST = "/etc/envoy/envoy.yaml"; + private static final String LAUNCH_ENVOY_SCRIPT = "envoy/launch_envoy.sh"; + private static final String LAUNCH_ENVOY_SCRIPT_DEST = "/usr/local/bin/launch_envoy.sh"; + + static final int ADMIN_PORT = 9901; + + private final String config; + @Nullable + private final String sedCommand; + + /** + * A {@link GenericContainer} implementation for envoy containers. + * + * @param sedCommand optional sed command which may be used to postprocess the provided {@param config}. + * This parameter will be fed into the command {@code sed -e }. + * An example command may be {@code "s/foo/bar/g;s/abc/def/g"}. + */ + EnvoyContainer(String config, @Nullable String sedCommand) { + super("envoyproxy/envoy:v1.30.1"); + this.config = config; + this.sedCommand = sedCommand; + } + + @Override + protected void configure() { + super.configure(); + + withClasspathResourceMapping(LAUNCH_ENVOY_SCRIPT, LAUNCH_ENVOY_SCRIPT_DEST, BindMode.READ_ONLY); + withClasspathResourceMapping(config, CONFIG_DEST, BindMode.READ_ONLY); + + if (sedCommand != null) { + withCommand("/bin/bash", "/usr/local/bin/launch_envoy.sh", + sedCommand, CONFIG_DEST, "-l", "debug"); + } + + addExposedPort(ADMIN_PORT); + } + + @Override + protected void containerIsStarting(InspectContainerResponse containerInfo) { + followOutput(new Slf4jLogConsumer(LOGGER).withPrefix("ENVOY")); + + super.containerIsStarting(containerInfo); + } +} diff --git a/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/HelloService.java b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/HelloService.java new file mode 100644 index 00000000000..9622c5e6d90 --- /dev/null +++ b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/HelloService.java @@ -0,0 +1,29 @@ +package example.armeria.grpc.envoy; + +import example.armeria.grpc.envoy.Hello.HelloReply; +import example.armeria.grpc.envoy.Hello.HelloRequest; +import example.armeria.grpc.envoy.HelloServiceGrpc.HelloServiceImplBase; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +public class HelloService extends HelloServiceImplBase { + + @Override + public void hello(HelloRequest request, StreamObserver responseObserver) { + if (request.getName().isEmpty()) { + responseObserver.onError( + Status.FAILED_PRECONDITION.withDescription("Name cannot be empty").asRuntimeException()); + } else { + responseObserver.onNext(buildReply(toMessage(request.getName()))); + responseObserver.onCompleted(); + } + } + + static String toMessage(String name) { + return "Hello, " + name + '!'; + } + + private static HelloReply buildReply(Object message) { + return HelloReply.newBuilder().setMessage(String.valueOf(message)).build(); + } +} diff --git a/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/Main.java b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/Main.java new file mode 100644 index 00000000000..c8d1f142475 --- /dev/null +++ b/examples/grpc-envoy/src/main/java/example/armeria/grpc/envoy/Main.java @@ -0,0 +1,52 @@ +package example.armeria.grpc.envoy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; + +import com.linecorp.armeria.common.util.ShutdownHooks; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.grpc.GrpcService; + +public final class Main { + + private static final Logger logger = LoggerFactory.getLogger(Main.class); + + private static final int serverPort = 8080; + // the port envoy binds to within the container + private static final int envoyPort = 10000; + + public static void main(String[] args) { + if (!DockerClientFactory.instance().isDockerAvailable()) { + throw new IllegalStateException("Docker is not available"); + } + + final Server backendServer = startBackendServer(serverPort); + backendServer.closeOnJvmShutdown(); + backendServer.start().join(); + logger.info("Serving backend at http://127.0.0.1:{}/", backendServer.activePort()); + + final EnvoyContainer envoyProxy = configureEnvoy(serverPort, envoyPort); + ShutdownHooks.addClosingTask(envoyProxy::stop); + envoyProxy.start(); + final Integer mappedEnvoyPort = envoyProxy.getMappedPort(envoyPort); + logger.info("Serving envoy at http://127.0.0.1:{}/", mappedEnvoyPort); + } + + private static Server startBackendServer(int serverPort) { + return Server.builder() + .http(serverPort) + .service(GrpcService.builder() + .addService(new HelloService()) + .build()) + .build(); + } + + static EnvoyContainer configureEnvoy(int serverPort, int envoyPort) { + final String sedPattern = String.format("s/SERVER_PORT/%s/g;s/ENVOY_PORT/%s/g", serverPort, envoyPort); + return new EnvoyContainer("envoy/envoy.yaml", sedPattern) + .withExposedPorts(envoyPort); + } + + private Main() {} +} diff --git a/examples/grpc-envoy/src/main/proto/hello.proto b/examples/grpc-envoy/src/main/proto/hello.proto new file mode 100644 index 00000000000..f5c4ac2c9a7 --- /dev/null +++ b/examples/grpc-envoy/src/main/proto/hello.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package example.grpc.hello; +option java_package = "example.armeria.grpc.envoy"; +option java_multiple_files = false; + +import "google/api/annotations.proto"; + +service HelloService { + rpc Hello (HelloRequest) returns (HelloReply); +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/examples/grpc-envoy/src/main/resources/.gitkeep b/examples/grpc-envoy/src/main/resources/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/grpc-envoy/src/main/resources/envoy/envoy.yaml b/examples/grpc-envoy/src/main/resources/envoy/envoy.yaml new file mode 100644 index 00000000000..983a7c0f775 --- /dev/null +++ b/examples/grpc-envoy/src/main/resources/envoy/envoy.yaml @@ -0,0 +1,52 @@ +admin: + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + address: 0.0.0.0 + port_value: ENVOY_PORT + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + http_protocol_options: + enable_trailers: true + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: grpc_service + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: grpc_service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: + enable_trailers: true + load_assignment: + cluster_name: grpc_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: host.testcontainers.internal + port_value: SERVER_PORT diff --git a/examples/grpc-envoy/src/main/resources/envoy/launch_envoy.sh b/examples/grpc-envoy/src/main/resources/envoy/launch_envoy.sh new file mode 100755 index 00000000000..34581990ee6 --- /dev/null +++ b/examples/grpc-envoy/src/main/resources/envoy/launch_envoy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SED_COMMAND=$1 + +CONFIG=$(cat $2) +CONFIG_DIR=$(mktemp -d) +CONFIG_FILE="$CONFIG_DIR/envoy.yaml" + +echo "${CONFIG}" | sed -e "${SED_COMMAND}" > "${CONFIG_FILE}" + + +shift 2 +/usr/local/bin/envoy --drain-time-s 1 -c "${CONFIG_FILE}" "$@" + +rm -rf "${CONFIG_DIR}" diff --git a/examples/grpc-envoy/src/test/java/example/armeria/grpc/envoy/GrpcEnvoyProxyTest.java b/examples/grpc-envoy/src/test/java/example/armeria/grpc/envoy/GrpcEnvoyProxyTest.java new file mode 100644 index 00000000000..da800025caf --- /dev/null +++ b/examples/grpc-envoy/src/test/java/example/armeria/grpc/envoy/GrpcEnvoyProxyTest.java @@ -0,0 +1,54 @@ +package example.armeria.grpc.envoy; + +import static example.armeria.grpc.envoy.Main.configureEnvoy; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.linecorp.armeria.client.grpc.GrpcClients; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; + +import example.armeria.grpc.envoy.Hello.HelloReply; +import example.armeria.grpc.envoy.Hello.HelloRequest; + +@Testcontainers(disabledWithoutDocker = true) +class GrpcEnvoyProxyTest { + + // the port envoy binds to within the container + private static final int ENVOY_PORT = 10000; + + @RegisterExtension + static ServerExtension server = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) throws Exception { + sb.service(GrpcService.builder() + .addService(new HelloService()) + .build()); + } + }; + + @ParameterizedTest + @EnumSource(value = SessionProtocol.class, names = {"H1C", "H2C"}) + void reverseProxy(SessionProtocol sessionProtocol) { + org.testcontainers.Testcontainers.exposeHostPorts(server.httpPort()); + try (EnvoyContainer envoy = configureEnvoy(server.httpPort(), ENVOY_PORT)) { + envoy.start(); + final String uri = sessionProtocol.uriText() + "://" + envoy.getHost() + + ':' + envoy.getMappedPort(ENVOY_PORT); + final HelloServiceGrpc.HelloServiceBlockingStub helloService = + GrpcClients.builder(uri) + .build(HelloServiceGrpc.HelloServiceBlockingStub.class); + final HelloReply reply = + helloService.hello(HelloRequest.newBuilder() + .setName("Armeria") + .build()); + assertThat(reply.getMessage()).isEqualTo("Hello, Armeria!"); + } + } +} diff --git a/settings.gradle b/settings.gradle index 28a00ca3983..72a889f1cfb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -265,6 +265,7 @@ includeWithFlags ':examples:graphql-kotlin-example', 'java17', 'ko project(':examples:graphql-kotlin-example').projectDir = file('examples/graphql-kotlin') includeWithFlags ':examples:graphql-sangria-example', 'java11', 'scala_2.13' project(':examples:graphql-sangria-example').projectDir = file('examples/graphql-sangria') +includeWithFlags ':examples:grpc-envoy', 'java11' includeWithFlags ':examples:grpc-example', 'java11' project(':examples:grpc-example').projectDir = file('examples/grpc') includeWithFlags ':examples:grpc-kotlin', 'java11', 'kotlin-grpc', 'kotlin'