Initial commit

.gitignore

@@ -0,0 +1,4 @@
+.idea
+.gradle
+*.iml
+build

README.md

@@ -0,0 +1,31 @@
+Writer's Reader Discarded
+=========================
+
+This is a project that aims to reproduce linkerd's `This writer's reader has been discarded` HTTP 502 error.
+
+`client` is a Java client that makes a request to `server` via linkerd, in a linker-to-linker configuration. It needs to be built with:
+```
+cd client
+gradle build
+```
+
+`server` is a Go server that simply returns HTTP 200 with body `done`.
+
+IMPORTANT: change `linkerd.yml:23` and `disco/server:1` to use the IP address of the machine the project is running on.
+
+Once the client is built, simply run `docker-compose build` and then `docker-compose up`.
+
+Request flow:
+ 1. `client` initiates request to `http://linkerd:4141/server`
+ 2. `linkerd` is found via the Docker network (`default`), by container name
+ 3. The outgoing linkerd router listening on 4141 looks up `server` in the namer (`io.l5d.fs`)
+ 4. Once the IP address and port is found, the port is tranformed to 4041 via a `io.l5d.port` transformer
+ 5. The request is sent to `http://$ipAddress:4041`, passing `l5d-dst-service: /in/out/server` HTTP header
+ 6. The incoming linkerd router listening on 4041 looks up `server` in the namer (`io.l5d.fs`)
+ 7. Any returned host returned by the namer that is not the one specified in the `io.l5d.specificHost` transformer is excluded
+ 8. The request is sent to `http://$ipAddress:8000`, which is where the server is listening on
+ 9. The server immediately returns HTTP 200 with response body `done`
+ 10. Once the client receives the response, start agin from point 1. until the `This writer's reader has been discarded` HTTP 502 error occurs.
+
+The first failing request is usually within the first 30 seconds.
+

client/Dockerfile

@@ -0,0 +1,5 @@
+FROM openjdk:8
+
+ADD build/libs/writersreaderdiscarded.jar /app.jar
+
+ENTRYPOINT ["/usr/bin/java", "-jar", "/app.jar", "http://linkerd:4141/server"]

client/build.gradle

@@ -0,0 +1,30 @@
+group 'writersreaderdiscarded'
+version '0.1.0-SNAPSHOT'
+
+apply plugin: 'java'
+
+jar.archiveName 'writersreaderdiscarded.jar'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  compile(
+      "org.slf4j:slf4j-api:1.7.25",
+      "ch.qos.logback:logback-classic:1.2.3",
+      "org.glassfish.jersey.core:jersey-client:2.26",
+      "org.glassfish.jersey.connectors:jersey-apache-connector:2.26",
+      "org.glassfish.jersey.inject:jersey-hk2:2.26"
+  )
+}
+
+jar {
+  manifest {
+    attributes 'Main-Class': 'writersreaderdiscarded.Application'
+  }
+
+  from {
+    configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
+  }
+}

client/src/main/java/writersreaderdiscarded/Application.java

@@ -0,0 +1,39 @@
+package writersreaderdiscarded;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import org.glassfish.jersey.apache.connector.ApacheClientProperties;
+import org.glassfish.jersey.apache.connector.ApacheConnectorProvider;
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.client.ClientProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Application {
+
+  private static final Logger logger = LoggerFactory.getLogger(Application.class);
+
+  public static void main(String[] args) throws InterruptedException {
+    logger.info("Application started. Base URL '{}'", args[0]);
+    WritersReaderDiscardedClient writersReaderDiscardedClient = new WritersReaderDiscardedClient(jerseyClient(), args[0]);
+
+    logger.info("Waiting 10 seconds");
+    Thread.sleep(10000);
+
+    while (true) {
+      writersReaderDiscardedClient.count();
+    }
+  }
+
+  private static Client jerseyClient() {
+    final ClientConfig clientConfig = new ClientConfig();
+    clientConfig.connectorProvider(new ApacheConnectorProvider());
+    clientConfig.property(ClientProperties.FOLLOW_REDIRECTS, true);
+    clientConfig.property(ClientProperties.CONNECT_TIMEOUT, 500); // This should be really short as it's only for the connection to linkerd
+    clientConfig.property(ClientProperties.READ_TIMEOUT, 0); // Set to infinite as linkerd will handle timeouts
+    clientConfig.property(ClientProperties.JSON_PROCESSING_FEATURE_DISABLE, true);
+    clientConfig.property(ClientProperties.MOXY_JSON_FEATURE_DISABLE, true);
+    clientConfig.property(ApacheClientProperties.DISABLE_COOKIES, true);
+    return ClientBuilder.newClient(clientConfig);
+  }
+}

client/src/main/java/writersreaderdiscarded/WritersReaderDiscardedClient.java

@@ -0,0 +1,35 @@
+package writersreaderdiscarded;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WritersReaderDiscardedClient {
+
+  private static final Logger logger = LoggerFactory.getLogger(WritersReaderDiscardedClient.class);
+
+  private final Client client;
+  private final String baseUrl;
+
+  public WritersReaderDiscardedClient(final Client client, final String baseUrl) {
+    this.client = client;
+    this.baseUrl = baseUrl;
+  }
+
+  void count() {
+    logger.info("Request to '{}'", baseUrl);
+    final Response response = client.target(baseUrl).request(MediaType.APPLICATION_JSON_TYPE).post(null);
+
+    final String raw = response.readEntity(String.class);
+    logger.info("Response status: {}", response.getStatus());
+    logger.info("Response body: '{}'", raw);
+    response.close();
+
+    if (response.getStatus()  != 200) {
+      String message = String.format("Got HTTP status code %d. Body was '%s'", response.getStatus(), raw);
+      throw new RuntimeException(message);
+    }
+  }
+}

disco/server

@@ -0,0 +1 @@
+10.64.2.206 8000

docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3.3'
+
+services:
+  client:
+    container_name: client
+    build:
+      context: ./client
+      dockerfile: ./Dockerfile
+    networks:
+      - default
+  server:
+    container_name: server
+    build:
+      context: ./server
+      dockerfile: ./Dockerfile
+    ports:
+      - 8000:8000
+    networks:
+      - default
+  linkerd:
+    container_name: linkerd
+    image: buoyantio/linkerd:1.2.1
+    ports:
+      - 9990:9990
+      - 4041:4041
+      - 4140:4140
+      - 4141:4141
+    networks:
+      - default
+    volumes:
+      - ./linkerd.yml:/io/buoyant/linkerd/config.yml:ro
+      - ./disco:/disco
+    command:
+      - "/io/buoyant/linkerd/config.yml"
+networks:
+  default:
+    external: false

linkerd.yml

@@ -0,0 +1,43 @@
+admin:
+  ip: 0.0.0.0
+  port: 9990
+
+namers:
+- kind: io.l5d.fs
+  rootDir: /disco
+
+routers:
+- protocol: http
+  label: http-internal-incoming
+  dstPrefix: /in
+  servers:
+  - ip: 0.0.0.0
+    port: 4041
+  identifier:
+    kind: io.l5d.header
+    header: l5d-dst-service
+  interpreter:
+    kind: default
+    transformers:
+    - kind: io.l5d.specificHost
+      host: 10.64.2.206
+  dtab: |
+    /in/out  => /#/io.l5d.fs;
+
+- protocol: http
+  label: http-internal-outgoing
+  dstPrefix: /out
+  servers:
+  - ip: 0.0.0.0
+    port: 4141
+  identifier:
+    kind: io.l5d.path
+    segments: 1
+    consume: true
+  interpreter:
+    kind: default
+    transformers:
+    - kind: io.l5d.port
+      port: 4041
+  dtab: |
+    /out     => /#/io.l5d.fs;

server/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:onbuild
+
+EXPOSE 8000
+
+ENTRYPOINT ["app"]

server/server.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+)
+
+func main() {
+	addr := flag.String("addr", ":8000", "service port to run on")
+	flag.Parse()
+
+	fmt.Printf("serving on %s\n", *addr)
+	http.HandleFunc("/", handleRequest)
+	http.ListenAndServe(*addr, nil)
+}
+
+func handleRequest(w http.ResponseWriter, req *http.Request) {
+	w.Write([]byte("done"))
+}