From 075af9991f455c6b93686bac87f80971083e4460 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 20 Mar 2018 11:32:01 +0800 Subject: [PATCH] Makes it possible to implement a protobuf3 span encoder While we haven't yet defined a proto3 format, this allows one to be defined. This also allows us to refine how stackdriver (or any future translated format) works in zipkin reporter. Implementation is relatively simple: this assumes a buffer of spans is analogous to a proto3 repeated field, and that the field name itself is ordinal <=127. Fixes #1942 --- .../src/main/java/zipkin2/codec/Encoding.java | 41 ++++++++++++++- .../test/java/zipkin2/codec/EncodingTest.java | 51 ++++++++++++++++--- 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/zipkin2/src/main/java/zipkin2/codec/Encoding.java b/zipkin2/src/main/java/zipkin2/codec/Encoding.java index 5f5fa41d323..f4cd8954181 100644 --- a/zipkin2/src/main/java/zipkin2/codec/Encoding.java +++ b/zipkin2/src/main/java/zipkin2/codec/Encoding.java @@ -1,5 +1,5 @@ /** - * Copyright 2015-2017 The OpenZipkin Authors + * Copyright 2015-2018 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -31,6 +31,45 @@ public enum Encoding { } return sizeInBytes; } + }, + /** + * Repeated (type 2) fields are length-prefixed, the value is a concatenation with no additional + * overhead. + * + *

See https://developers.google.com/protocol-buffers/docs/encoding#optional + */ + PROTO3 { + /** Returns the size of a length-prefixed field in a protobuf message */ + @Override public int listSizeInBytes(int encodedSizeInBytes) { + return 1 // assumes field number <= 127 + + unsignedVarint(encodedSizeInBytes) // bytes to encode the length + + encodedSizeInBytes; // the actual length + } + + /** Returns a concatenation of length-prefixed size for each value */ + @Override public int listSizeInBytes(List values) { + int sizeInBytes = 0; + for (int i = 0, length = values.size(); i < length; ) { + sizeInBytes += listSizeInBytes(values.get(i++).length); + } + return sizeInBytes; + } + + /** + * A base 128 varint encodes 7 bits at a time, this checks how many bytes are needed to + * represent the value. + * + *

See https://developers.google.com/protocol-buffers/docs/encoding#varints + * + *

This logic is the same as {@code com.squareup.wire.WireOutput.varint32Size} v2.3.0 + */ + int unsignedVarint(int value) { + if ((value & (0xffffffff << 7)) == 0) return 1; + if ((value & (0xffffffff << 14)) == 0) return 2; + if ((value & (0xffffffff << 21)) == 0) return 3; + if ((value & (0xffffffff << 28)) == 0) return 4; + return 5; + } }; /** Like {@link #listSizeInBytes(List)}, except for a single element. */ diff --git a/zipkin2/src/test/java/zipkin2/codec/EncodingTest.java b/zipkin2/src/test/java/zipkin2/codec/EncodingTest.java index 9c231989340..e221f40aa57 100644 --- a/zipkin2/src/test/java/zipkin2/codec/EncodingTest.java +++ b/zipkin2/src/test/java/zipkin2/codec/EncodingTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2015-2017 The OpenZipkin Authors + * Copyright 2015-2018 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -13,7 +13,6 @@ */ package zipkin2.codec; -import java.io.IOException; import java.util.Arrays; import java.util.List; import org.junit.Test; @@ -22,13 +21,13 @@ public class EncodingTest { - @Test public void emptyList_json() throws IOException { + @Test public void emptyList_json() { List encoded = Arrays.asList(); assertThat(Encoding.JSON.listSizeInBytes(encoded)) .isEqualTo(2 /* [] */); } - @Test public void singletonList_json() throws IOException { + @Test public void singletonList_json() { List encoded = Arrays.asList(new byte[10]); assertThat(Encoding.JSON.listSizeInBytes(encoded.get(0).length)) @@ -37,10 +36,46 @@ public class EncodingTest { .isEqualTo(2 /* [] */ + 10); } - - @Test public void multiItemList_json() throws IOException { - List encoded = Arrays.asList(new byte[3], new byte[4], new byte[5]); + @Test public void multiItemList_json() { + List encoded = Arrays.asList(new byte[3], new byte[4], new byte[128]); assertThat(Encoding.JSON.listSizeInBytes(encoded)) - .isEqualTo(2 /* [] */ + 3 + 1 /* , */ + 4 + 1 /* , */ + 5); + .isEqualTo(2 /* [] */ + 3 + 1 /* , */ + 4 + 1 /* , */ + 128); + } + + @Test public void emptyList_proto3() { + List encoded = Arrays.asList(); + assertThat(Encoding.PROTO3.listSizeInBytes(encoded)) + .isEqualTo(0); + } + + @Test public void singletonList_proto3() { + List encoded = Arrays.asList(new byte[10]); + + assertThat(Encoding.PROTO3.listSizeInBytes(encoded.get(0).length)) + .isEqualTo(1 + 1 /* tag, length */ + 10); + assertThat(Encoding.PROTO3.listSizeInBytes(encoded)) + .isEqualTo(1 + 1 /* tag, length */ + 10); + } + + @Test public void multiItemList_proto3() { + List encoded = Arrays.asList(new byte[3], new byte[4], new byte[128]); + assertThat(Encoding.PROTO3.listSizeInBytes(encoded)) + .isEqualTo(0 + + (1 + 1 /* tag, length */ + 3) + + (1 + 1 /* tag, length */ + 4) + + (1 + 2 /* tag, length */ + 128) + ); + } + + @Test public void singletonList_proto3_big() { + // Not good to have a 5MiB span, but lets test the length prefix + int bigSpan = 5 * 1024 * 1024; + assertThat(Encoding.PROTO3.listSizeInBytes(bigSpan)) + .isEqualTo(1 + 4 /* tag, length */ + bigSpan); + + // Terrible value in real life as this would be a 536 meg span! + int twentyNineBitNumber = 536870911; + assertThat(Encoding.PROTO3.listSizeInBytes(twentyNineBitNumber)) + .isEqualTo(1 + 5 /* tag, length */ + twentyNineBitNumber); } }