getOperationLatencyViewTagValueStrings() {
+ return Stats.getViewManager().getView(BuiltinViewConstants.OPERATION_LATENCIES_VIEW.getName())
+ .getAggregationMap().entrySet().stream()
+ .map(Map.Entry::getKey)
+ .flatMap(x -> x.stream())
+ .map(x -> x.asString())
+ .collect(Collectors.toCollection(ArrayList::new));
+ }
}
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java
index 6a902029eb..6f636bf55d 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java
@@ -21,25 +21,26 @@
import com.google.api.gax.rpc.ResponseObserver;
import com.google.api.gax.rpc.ServerStreamingCallable;
import com.google.api.gax.rpc.StreamController;
+import com.google.bigtable.v2.ResponseParams;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
+import com.google.protobuf.InvalidProtocolBufferException;
import io.grpc.Metadata;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
/**
* This callable will
- *
- * -inject a {@link GrpcResponseMetadata} to access the headers and trailers returned by gRPC
- * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in
- * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()}
- * returned null, it probably means that the request has never reached GFE, and it'll increment the
- * gfe_header_missing_counter in this case.
- *
- *
-Call {@link BigtableTracer#onRequest()} to record the request events in a stream.
- *
- *
This class is considered an internal implementation detail and not meant to be used by
- * applications.
+ *
-Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC methods upon
+ * completion. The {@link BigtableTracer} will process metrics that were injected in the
+ * header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()}
+ * returned null, it probably means that the request has never reached GFE, and it'll increment
+ * the gfe_header_missing_counter in this case.
+ * -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and
+ * cluster ids.
+ * -Call {@link BigtableTracer#onRequest(int)} to record the request events in a stream.
+ * This class is considered an internal implementation detail and not meant to be used by
+ * applications.
*/
@InternalApi
public class BigtableTracerStreamingCallable
@@ -102,6 +103,14 @@ public void onError(Throwable t) {
Metadata metadata = responseMetadata.getMetadata();
Long latency = Util.getGfeLatency(metadata);
tracer.recordGfeMetadata(latency, t);
+ try {
+ byte[] trailers =
+ metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER));
+ ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers);
+ tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId());
+ } catch (NullPointerException | InvalidProtocolBufferException e) {
+ }
+
outerObserver.onError(t);
}
@@ -110,6 +119,14 @@ public void onComplete() {
Metadata metadata = responseMetadata.getMetadata();
Long latency = Util.getGfeLatency(metadata);
tracer.recordGfeMetadata(latency, null);
+ try {
+ byte[] trailers =
+ metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER));
+ ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers);
+ tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId());
+ } catch (NullPointerException | InvalidProtocolBufferException e) {
+ }
+
outerObserver.onComplete();
}
}
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java
index de53a0c94e..0efc99fe4d 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java
@@ -22,20 +22,24 @@
import com.google.api.gax.grpc.GrpcResponseMetadata;
import com.google.api.gax.rpc.ApiCallContext;
import com.google.api.gax.rpc.UnaryCallable;
+import com.google.bigtable.v2.ResponseParams;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.protobuf.InvalidProtocolBufferException;
import io.grpc.Metadata;
import javax.annotation.Nonnull;
/**
- * This callable will inject a {@link GrpcResponseMetadata} to access the headers and trailers
- * returned by gRPC methods upon completion. The {@link BigtableTracer} will process metrics that
- * were injected in the header/trailer and publish them to OpenCensus. If {@link
- * GrpcResponseMetadata#getMetadata()} returned null, it probably means that the request has never
- * reached GFE, and it'll increment the gfe_header_missing_counter in this case.
- *
- * This class is considered an internal implementation detail and not meant to be used by
- * applications.
+ * This callable will:
+ *
- Inject a {@link GrpcResponseMetadata} to access the headers returned by gRPC methods upon
+ * completion. The {@link BigtableTracer} will process metrics that were injected in the
+ * header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()}
+ * returned null, it probably means that the request has never reached GFE, and it'll increment
+ * the gfe_header_missing_counter in this case.
+ * -This class will also access trailers from {@link GrpcResponseMetadata} to record zone and
+ * cluster ids.
+ * This class is considered an internal implementation detail and not meant to be used by
+ * applications.
*/
@InternalApi
public class BigtableTracerUnaryCallable
@@ -78,6 +82,13 @@ public void onFailure(Throwable throwable) {
Metadata metadata = responseMetadata.getMetadata();
Long latency = Util.getGfeLatency(metadata);
tracer.recordGfeMetadata(latency, throwable);
+ try {
+ byte[] trailers =
+ metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER));
+ ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers);
+ tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId());
+ } catch (NullPointerException | InvalidProtocolBufferException e) {
+ }
}
@Override
@@ -85,6 +96,13 @@ public void onSuccess(ResponseT response) {
Metadata metadata = responseMetadata.getMetadata();
Long latency = Util.getGfeLatency(metadata);
tracer.recordGfeMetadata(latency, null);
+ try {
+ byte[] trailers =
+ metadata.get(Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER));
+ ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers);
+ tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId());
+ } catch (NullPointerException | InvalidProtocolBufferException e) {
+ }
}
}
}
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java
index 0440029027..7487703fc0 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java
@@ -58,7 +58,7 @@ public class Util {
Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER);
private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?\\d+)");
- static final String TRAILER_KEY = "x-goog-ext-425905942-bin";
+ static final String RESPONSE_PRAMS_KEY = "x-goog-ext-425905942-bin";
/** Convert an exception into a value that can be used to create an OpenCensus tag value. */
static String extractStatus(@Nullable Throwable error) {
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java
new file mode 100644
index 0000000000..2f9c8ff639
--- /dev/null
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
+
+import com.google.api.core.ApiFuture;
+import com.google.api.gax.rpc.NotFoundException;
+import com.google.cloud.bigtable.admin.v2.models.Cluster;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.stats.BuiltinViews;
+import com.google.cloud.bigtable.stats.StatsWrapper;
+import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
+import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+public class StreamingMetricsMetadataIT {
+ @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+
+ @BeforeClass
+ public static void setUpClass() {
+ assume()
+ .withMessage("StreamingMetricsMetadataIT is not supported on Emulator")
+ .that(testEnvRule.env())
+ .isNotInstanceOf(EmulatorEnv.class);
+ BuiltinViews.registerBigtableBuiltinViews();
+ }
+
+ @Test
+ public void testSuccess() throws Exception {
+ String prefix = UUID.randomUUID().toString();
+ String uniqueKey = prefix + "-read";
+
+ Query query = Query.create(testEnvRule.env().getTableId()).rowKey(uniqueKey);
+ ArrayList rows = Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query));
+
+ ApiFuture> clustersFuture =
+ testEnvRule
+ .env()
+ .getInstanceAdminClient()
+ .listClustersAsync(testEnvRule.env().getInstanceId());
+
+ List clusters = clustersFuture.get(1, TimeUnit.MINUTES);
+
+ // give opencensus some time to populate view data
+ Thread.sleep(100);
+
+ List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings();
+ assertThat(tagValueStrings).contains(clusters.get(0).getZone());
+ assertThat(tagValueStrings).contains(clusters.get(0).getId());
+ }
+
+ @Test
+ public void testFailure() throws InterruptedException {
+ Query query = Query.create("non-exist-table");
+ try {
+ Lists.newArrayList(testEnvRule.env().getDataClient().readRows(query));
+ } catch (NotFoundException e) {
+ }
+
+ // give opencensus some time to populate view data
+ Thread.sleep(100);
+
+ List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings();
+ assertThat(tagValueStrings).contains("undefined");
+ assertThat(tagValueStrings).contains("undefined");
+ }
+}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java
new file mode 100644
index 0000000000..9fd132ed89
--- /dev/null
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
+
+import com.google.api.core.ApiFuture;
+import com.google.api.gax.rpc.NotFoundException;
+import com.google.cloud.bigtable.admin.v2.models.Cluster;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.cloud.bigtable.stats.BuiltinViews;
+import com.google.cloud.bigtable.stats.StatsWrapper;
+import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
+import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+public class UnaryMetricsMetadataIT {
+ @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+
+ @BeforeClass
+ public static void setUpClass() {
+ assume()
+ .withMessage("UnaryMetricsMetadataIT is not supported on Emulator")
+ .that(testEnvRule.env())
+ .isNotInstanceOf(EmulatorEnv.class);
+ BuiltinViews.registerBigtableBuiltinViews();
+ }
+
+ @Test
+ public void testSuccess() throws Exception {
+ String rowKey = UUID.randomUUID().toString();
+ String familyId = testEnvRule.env().getFamilyId();
+
+ ApiFuture future =
+ testEnvRule
+ .env()
+ .getDataClient()
+ .mutateRowCallable()
+ .futureCall(
+ RowMutation.create(testEnvRule.env().getTableId(), rowKey)
+ .setCell(familyId, "q", "myVal"));
+
+ future.get(1, TimeUnit.MINUTES);
+
+ ApiFuture> clustersFuture =
+ testEnvRule
+ .env()
+ .getInstanceAdminClient()
+ .listClustersAsync(testEnvRule.env().getInstanceId());
+ List clusters = clustersFuture.get(1, TimeUnit.MINUTES);
+
+ // give opencensus some time to populate view data
+ Thread.sleep(100);
+
+ List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings();
+ assertThat(tagValueStrings).contains(clusters.get(0).getZone());
+ assertThat(tagValueStrings).contains(clusters.get(0).getId());
+ }
+
+ @Test
+ public void testFailure() throws InterruptedException {
+ String rowKey = UUID.randomUUID().toString();
+ String familyId = testEnvRule.env().getFamilyId();
+
+ try {
+ testEnvRule
+ .env()
+ .getDataClient()
+ .mutateRowCallable()
+ .call(RowMutation.create("non-exist-table", rowKey).setCell(familyId, "q", "myVal"));
+ } catch (NotFoundException e) {
+ }
+
+ // give opencensus some time to populate view data
+ Thread.sleep(100);
+
+ List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings();
+ assertThat(tagValueStrings).contains("undefined");
+ assertThat(tagValueStrings).contains("undefined");
+ }
+}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java
index 133f06767a..b9bd2a926c 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java
@@ -34,6 +34,7 @@
import com.google.bigtable.v2.MutateRowResponse;
import com.google.bigtable.v2.ReadRowsRequest;
import com.google.bigtable.v2.ReadRowsResponse;
+import com.google.bigtable.v2.ResponseParams;
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
import com.google.cloud.bigtable.data.v2.FakeServiceBuilder;
import com.google.cloud.bigtable.data.v2.models.Query;
@@ -84,14 +85,15 @@ public class BuiltinMetricsTracerTest {
private static final String INSTANCE_ID = "fake-instance";
private static final String APP_PROFILE_ID = "default";
private static final String TABLE_ID = "fake-table";
- private static final String UNDEFINED = "undefined";
+ private static final String ZONE = "us-west-1";
+ private static final String CLUSTER = "cluster-0";
private static final long FAKE_SERVER_TIMING = 50;
private static final long SERVER_LATENCY = 100;
private static final long APPLICATION_LATENCY = 200;
@Rule public final MockitoRule mockitoRule = MockitoJUnit.rule();
- private FakeService fakeService;
+ private final FakeService fakeService = new FakeService();
private Server server;
private EnhancedBigtableStub stub;
@@ -106,8 +108,6 @@ public class BuiltinMetricsTracerTest {
@Before
public void setUp() throws Exception {
- fakeService = new FakeService();
-
// Add an interceptor to add server-timing in headers
ServerInterceptor trailersInterceptor =
new ServerInterceptor() {
@@ -123,6 +123,14 @@ public void sendHeaders(Metadata headers) {
headers.put(
Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER),
String.format("gfet4t7; dur=%d", FAKE_SERVER_TIMING));
+
+ ResponseParams params =
+ ResponseParams.newBuilder().setZoneId(ZONE).setClusterId(CLUSTER).build();
+ byte[] byteArray = params.toByteArray();
+ headers.put(
+ Metadata.Key.of(Util.RESPONSE_PRAMS_KEY, Metadata.BINARY_BYTE_MARSHALLER),
+ byteArray);
+
super.sendHeaders(headers);
}
},
@@ -333,8 +341,8 @@ public void testMutateRowAttempts() {
// and when the record() is called in onOperationCompletion().
verify(statsRecorderWrapper, timeout(50).times(fakeService.getAttemptCounter().get() + 1))
.record(status.capture(), tableId.capture(), zone.capture(), cluster.capture());
- assertThat(zone.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED);
- assertThat(cluster.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED);
+ assertThat(zone.getAllValues()).containsExactly("undefined", "undefined", ZONE, ZONE);
+ assertThat(cluster.getAllValues()).containsExactly("undefined", "undefined", CLUSTER, CLUSTER);
assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "UNAVAILABLE", "OK", "OK");
}
diff --git a/pom.xml b/pom.xml
index f898b3952b..7ae4cba9c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,6 +297,13 @@
maven-shade-plugin
3.3.0
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ true
+
+