Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add stackdriver exporter #1247

Merged
merged 30 commits into from
Aug 12, 2022
Merged
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ef67fbd
remove status from application latency
mutianf May 11, 2022
8c943fb
feat: update tracers to use built in metrics
mutianf May 10, 2022
8253a9a
feat: add response protos
mutianf Sep 8, 2021
92b1f48
feat: add response protos
mutianf Sep 8, 2021
50a1eab
feat: add stackdriver exporter
mutianf May 12, 2022
07615bd
fix tests
mutianf Jul 11, 2022
f499a1b
fix dependency
mutianf Jul 11, 2022
3ea3739
remove unused dependency
mutianf Jul 12, 2022
2032527
clean up code
mutianf Jul 12, 2022
fa1756e
udpates on comments
mutianf Jul 14, 2022
f5678b4
remove unused setting
mutianf Jul 15, 2022
1d77308
make metrics consistent with cloud monitoring
mutianf Jul 26, 2022
6c92a39
convert undefined to global
mutianf Aug 1, 2022
c3afccb
update
mutianf Aug 2, 2022
c966671
add bigtable tracer back in the base callable
mutianf Aug 2, 2022
aee2e15
fix format
mutianf Aug 2, 2022
8129f02
fix the tag name
mutianf Aug 2, 2022
551cff8
add the link to the form
mutianf Aug 3, 2022
da85a10
fix format
mutianf Aug 3, 2022
541feb1
fix dependency conflicts
mutianf Aug 4, 2022
d8f6162
fix image tests
mutianf Aug 4, 2022
b219200
update undefined cluster to global
mutianf Aug 5, 2022
a5530dd
address comments
mutianf Aug 9, 2022
8feaeec
tweak export interval
mutianf Aug 9, 2022
e960878
remove unused metric kind
mutianf Aug 10, 2022
ca63660
get project id from the metrics
mutianf Aug 11, 2022
5996119
clean up imports
mutianf Aug 11, 2022
ee04276
remove unused method and rewrite create timeseries exporter
mutianf Aug 11, 2022
7607480
fix integration test
mutianf Aug 12, 2022
559a168
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Aug 12, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
make metrics consistent with cloud monitoring
  • Loading branch information
mutianf committed Aug 4, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 1d77308cfa8213c471eb9f92ac3ba60f15d8b434
Original file line number Diff line number Diff line change
@@ -60,8 +60,6 @@ class BigtableStackdriverExportUtils {
private static final io.opencensus.common.Function<ExplicitOptions, BucketOptions>
bucketOptionsExplicitFunction;

private static final String DOMAIN = "bigtable.googleapis.com/internal/client/";

private static final Set<String> PROMOTED_RESOURCE_LABELS =
mutianf marked this conversation as resolved.
Show resolved Hide resolved
ImmutableSet.of(
BuiltinMeasureConstants.PROJECT_ID.getName(),
@@ -142,7 +140,7 @@ static TimeSeries convertTimeSeries(
metricTagValues.add(labelValues.get(i));
}
}
metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_ID.getName(), "client id"));
metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id"));
mutianf marked this conversation as resolved.
Show resolved Hide resolved
metricTagValues.add(LabelValue.create(clientId));

TimeSeries.Builder builder = TimeSeries.newBuilder();
@@ -161,7 +159,7 @@ static TimeSeries convertTimeSeries(
static com.google.api.Metric createMetric(
String metricName, List<LabelKey> labelKeys, List<LabelValue> labelValues) {
com.google.api.Metric.Builder builder = com.google.api.Metric.newBuilder();
builder.setType(DOMAIN + metricName);
builder.setType(metricName);
Map<String, String> stringTagMap = Maps.newHashMap();

for (int i = 0; i < labelValues.size(); ++i) {
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ public class BigtableStackdriverStatsExporter {
@GuardedBy("monitor")
private static BigtableStackdriverStatsExporter instance = null;

private static final Duration EXPORT_INTERVAL = Duration.create(600, 0);
private static final Duration EXPORT_INTERVAL = Duration.create(60, 0);
private static final String RESOURCE_TYPE = "bigtable_client_raw";

private final IntervalMetricReader intervalMetricReader;
Original file line number Diff line number Diff line change
@@ -23,12 +23,11 @@
class BuiltinMeasureConstants {
// Monitored resource TagKeys
static final TagKey PROJECT_ID = TagKey.create("project_id");
static final TagKey INSTANCE_ID = TagKey.create("instance_id");
static final TagKey INSTANCE_ID = TagKey.create("instance");
static final TagKey CLUSTER = TagKey.create("cluster");
static final TagKey TABLE = TagKey.create("table");
static final TagKey ZONE = TagKey.create("zone");
// Placeholder TagKey to be used in Stackdriver exporter
static final TagKey CLIENT_ID = TagKey.create("client_id");
static final TagKey CLIENT_UID = TagKey.create("client_uid");

// Metrics TagKeys
static final TagKey APP_PROFILE = TagKey.create("app_profile");
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ class BuiltinViewConstants {
1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0,
100.0)));

private static final Aggregation AGGREGATION_ERROR_COUNT = Sum.create();
private static final Aggregation AGGREGATION_COUNT = Sum.create();

static final View OPERATION_LATENCIES_VIEW =
View.create(
@@ -102,7 +102,7 @@ class BuiltinViewConstants {
View.Name.create("bigtable.googleapis.com/internal/client/retry_count"),
"The number of additional RPCs sent after the initial attempt.",
RETRY_COUNT,
AGGREGATION_RETRY_COUNT,
AGGREGATION_COUNT,
ImmutableList.of(
PROJECT_ID,
INSTANCE_ID,
@@ -154,7 +154,7 @@ class BuiltinViewConstants {
View.Name.create("bigtable.googleapis.com/internal/client/connectivity_error_count"),
"Number of requests that failed to reach the Google datacenter. (Requests without google response headers).",
CONNECTIVITY_ERROR_COUNT,
AGGREGATION_ERROR_COUNT,
AGGREGATION_COUNT,
ImmutableList.of(
PROJECT_ID,
INSTANCE_ID,
@@ -173,15 +173,7 @@ class BuiltinViewConstants {
APPLICATION_LATENCIES,
AGGREGATION_WITH_MILLIS_HISTOGRAM,
ImmutableList.of(
PROJECT_ID,
INSTANCE_ID,
APP_PROFILE,
METHOD,
STREAMING,
CLIENT_NAME,
CLUSTER,
ZONE,
TABLE));
PROJECT_ID, INSTANCE_ID, APP_PROFILE, METHOD, CLIENT_NAME, CLUSTER, ZONE, TABLE));

static final View THROTTLING_LATENCIES_VIEW =
View.create(
Original file line number Diff line number Diff line change
@@ -134,7 +134,7 @@ public void testTimeSeries() {
assertThat(timeSeries.getMetric().getLabelsMap())
.containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId);
assertThat(timeSeries.getMetric().getLabelsMap())
.containsKey(BuiltinMeasureConstants.CLIENT_ID.getName());
.containsKey(BuiltinMeasureConstants.CLIENT_UID.getName());

assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue);
}
Original file line number Diff line number Diff line change
@@ -201,7 +201,7 @@ public static EnhancedBigtableStubSettings finalizeSettings(
ImmutableMap<String, String> builtinAttributes =
ImmutableMap.<String, String>builder()
.put("project_id", settings.getProjectId())
.put("instance_id", settings.getInstanceId())
.put("instance", settings.getInstanceId())
.put("app_profile", settings.getAppProfileId())
.build();
// Inject Opencensus instrumentation
@@ -335,10 +335,13 @@ public <RowT> ServerStreamingCallable<Query, RowT> createReadRowsCallable(
ServerStreamingCallable<Query, RowT> readRowsUserCallable =
new ReadRowsUserCallable<>(readRowsCallable, requestContext);

ServerStreamingCallable<Query, RowT> withBigtableTracer =
new BigtableTracerStreamingCallable<>(readRowsUserCallable);

SpanName span = getSpanName("ReadRows");
ServerStreamingCallable<Query, RowT> traced =
new TracedServerStreamingCallable<>(
readRowsUserCallable, clientContext.getTracerFactory(), span);
withBigtableTracer, clientContext.getTracerFactory(), span);

return traced.withDefaultCallContext(clientContext.getDefaultCallContext());
}
@@ -438,13 +441,10 @@ public Map<String, String> extract(ReadRowsRequest readRowsRequest) {
ServerStreamingCallable<ReadRowsRequest, RowT> watched =
Callables.watched(merging, innerSettings, clientContext);

ServerStreamingCallable<ReadRowsRequest, RowT> withBigtableTracer =
new BigtableTracerStreamingCallable<>(watched);

// Retry logic is split into 2 parts to workaround a rare edge case described in
// ReadRowsRetryCompletedCallable
ServerStreamingCallable<ReadRowsRequest, RowT> retrying1 =
new ReadRowsRetryCompletedCallable<>(withBigtableTracer);
new ReadRowsRetryCompletedCallable<>(watched);

ServerStreamingCallable<ReadRowsRequest, RowT> retrying2 =
Callables.retrying(retrying1, innerSettings, clientContext);
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@
import com.google.api.gax.rpc.ServerStreamingCallable;
import com.google.api.gax.rpc.StreamController;
import com.google.bigtable.v2.ResponseParams;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.protobuf.InvalidProtocolBufferException;
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import com.google.api.gax.rpc.ApiCallContext;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.bigtable.v2.ResponseParams;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.InvalidProtocolBufferException;