Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add stackdriver exporter #1247

Merged
merged 30 commits into from
Aug 12, 2022
Merged
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ef67fbd
remove status from application latency
mutianf May 11, 2022
8c943fb
feat: update tracers to use built in metrics
mutianf May 10, 2022
8253a9a
feat: add response protos
mutianf Sep 8, 2021
92b1f48
feat: add response protos
mutianf Sep 8, 2021
50a1eab
feat: add stackdriver exporter
mutianf May 12, 2022
07615bd
fix tests
mutianf Jul 11, 2022
f499a1b
fix dependency
mutianf Jul 11, 2022
3ea3739
remove unused dependency
mutianf Jul 12, 2022
2032527
clean up code
mutianf Jul 12, 2022
fa1756e
udpates on comments
mutianf Jul 14, 2022
f5678b4
remove unused setting
mutianf Jul 15, 2022
1d77308
make metrics consistent with cloud monitoring
mutianf Jul 26, 2022
6c92a39
convert undefined to global
mutianf Aug 1, 2022
c3afccb
update
mutianf Aug 2, 2022
c966671
add bigtable tracer back in the base callable
mutianf Aug 2, 2022
aee2e15
fix format
mutianf Aug 2, 2022
8129f02
fix the tag name
mutianf Aug 2, 2022
551cff8
add the link to the form
mutianf Aug 3, 2022
da85a10
fix format
mutianf Aug 3, 2022
541feb1
fix dependency conflicts
mutianf Aug 4, 2022
d8f6162
fix image tests
mutianf Aug 4, 2022
b219200
update undefined cluster to global
mutianf Aug 5, 2022
a5530dd
address comments
mutianf Aug 9, 2022
8feaeec
tweak export interval
mutianf Aug 9, 2022
e960878
remove unused metric kind
mutianf Aug 10, 2022
ca63660
get project id from the metrics
mutianf Aug 11, 2022
5996119
clean up imports
mutianf Aug 11, 2022
ee04276
remove unused method and rewrite create timeseries exporter
mutianf Aug 11, 2022
7607480
fix integration test
mutianf Aug 12, 2022
559a168
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Aug 12, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix dependency
  • Loading branch information
mutianf committed Aug 4, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit f499a1b31f6fa25a50a4a748c271f73d4ac70d39
53 changes: 44 additions & 9 deletions google-cloud-bigtable-stats/pom.xml
Original file line number Diff line number Diff line change
@@ -25,6 +25,21 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-metrics-util</artifactId>
<version>0.31.1</version>
mutianf marked this conversation as resolved.
Show resolved Hide resolved
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-monitoring-v3</artifactId>
<version>3.3.1</version>
</dependency>
</dependencies>
</dependencyManagement>

@@ -51,32 +66,52 @@
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
</dependency>
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-credentials</artifactId>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-monitoring-v3</artifactId>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
</dependency>
<dependency>
<groupId>org.threeten</groupId>
<artifactId>threetenbp</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-stats-stackdriver</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
</exclusion>
</exclusions>
<artifactId>opencensus-exporter-metrics-util</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-common-protos</artifactId>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-impl</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2021 Google LLC
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,10 +34,12 @@
final class BigtableCreateTimeSeriesExporter extends MetricExporter {
private static final Logger logger =
Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName());

private static final String DOMAIN = "bigtable.googleapis.com/internal/client/";

private final ProjectName projectName;
private final MetricServiceClient metricServiceClient;
private final MonitoredResource monitoredResource;
private final String domain;

BigtableCreateTimeSeriesExporter(
String projectId,
@@ -46,7 +48,6 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter {
this.projectName = ProjectName.newBuilder().setProject(projectId).build();
this.metricServiceClient = metricServiceClient;
this.monitoredResource = monitoredResource;
this.domain = "bigtable.googleapis.com/client/";
}

public void export(Collection<Metric> metrics) {
@@ -101,8 +102,7 @@ public void export(Collection<Metric> metrics) {
updatedValues,
timeSeries,
monitoredResourceBuilder.build(),
this.domain,
this.projectName.getProject()));
DOMAIN));
}
}

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2021 Google LLC
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -26,9 +26,7 @@
import com.google.monitoring.v3.TypedValue;
import com.google.protobuf.Any;
import com.google.protobuf.ByteString;
import io.opencensus.common.Function;
import io.opencensus.common.Functions;
import io.opencensus.contrib.exemplar.util.AttachmentValueSpanContext;
import io.opencensus.metrics.LabelKey;
import io.opencensus.metrics.LabelValue;
import io.opencensus.metrics.data.AttachmentValue;
@@ -115,8 +113,7 @@ static TimeSeries convertTimeSeries(
List<LabelValue> labelValues,
io.opencensus.metrics.export.TimeSeries timeSeries,
MonitoredResource monitoredResource,
String domain,
String projectId) {
String domain) {

TimeSeries.Builder builder = TimeSeries.newBuilder();
builder.setMetricKind(createMetricKind(metricType));
@@ -138,9 +135,9 @@ static com.google.api.Metric createMetric(
Map<String, String> stringTagMap = Maps.newHashMap();

for (int i = 0; i < labelValues.size(); ++i) {
String value = ((LabelValue) labelValues.get(i)).getValue();
String value = labelValues.get(i).getValue();
if (value != null) {
stringTagMap.put(((LabelKey) labelKeys.get(i)).getKey(), value);
stringTagMap.put(labelKeys.get(i).getKey(), value);
}
}

@@ -166,13 +163,12 @@ static com.google.monitoring.v3.Point createPoint(

@VisibleForTesting
static TypedValue createTypedValue(Value value) {
return (TypedValue)
value.match(
typedValueDoubleFunction,
typedValueLongFunction,
typedValueDistributionFunction,
typedValueSummaryFunction,
Functions.throwIllegalArgumentException());
return value.match(
typedValueDoubleFunction,
typedValueLongFunction,
typedValueDistributionFunction,
typedValueSummaryFunction,
Functions.throwIllegalArgumentException());
}

@VisibleForTesting
@@ -197,9 +193,8 @@ static BucketOptions createBucketOptions(
com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder();
return bucketOptions == null
? builder.build()
: (BucketOptions)
bucketOptions.match(
bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException());
: bucketOptions.match(
bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException());
}

static MonitoredResource getDefaultResource() {
@@ -227,16 +222,11 @@ private static com.google.api.Distribution.Exemplar toProtoExemplar(
com.google.api.Distribution.Exemplar.newBuilder()
.setValue(exemplar.getValue())
.setTimestamp(convertTimestamp(exemplar.getTimestamp()));
io.opencensus.trace.SpanContext spanContext = null;

for (Map.Entry<String, AttachmentValue> attachment : exemplar.getAttachments().entrySet()) {
String key = (String) attachment.getKey();
AttachmentValue value = (AttachmentValue) attachment.getValue();
if ("SpanContext".equals(key)) {
spanContext = ((AttachmentValueSpanContext) value).getSpanContext();
} else {
builder.addAttachments(toProtoStringAttachment(value));
}
String key = attachment.getKey();
AttachmentValue value = attachment.getValue();
builder.addAttachments(toProtoStringAttachment(value));
}

return builder.build();
@@ -249,25 +239,6 @@ private static Any toProtoStringAttachment(AttachmentValue attachmentValue) {
.build();
}

private static Any toProtoSpanContextAttachment(
com.google.monitoring.v3.SpanContext protoSpanContext) {
return Any.newBuilder()
.setTypeUrl("type.googleapis.com/google.monitoring.v3.SpanContext")
.setValue(protoSpanContext.toByteString())
.build();
}

private static com.google.monitoring.v3.SpanContext toProtoSpanContext(
io.opencensus.trace.SpanContext spanContext, String projectId) {
String spanName =
String.format(
"projects/%s/traces/%s/spans/%s",
projectId,
spanContext.getTraceId().toLowerBase16(),
spanContext.getSpanId().toLowerBase16());
return com.google.monitoring.v3.SpanContext.newBuilder().setSpanName(spanName).build();
}

@VisibleForTesting
static com.google.protobuf.Timestamp convertTimestamp(
io.opencensus.common.Timestamp censusTimestamp) {
@@ -284,48 +255,37 @@ private BigtableStackdriverExportUtils() {}
static {
mutianf marked this conversation as resolved.
Show resolved Hide resolved
logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName());
typedValueDoubleFunction =
new io.opencensus.common.Function<Double, TypedValue>() {
public TypedValue apply(Double arg) {
com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder();
builder.setDoubleValue(arg);
return builder.build();
}
arg -> {
TypedValue.Builder builder = TypedValue.newBuilder();
builder.setDoubleValue(arg);
return builder.build();
};
typedValueLongFunction =
new io.opencensus.common.Function<Long, TypedValue>() {
public TypedValue apply(Long arg) {
com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder();
builder.setInt64Value(arg);
return builder.build();
}
arg -> {
TypedValue.Builder builder = TypedValue.newBuilder();
builder.setInt64Value(arg);
return builder.build();
};
typedValueDistributionFunction =
new io.opencensus.common.Function<io.opencensus.metrics.export.Distribution, TypedValue>() {
public TypedValue apply(io.opencensus.metrics.export.Distribution arg) {
com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder();
return builder
.setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg))
.build();
}
arg -> {
TypedValue.Builder builder = TypedValue.newBuilder();
return builder
.setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg))
.build();
};
typedValueSummaryFunction =
new io.opencensus.common.Function<Summary, TypedValue>() {
public TypedValue apply(Summary arg) {
com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder();
return builder.build();
}
arg -> {
TypedValue.Builder builder = TypedValue.newBuilder();
return builder.build();
};
bucketOptionsExplicitFunction =
new Function<ExplicitOptions, BucketOptions>() {
public BucketOptions apply(ExplicitOptions arg) {
com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder();
com.google.api.Distribution.BucketOptions.Explicit.Builder explicitBuilder =
Explicit.newBuilder();
explicitBuilder.addBounds(0.0D);
explicitBuilder.addAllBounds(arg.getBucketBoundaries());
builder.setExplicitBuckets(explicitBuilder.build());
return builder.build();
}
arg -> {
BucketOptions.Builder builder = BucketOptions.newBuilder();
Explicit.Builder explicitBuilder = Explicit.newBuilder();
explicitBuilder.addBounds(0.0D);
explicitBuilder.addAllBounds(arg.getBucketBoundaries());
builder.setExplicitBuckets(explicitBuilder.build());
return builder.build();
};
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2021 Google LLC
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,17 +36,14 @@
@ThreadSafe
mutianf marked this conversation as resolved.
Show resolved Hide resolved
@InternalApi
public class BigtableStackdriverStatsExporter {
@VisibleForTesting static final Object monitor = new Object();
static final Object monitor = new Object();
mutianf marked this conversation as resolved.
Show resolved Hide resolved

@Nullable
@GuardedBy("monitor")
private static BigtableStackdriverStatsExporter instance = null;

private static final String EXPORTER_SPAN_NAME = "ExportMetricsToStackdriver";
// private static final String USER_AGENT_KEY = "user-agent";
// private static final String USER_AGENT = "bigtable-java" + Version.VERSION;
// private static final HeaderProvider OPENCENSUS_USER_AGENT_HEADER_PROVIDER =
// FixedHeaderProvider.create(new String[] {USER_AGENT_KEY, USER_AGENT});
private static final String EXPORTER_SPAN_NAME = "BigtableExportMetricsToStackdriver";
private static final Duration EXPORT_INTERVAL = Duration.create(600, 0);
mutianf marked this conversation as resolved.
Show resolved Hide resolved
private final IntervalMetricReader intervalMetricReader;

private BigtableStackdriverStatsExporter(
@@ -72,13 +69,14 @@ private BigtableStackdriverStatsExporter(
public static void register(@Nullable Credentials credentials, String projectId)
throws IOException {
synchronized (monitor) {
Preconditions.checkState(instance == null, "Stackdriver stats exporter is already created");
Preconditions.checkState(
instance == null, "Bigtable Stackdriver stats exporter is already created");
mutianf marked this conversation as resolved.
Show resolved Hide resolved
MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0));
instance =
new BigtableStackdriverStatsExporter(
projectId,
client,
Duration.create(600, 0),
EXPORT_INTERVAL,
BigtableStackdriverExportUtils.getDefaultResource());
}
}
@@ -88,13 +86,8 @@ public static void register(@Nullable Credentials credentials, String projectId)
static MetricServiceClient createMetricServiceClient(
@Nullable Credentials credentials, Duration deadline) throws IOException {
com.google.cloud.monitoring.v3.MetricServiceSettings.Builder settingsBuilder =
(com.google.cloud.monitoring.v3.MetricServiceSettings.Builder)
MetricServiceSettings.newBuilder()
.setTransportChannelProvider(
InstantiatingGrpcChannelProvider.newBuilder()
//
// .setHeaderProvider(OPENCENSUS_USER_AGENT_HEADER_PROVIDER)
.build());
MetricServiceSettings.newBuilder()
.setTransportChannelProvider(InstantiatingGrpcChannelProvider.newBuilder().build());
if (credentials != null) {
settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
}
Original file line number Diff line number Diff line change
@@ -21,8 +21,6 @@

import com.google.api.MonitoredResource;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.cloud.bigtable.stats.BigtableCreateTimeSeriesExporter;
import com.google.cloud.bigtable.stats.BuiltinMeasureConstants;
import com.google.cloud.monitoring.v3.MetricServiceClient;
import com.google.cloud.monitoring.v3.stub.MetricServiceStub;
import com.google.monitoring.v3.CreateTimeSeriesRequest;
8 changes: 8 additions & 0 deletions google-cloud-bigtable/pom.xml
Original file line number Diff line number Diff line change
@@ -60,9 +60,17 @@
<dependencies>
<!-- NOTE: Dependencies are organized into two groups, production and test.
Within a group, dependencies are sorted by (groupId, artifactId) -->

<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable-stats</artifactId>
<!-- Exclude all the shaded artifacts -->
mutianf marked this conversation as resolved.
Show resolved Hide resolved
<exclusions>
<exclusion>
<groupId>io.opencensus</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Production dependencies -->
<dependency>
Loading