From c6cd148be30628018a9d876dba368fc6f78f99fe Mon Sep 17 00:00:00 2001 From: Mari Date: Thu, 7 Sep 2023 11:14:37 +0200 Subject: [PATCH 1/8] add otel push exporter --- autometrics/Cargo.toml | 33 ++++++ autometrics/src/lib.rs | 2 + autometrics/src/otel_push_exporter.rs | 93 ++++++++++++++++ autometrics/tests/compile_test.rs | 2 +- examples/README.md | 1 + examples/opentelemetry-push-http/Cargo.toml | 10 ++ examples/opentelemetry-push-http/README.md | 103 ++++++++++++++++++ .../otel-collector-config.yml | 21 ++++ examples/opentelemetry-push-http/src/main.rs | 27 +++++ examples/opentelemetry-push/src/main.rs | 1 + 10 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 autometrics/src/otel_push_exporter.rs create mode 100644 examples/opentelemetry-push-http/Cargo.toml create mode 100644 examples/opentelemetry-push-http/README.md create mode 100644 examples/opentelemetry-push-http/otel-collector-config.yml create mode 100644 examples/opentelemetry-push-http/src/main.rs diff --git a/autometrics/Cargo.toml b/autometrics/Cargo.toml index d81a17e..408b40a 100644 --- a/autometrics/Cargo.toml +++ b/autometrics/Cargo.toml @@ -36,6 +36,38 @@ prometheus-exporter = [ "dep:prometheus-client", ] +otel-push-exporter = [ + "opentelemetry_sdk", + "opentelemetry_api", + "opentelemetry-otlp", + "opentelemetry-otlp/tls-roots", +] + +otel-push-exporter-http = [ + "otel-push-exporter", + "opentelemetry-otlp/http-proto" +] + +otel-push-exporter-grpc = [ + "otel-push-exporter", + "opentelemetry-otlp/grpc-tonic" +] + +otel-push-exporter-tokio = [ + "otel-push-exporter", + "opentelemetry_sdk/rt-tokio" +] + +otel-push-exporter-tokio-current-thread = [ + "otel-push-exporter", + "opentelemetry_sdk/rt-tokio-current-thread" +] + +otel-push-exporter-async-std = [ + "otel-push-exporter", + "opentelemetry_sdk/rt-async-std" +] + # Exemplars exemplars-tracing = ["tracing", "tracing-subscriber"] exemplars-tracing-opentelemetry-0_20 = [ @@ -68,6 +100,7 @@ opentelemetry-prometheus = { version = "0.13.0", optional = true } opentelemetry_sdk = { version = "0.20", default-features = false, features = [ "metrics", ], optional = true } +opentelemetry-otlp = { version = "0.13.0", default-features = false, optional = true } prometheus = { version = "0.13", default-features = false, optional = true } # Used for prometheus-client feature diff --git a/autometrics/src/lib.rs b/autometrics/src/lib.rs index 1033b68..46261c2 100644 --- a/autometrics/src/lib.rs +++ b/autometrics/src/lib.rs @@ -14,6 +14,8 @@ mod constants; pub mod exemplars; mod labels; pub mod objectives; +#[cfg(feature = "otel-push-exporter")] +pub mod otel_push_exporter; #[cfg(feature = "prometheus-exporter")] pub mod prometheus_exporter; pub mod settings; diff --git a/autometrics/src/otel_push_exporter.rs b/autometrics/src/otel_push_exporter.rs new file mode 100644 index 0000000..81bf060 --- /dev/null +++ b/autometrics/src/otel_push_exporter.rs @@ -0,0 +1,93 @@ +use opentelemetry_api::metrics::MetricsError; +use opentelemetry_otlp::OtlpMetricPipeline; +use opentelemetry_otlp::{ExportConfig, Protocol, WithExportConfig}; +use opentelemetry_sdk::metrics::MeterProvider; +use opentelemetry_sdk::runtime; +use std::ops::Deref; +use std::time::Duration; + +#[repr(transparent)] +pub struct OtelMeterProvider(MeterProvider); + +impl Deref for OtelMeterProvider { + type Target = MeterProvider; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Drop for OtelMeterProvider { + fn drop(&mut self) { + // this will only error if `.shutdown` gets called multiple times + let _ = self.0.shutdown(); + } +} + +#[cfg(feature = "otel-push-exporter-http")] +pub fn init_http(url: impl Into) -> Result { + runtime() + .with_exporter( + opentelemetry_otlp::new_exporter() + .http() + .with_export_config(ExportConfig { + endpoint: url.into(), + protocol: Protocol::HttpBinary, + ..Default::default() + }), + ) + .with_period(Duration::from_secs(1)) + .build() + .map(OtelMeterProvider) +} + +#[cfg(feature = "otel-push-exporter-grpc")] +pub fn init_grpc(url: impl Into) -> Result { + runtime() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(ExportConfig { + endpoint: url.into(), + protocol: Protocol::Grpc, + ..Default::default() + }), + ) + .with_period(Duration::from_secs(1)) + .build() + .map(OtelMeterProvider) +} + +#[cfg(all( + feature = "otel-push-exporter-tokio", + not(any( + feature = "otel-push-exporter-tokio-current-thread", + feature = "otel-push-exporter-async-std" + )) +))] +fn runtime() -> OtlpMetricPipeline { + return opentelemetry_otlp::new_pipeline().metrics(opentelemetry_sdk::runtime::Tokio); +} + +#[cfg(all( + feature = "otel-push-exporter-tokio-current-thread", + not(any( + feature = "otel-push-exporter-tokio", + feature = "otel-push-exporter-async-std" + )) +))] +fn runtime() -> OtlpMetricPipeline { + return opentelemetry_otlp::new_pipeline() + .metrics(opentelemetry_sdk::runtime::TokioCurrentThread); +} + +#[cfg(all( + feature = "otel-push-exporter-async-std", + not(any( + feature = "otel-push-exporter-tokio", + feature = "otel-push-exporter-tokio-current-thread" + )) +))] +fn runtime() -> OtlpMetricPipeline { + return opentelemetry_otlp::new_pipeline().metrics(opentelemetry_sdk::runtime::AsyncStd); +} diff --git a/autometrics/tests/compile_test.rs b/autometrics/tests/compile_test.rs index 7edb0cb..ec54f77 100644 --- a/autometrics/tests/compile_test.rs +++ b/autometrics/tests/compile_test.rs @@ -1,5 +1,5 @@ -use std::io; use autometrics::autometrics; +use std::io; // general purpose `Result`, part of the std prelude. // notice both `Ok` and `Err` generic type arguments are explicitly provided diff --git a/examples/README.md b/examples/README.md index 123fb9b..8e2633f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,6 +14,7 @@ cargo run --package example-{name of example} - [custom-metrics](./custom-metrics/) - Define your own custom metrics alongside the ones generated by autometrics (using any of the metrics collection crates) - [exemplars-tracing](./exemplars-tracing/) - Use fields from `tracing::Span`s as Prometheus exemplars - [opentelemetry-push](./opentelemetry-push/) - Push metrics to an OpenTelemetry Collector via the OTLP gRPC protocol +- [opentelemetry-push-http](./opentelemetry-push-http/) - Push metrics to an OpenTelemetry Collector via the OTLP HTTP protocol using the Autometrics provided interface ## Full Example diff --git a/examples/opentelemetry-push-http/Cargo.toml b/examples/opentelemetry-push-http/Cargo.toml new file mode 100644 index 0000000..0a1b995 --- /dev/null +++ b/examples/opentelemetry-push-http/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "example-opentelemetry-push-http" +version = "0.0.0" +publish = false +edition = "2021" + +[dependencies] +autometrics = { path = "../../autometrics", features = ["opentelemetry-0_20", "otel-push-exporter-http", "otel-push-exporter-tokio"] } +autometrics-example-util = { path = "../util" } +tokio = { version = "1", features = ["full"] } diff --git a/examples/opentelemetry-push-http/README.md b/examples/opentelemetry-push-http/README.md new file mode 100644 index 0000000..48c7fb8 --- /dev/null +++ b/examples/opentelemetry-push-http/README.md @@ -0,0 +1,103 @@ +# Autometrics + OTLP push controller + +This example demonstrates how you can push autometrics via OTLP HTTP protocol +to the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or another OTel-compatible solution. It +uses the Autometrics provided wrapper to achieve this. + +## Running the example + +### Start a basic OTEL-Collector + +You can use the [`otel-collector-config.yml`](./otel-collector-config.yml) file to start an otel-collector container that listens on 0.0.0.0:4317 for incoming otlp-gRPC traffic as well as on 0.0.0.0:4318 for incoming otlp-http traffic, and exports the metrics it receives to stdout. + +Run the following command in a second terminal to start a container in interactive mode: + +```bash +docker run -it --name otel-col \ + -p 4317:4317 -p 13133:13133 \ + -v $PWD/otel-collector-config.yml:/etc/otelcol/config.yaml \ + otel/opentelemetry-collector-contrib:latest +``` + +You should see the collector initialization output, that should end with something like: + +```text +... +2023-06-07T15:56:42.617Z info otlpreceiver@v0.75.0/otlp.go:94 Starting GRPC server {"kind": "receiver", "name": "otlp", "data_type": "metrics", "endpoint": "0.0.0.0:4317"} +2023-06-07T15:56:42.618Z info service/service.go:146 Everything is ready. Begin running and processing data. +``` + +### Execute example code + +Then come back on your primary shell and run this example: + +```shell +cargo run -p example-opentelemetry-push-http +``` + +### Check the output + +On the stdout of the terminal where you started the opentelemetry-collector container, you should see the metrics generated by autometrics macro being pushed every 10 seconds since example exit, like: + +```text +... +Metric #0 +Descriptor: + -> Name: function.calls + -> Description: Autometrics counter for tracking function calls + -> Unit: + -> DataType: Sum + -> IsMonotonic: true + -> AggregationTemporality: Cumulative +NumberDataPoints #0 +Data point attributes: + -> caller: Str() + -> function: Str(do_stuff) + -> module: Str(example_opentelemetry_push) +StartTimestamp: 2023-06-07 16:01:08.549300623 +0000 UTC +Timestamp: 2023-06-07 16:01:48.551531429 +0000 UTC +Value: 10 +Metric #1 +Descriptor: + -> Name: build_info + -> Description: Autometrics info metric for tracking software version and build details + -> Unit: + -> DataType: Sum + -> IsMonotonic: false + -> AggregationTemporality: Cumulative +NumberDataPoints #0 +Data point attributes: + -> branch: Str() + -> commit: Str() + -> version: Str(0.0.0) +StartTimestamp: 2023-06-07 16:01:08.549300623 +0000 UTC +Timestamp: 2023-06-07 16:01:48.551531429 +0000 UTC +Value: 1.000000 +Metric #2 +Descriptor: + -> Name: function.calls.duration + -> Description: Autometrics histogram for tracking function call duration + -> Unit: + -> DataType: Sum + -> IsMonotonic: false + -> AggregationTemporality: Cumulative +NumberDataPoints #0 +Data point attributes: + -> function: Str(do_stuff) + -> module: Str(example_opentelemetry_push) +StartTimestamp: 2023-06-07 16:01:08.549300623 +0000 UTC +Timestamp: 2023-06-07 16:01:48.551531429 +0000 UTC +Value: 0.000122 + {"kind": "exporter", "data_type": "metrics", "name": "logging"} +... +``` + +### Cleanup + +In the end, to stop the opentelemetry collector container just hit `^C`. + +Then delete the container with + +```bash +docker rm otel-col +``` diff --git a/examples/opentelemetry-push-http/otel-collector-config.yml b/examples/opentelemetry-push-http/otel-collector-config.yml new file mode 100644 index 0000000..1303b66 --- /dev/null +++ b/examples/opentelemetry-push-http/otel-collector-config.yml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + logging: + loglevel: debug + +processors: + batch: + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [] + exporters: [logging] diff --git a/examples/opentelemetry-push-http/src/main.rs b/examples/opentelemetry-push-http/src/main.rs new file mode 100644 index 0000000..f6bbc69 --- /dev/null +++ b/examples/opentelemetry-push-http/src/main.rs @@ -0,0 +1,27 @@ +use autometrics::{autometrics, otel_push_exporter}; +use autometrics_example_util::sleep_random_duration; +use std::error::Error; +use std::time::Duration; +use tokio::time::sleep; + +#[autometrics] +async fn do_stuff() { + println!("Doing stuff..."); + sleep_random_duration().await; +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let _meter_provider = otel_push_exporter::init_http("http://0.0.0.0:4318")?; + // or: otel_push_exporter::init_grpc("http://0.0.0.0:4317"); + + for _ in 0..100 { + do_stuff().await; + } + + println!("Waiting so that we could see metrics going down..."); + sleep(Duration::from_secs(10)).await; + + // no need to call `.shutdown` as the returned `OtelMeterProvider` has a `Drop` implementation + Ok(()) +} diff --git a/examples/opentelemetry-push/src/main.rs b/examples/opentelemetry-push/src/main.rs index cee72c6..57ba0b2 100644 --- a/examples/opentelemetry-push/src/main.rs +++ b/examples/opentelemetry-push/src/main.rs @@ -44,5 +44,6 @@ async fn main() -> Result<(), Box> { sleep(Duration::from_secs(10)).await; meter_provider.force_flush(&cx)?; + meter_provider.shutdown()?; Ok(()) } From ca685a612ecc704196007f94b3ff11cca64dccbf Mon Sep 17 00:00:00 2001 From: Mari Date: Thu, 7 Sep 2023 12:15:25 +0200 Subject: [PATCH 2/8] rename examples to make it more understanding --- examples/README.md | 4 +- examples/opentelemetry-push-custom/Cargo.toml | 15 ++++++ .../README.md | 17 ++++--- .../otel-collector-config.yml | 2 - .../opentelemetry-push-custom/src/main.rs | 49 +++++++++++++++++++ examples/opentelemetry-push-http/Cargo.toml | 10 ---- examples/opentelemetry-push-http/src/main.rs | 27 ---------- examples/opentelemetry-push/Cargo.toml | 7 +-- examples/opentelemetry-push/README.md | 12 ++--- .../otel-collector-config.yml | 2 + examples/opentelemetry-push/src/main.rs | 30 ++---------- 11 files changed, 88 insertions(+), 87 deletions(-) create mode 100644 examples/opentelemetry-push-custom/Cargo.toml rename examples/{opentelemetry-push-http => opentelemetry-push-custom}/README.md (81%) rename examples/{opentelemetry-push-http => opentelemetry-push-custom}/otel-collector-config.yml (85%) create mode 100644 examples/opentelemetry-push-custom/src/main.rs delete mode 100644 examples/opentelemetry-push-http/Cargo.toml delete mode 100644 examples/opentelemetry-push-http/src/main.rs diff --git a/examples/README.md b/examples/README.md index 8e2633f..7dec046 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,8 +13,8 @@ cargo run --package example-{name of example} - [axum](./axum) - Use autometrics to instrument HTTP handlers using the `axum` framework - [custom-metrics](./custom-metrics/) - Define your own custom metrics alongside the ones generated by autometrics (using any of the metrics collection crates) - [exemplars-tracing](./exemplars-tracing/) - Use fields from `tracing::Span`s as Prometheus exemplars -- [opentelemetry-push](./opentelemetry-push/) - Push metrics to an OpenTelemetry Collector via the OTLP gRPC protocol -- [opentelemetry-push-http](./opentelemetry-push-http/) - Push metrics to an OpenTelemetry Collector via the OTLP HTTP protocol using the Autometrics provided interface +- [opentelemetry-push](./opentelemetry-push/) - Push metrics to an OpenTelemetry Collector via the OTLP HTTP or gRPC protocol using the Autometrics provided interface +- [opentelemetry-push-custom](./opentelemetry-push-custom/) - Push metrics to an OpenTelemetry Collector via the OTLP gRPC protocol using custom options ## Full Example diff --git a/examples/opentelemetry-push-custom/Cargo.toml b/examples/opentelemetry-push-custom/Cargo.toml new file mode 100644 index 0000000..fca59cc --- /dev/null +++ b/examples/opentelemetry-push-custom/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "example-opentelemetry-push-custom" +version = "0.0.0" +publish = false +edition = "2021" + +[dependencies] +autometrics = { path = "../../autometrics", features = ["opentelemetry-0_20"] } +autometrics-example-util = { path = "../util" } +# Note that the version of the opentelemetry crate MUST match +# the version used by autometrics +opentelemetry = { version = "0.20", features = ["metrics", "rt-tokio"] } +opentelemetry-otlp = { version = "0.13", features = ["tonic", "metrics"] } +opentelemetry-semantic-conventions = { version = "0.12.0" } +tokio = { version = "1", features = ["full"] } diff --git a/examples/opentelemetry-push-http/README.md b/examples/opentelemetry-push-custom/README.md similarity index 81% rename from examples/opentelemetry-push-http/README.md rename to examples/opentelemetry-push-custom/README.md index 48c7fb8..8ff8f73 100644 --- a/examples/opentelemetry-push-http/README.md +++ b/examples/opentelemetry-push-custom/README.md @@ -1,14 +1,13 @@ -# Autometrics + OTLP push controller +# Autometrics + OTLP push controller (custom) -This example demonstrates how you can push autometrics via OTLP HTTP protocol -to the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or another OTel-compatible solution. It -uses the Autometrics provided wrapper to achieve this. +This example demonstrates how you can push autometrics via OTLP gRPC protocol to the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or another OTel-compatible solution +without using the Autometrics provided interface. ## Running the example ### Start a basic OTEL-Collector -You can use the [`otel-collector-config.yml`](./otel-collector-config.yml) file to start an otel-collector container that listens on 0.0.0.0:4317 for incoming otlp-gRPC traffic as well as on 0.0.0.0:4318 for incoming otlp-http traffic, and exports the metrics it receives to stdout. +You can use the [`otel-collector-config.yml`](./otel-collector-config.yml) file to start an otel-collector container that listens on 0.0.0.0:4317 for incoming otlp-gRPC traffic, and exports the metrics it receives to stdout. Run the following command in a second terminal to start a container in interactive mode: @@ -16,7 +15,7 @@ Run the following command in a second terminal to start a container in interacti docker run -it --name otel-col \ -p 4317:4317 -p 13133:13133 \ -v $PWD/otel-collector-config.yml:/etc/otelcol/config.yaml \ - otel/opentelemetry-collector-contrib:latest + otel/opentelemetry-collector:latest ``` You should see the collector initialization output, that should end with something like: @@ -32,7 +31,7 @@ You should see the collector initialization output, that should end with somethi Then come back on your primary shell and run this example: ```shell -cargo run -p example-opentelemetry-push-http +cargo run -p example-opentelemetry-push-custom ``` ### Check the output @@ -101,3 +100,7 @@ Then delete the container with ```bash docker rm otel-col ``` + +## OpenTelemetry Metrics Push Controller + +The metric push controller is implemented as from this [example](https://github.com/open-telemetry/opentelemetry-rust/blob/f20c9b40547ee20b6ec99414bb21abdd3a54d99b/examples/basic-otlp/src/main.rs#L35-L52) from `opentelemetry-rust` crate. diff --git a/examples/opentelemetry-push-http/otel-collector-config.yml b/examples/opentelemetry-push-custom/otel-collector-config.yml similarity index 85% rename from examples/opentelemetry-push-http/otel-collector-config.yml rename to examples/opentelemetry-push-custom/otel-collector-config.yml index 1303b66..731c5d0 100644 --- a/examples/opentelemetry-push-http/otel-collector-config.yml +++ b/examples/opentelemetry-push-custom/otel-collector-config.yml @@ -1,8 +1,6 @@ receivers: otlp: protocols: - http: - endpoint: 0.0.0.0:4318 grpc: endpoint: 0.0.0.0:4317 diff --git a/examples/opentelemetry-push-custom/src/main.rs b/examples/opentelemetry-push-custom/src/main.rs new file mode 100644 index 0000000..57ba0b2 --- /dev/null +++ b/examples/opentelemetry-push-custom/src/main.rs @@ -0,0 +1,49 @@ +use autometrics::autometrics; +use autometrics_example_util::sleep_random_duration; +use opentelemetry::metrics::MetricsError; +use opentelemetry::sdk::metrics::MeterProvider; +use opentelemetry::{runtime, Context}; +use opentelemetry_otlp::{ExportConfig, WithExportConfig}; +use std::error::Error; +use std::time::Duration; +use tokio::time::sleep; + +fn init_metrics() -> Result { + let export_config = ExportConfig { + endpoint: "http://localhost:4317".to_string(), + ..ExportConfig::default() + }; + let push_interval = Duration::from_secs(1); + opentelemetry_otlp::new_pipeline() + .metrics(runtime::Tokio) + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(export_config), + ) + .with_period(push_interval) + .build() +} + +#[autometrics] +async fn do_stuff() { + println!("Doing stuff..."); + sleep_random_duration().await; +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let meter_provider = init_metrics()?; + let cx = Context::current(); + + for _ in 0..100 { + do_stuff().await; + } + + println!("Waiting so that we could see metrics going down..."); + sleep(Duration::from_secs(10)).await; + meter_provider.force_flush(&cx)?; + + meter_provider.shutdown()?; + Ok(()) +} diff --git a/examples/opentelemetry-push-http/Cargo.toml b/examples/opentelemetry-push-http/Cargo.toml deleted file mode 100644 index 0a1b995..0000000 --- a/examples/opentelemetry-push-http/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "example-opentelemetry-push-http" -version = "0.0.0" -publish = false -edition = "2021" - -[dependencies] -autometrics = { path = "../../autometrics", features = ["opentelemetry-0_20", "otel-push-exporter-http", "otel-push-exporter-tokio"] } -autometrics-example-util = { path = "../util" } -tokio = { version = "1", features = ["full"] } diff --git a/examples/opentelemetry-push-http/src/main.rs b/examples/opentelemetry-push-http/src/main.rs deleted file mode 100644 index f6bbc69..0000000 --- a/examples/opentelemetry-push-http/src/main.rs +++ /dev/null @@ -1,27 +0,0 @@ -use autometrics::{autometrics, otel_push_exporter}; -use autometrics_example_util::sleep_random_duration; -use std::error::Error; -use std::time::Duration; -use tokio::time::sleep; - -#[autometrics] -async fn do_stuff() { - println!("Doing stuff..."); - sleep_random_duration().await; -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let _meter_provider = otel_push_exporter::init_http("http://0.0.0.0:4318")?; - // or: otel_push_exporter::init_grpc("http://0.0.0.0:4317"); - - for _ in 0..100 { - do_stuff().await; - } - - println!("Waiting so that we could see metrics going down..."); - sleep(Duration::from_secs(10)).await; - - // no need to call `.shutdown` as the returned `OtelMeterProvider` has a `Drop` implementation - Ok(()) -} diff --git a/examples/opentelemetry-push/Cargo.toml b/examples/opentelemetry-push/Cargo.toml index dd0262f..b33a36e 100644 --- a/examples/opentelemetry-push/Cargo.toml +++ b/examples/opentelemetry-push/Cargo.toml @@ -5,11 +5,6 @@ publish = false edition = "2021" [dependencies] -autometrics = { path = "../../autometrics", features = ["opentelemetry-0_20"] } +autometrics = { path = "../../autometrics", features = ["opentelemetry-0_20", "otel-push-exporter-http", "otel-push-exporter-tokio"] } autometrics-example-util = { path = "../util" } -# Note that the version of the opentelemetry crate MUST match -# the version used by autometrics -opentelemetry = { version = "0.20", features = ["metrics", "rt-tokio"] } -opentelemetry-otlp = { version = "0.13", features = ["tonic", "metrics"] } -opentelemetry-semantic-conventions = { version = "0.12.0" } tokio = { version = "1", features = ["full"] } diff --git a/examples/opentelemetry-push/README.md b/examples/opentelemetry-push/README.md index 020afd8..7822135 100644 --- a/examples/opentelemetry-push/README.md +++ b/examples/opentelemetry-push/README.md @@ -1,12 +1,14 @@ # Autometrics + OTLP push controller -This example demonstrates how you can push autometrics via OTLP gRPC protocol to the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or another OTel-compatible solution. +This example demonstrates how you can push autometrics via OTLP HTTP and GRPC protocol +to the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or another OTel-compatible solution. It +uses the Autometrics provided wrapper to achieve this. ## Running the example ### Start a basic OTEL-Collector -You can use the [`otel-collector-config.yml`](./otel-collector-config.yml) file to start an otel-collector container that listens on 0.0.0.0:4317 for incoming otlp-gRPC traffic, and exports the metrics it receives to stdout. +You can use the [`otel-collector-config.yml`](./otel-collector-config.yml) file to start an otel-collector container that listens on 0.0.0.0:4317 for incoming otlp-gRPC traffic as well as on 0.0.0.0:4318 for incoming otlp-http traffic, and exports the metrics it receives to stdout. Run the following command in a second terminal to start a container in interactive mode: @@ -14,7 +16,7 @@ Run the following command in a second terminal to start a container in interacti docker run -it --name otel-col \ -p 4317:4317 -p 13133:13133 \ -v $PWD/otel-collector-config.yml:/etc/otelcol/config.yaml \ - otel/opentelemetry-collector:latest + otel/opentelemetry-collector-contrib:latest ``` You should see the collector initialization output, that should end with something like: @@ -99,7 +101,3 @@ Then delete the container with ```bash docker rm otel-col ``` - -## OpenTelemetry Metrics Push Controller - -The metric push controller is implemented as from this [example](https://github.com/open-telemetry/opentelemetry-rust/blob/f20c9b40547ee20b6ec99414bb21abdd3a54d99b/examples/basic-otlp/src/main.rs#L35-L52) from `opentelemetry-rust` crate. diff --git a/examples/opentelemetry-push/otel-collector-config.yml b/examples/opentelemetry-push/otel-collector-config.yml index 731c5d0..1303b66 100644 --- a/examples/opentelemetry-push/otel-collector-config.yml +++ b/examples/opentelemetry-push/otel-collector-config.yml @@ -1,6 +1,8 @@ receivers: otlp: protocols: + http: + endpoint: 0.0.0.0:4318 grpc: endpoint: 0.0.0.0:4317 diff --git a/examples/opentelemetry-push/src/main.rs b/examples/opentelemetry-push/src/main.rs index 57ba0b2..f6bbc69 100644 --- a/examples/opentelemetry-push/src/main.rs +++ b/examples/opentelemetry-push/src/main.rs @@ -1,30 +1,9 @@ -use autometrics::autometrics; +use autometrics::{autometrics, otel_push_exporter}; use autometrics_example_util::sleep_random_duration; -use opentelemetry::metrics::MetricsError; -use opentelemetry::sdk::metrics::MeterProvider; -use opentelemetry::{runtime, Context}; -use opentelemetry_otlp::{ExportConfig, WithExportConfig}; use std::error::Error; use std::time::Duration; use tokio::time::sleep; -fn init_metrics() -> Result { - let export_config = ExportConfig { - endpoint: "http://localhost:4317".to_string(), - ..ExportConfig::default() - }; - let push_interval = Duration::from_secs(1); - opentelemetry_otlp::new_pipeline() - .metrics(runtime::Tokio) - .with_exporter( - opentelemetry_otlp::new_exporter() - .tonic() - .with_export_config(export_config), - ) - .with_period(push_interval) - .build() -} - #[autometrics] async fn do_stuff() { println!("Doing stuff..."); @@ -33,8 +12,8 @@ async fn do_stuff() { #[tokio::main] async fn main() -> Result<(), Box> { - let meter_provider = init_metrics()?; - let cx = Context::current(); + let _meter_provider = otel_push_exporter::init_http("http://0.0.0.0:4318")?; + // or: otel_push_exporter::init_grpc("http://0.0.0.0:4317"); for _ in 0..100 { do_stuff().await; @@ -42,8 +21,7 @@ async fn main() -> Result<(), Box> { println!("Waiting so that we could see metrics going down..."); sleep(Duration::from_secs(10)).await; - meter_provider.force_flush(&cx)?; - meter_provider.shutdown()?; + // no need to call `.shutdown` as the returned `OtelMeterProvider` has a `Drop` implementation Ok(()) } From c8387811a8ec0b0009cdfa2395ff11a809df4a13 Mon Sep 17 00:00:00 2001 From: Mari Date: Fri, 8 Sep 2023 11:56:23 +0200 Subject: [PATCH 3/8] add must use notice as well functions to customize timeout and period --- autometrics/src/otel_push_exporter.rs | 35 ++++++++++++++++++++++--- examples/opentelemetry-push/src/main.rs | 2 ++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/autometrics/src/otel_push_exporter.rs b/autometrics/src/otel_push_exporter.rs index 81bf060..a4cd907 100644 --- a/autometrics/src/otel_push_exporter.rs +++ b/autometrics/src/otel_push_exporter.rs @@ -1,12 +1,13 @@ use opentelemetry_api::metrics::MetricsError; -use opentelemetry_otlp::OtlpMetricPipeline; use opentelemetry_otlp::{ExportConfig, Protocol, WithExportConfig}; +use opentelemetry_otlp::{OtlpMetricPipeline, OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT}; use opentelemetry_sdk::metrics::MeterProvider; use opentelemetry_sdk::runtime; use std::ops::Deref; use std::time::Duration; #[repr(transparent)] +#[must_use = "Assign this to a unused variable instead: `let _meter = ...` (NOT `let _ = ...`), as else it will be dropped immediately - which will cause it to be shut down"] pub struct OtelMeterProvider(MeterProvider); impl Deref for OtelMeterProvider { @@ -26,6 +27,19 @@ impl Drop for OtelMeterProvider { #[cfg(feature = "otel-push-exporter-http")] pub fn init_http(url: impl Into) -> Result { + init_http_with_timeout_period( + url, + Duration::from_secs(OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), + Duration::from_secs(1), + ) +} + +#[cfg(feature = "otel-push-exporter-http")] +pub fn init_http_with_timeout_period( + url: impl Into, + timeout: Duration, + period: Duration, +) -> Result { runtime() .with_exporter( opentelemetry_otlp::new_exporter() @@ -33,16 +47,30 @@ pub fn init_http(url: impl Into) -> Result) -> Result { + init_grpc_with_timeout_period( + url, + Duration::from_secs(OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), + Duration::from_secs(1), + ) +} + +#[cfg(feature = "otel-push-exporter-grpc")] +pub fn init_grpc_with_timeout_period( + url: impl Into, + timeout: Duration, + period: Duration, +) -> Result { runtime() .with_exporter( opentelemetry_otlp::new_exporter() @@ -50,10 +78,11 @@ pub fn init_grpc(url: impl Into) -> Result Result<(), Box> { + // NOTICE: the variable gets assigned to `_meter_provider` instead of just `_`, as the later case + // would cause it to be dropped immediately and thus shut down. let _meter_provider = otel_push_exporter::init_http("http://0.0.0.0:4318")?; // or: otel_push_exporter::init_grpc("http://0.0.0.0:4317"); From 89d6504265cd93549d4c6aefc2fc52a7e3db4088 Mon Sep 17 00:00:00 2001 From: Mari Date: Fri, 8 Sep 2023 12:00:37 +0200 Subject: [PATCH 4/8] fail compile if no runtime is enabled --- autometrics/src/otel_push_exporter.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/autometrics/src/otel_push_exporter.rs b/autometrics/src/otel_push_exporter.rs index a4cd907..3f39043 100644 --- a/autometrics/src/otel_push_exporter.rs +++ b/autometrics/src/otel_push_exporter.rs @@ -120,3 +120,12 @@ fn runtime() -> OtlpMetricPipeline OtlpMetricPipeline { return opentelemetry_otlp::new_pipeline().metrics(opentelemetry_sdk::runtime::AsyncStd); } + +#[cfg(not(any( + feature = "otel-push-exporter-tokio", + feature = "otel-push-exporter-tokio-current-thread", + feature = "otel-push-exporter-async-std" +)))] +fn runtime() -> ! { + compile_error!("select your runtime (`otel-push-exporter-tokio`, `otel-push-exporter-tokio-current-thread` or `otel-push-exporter-async-std`) for the autometrics push exporter or use the custom push exporter if none fit") +} From 7e41d343d2e1635988e9996c801b7867274772c3 Mon Sep 17 00:00:00 2001 From: Mari Date: Fri, 8 Sep 2023 12:22:22 +0200 Subject: [PATCH 5/8] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3008f7..618c9c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- +- New exporter, `otel_push_exporter` is now available in addition to the existing + `prometheus_exporter`. It can be used to push metrics in the OTEL format via + HTTP or gRPC to a OTEL-collector-(compatible) server. ### Changed From e0a7ab438fde5ac8e0c23cccc5bd8a9d1a153571 Mon Sep 17 00:00:00 2001 From: Mari Date: Mon, 11 Sep 2023 16:46:23 +0200 Subject: [PATCH 6/8] add feature flag documentation --- autometrics/src/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/autometrics/src/README.md b/autometrics/src/README.md index ffcaa5c..d163d0c 100644 --- a/autometrics/src/README.md +++ b/autometrics/src/README.md @@ -125,6 +125,25 @@ pub fn main() { - `prometheus-exporter` - exports a Prometheus metrics collector and exporter. This is compatible with any of the [Metrics backends](#metrics-backends) and uses `prometheus-client` by default if none are explicitly selected +### Pushing metrics + +Easily push collected metrics to a OpenTelemetry collector and compatible software. +Combine one of the transport feature flags together with your runtime feature flag: + +**Transport feature flags**: + +- `otel-push-exporter-http` - metrics sent over HTTP(s) using `hyper` +- `otel-push-exporter-grpc` - metrics sent over gRPC using `tonic` + +**Runtime feature flags**: + +- `otel-push-exporter-tokio` - tokio +- `otel-push-exporter-tokio-current-thread` - tokio with `flavor = "current_thread"` +- `otel-push-exporter-async-std` - async-std + +If you require more customization than these offered feature flags, enable just +`otel-push-exporter` and follow the [example](https://github.com/autometrics-dev/autometrics-rs/tree/main/examples/opentelemetry-push-custom). + ### Metrics backends > If you are exporting metrics yourself rather than using the `prometheus-exporter`, you must ensure that you are using the exact same version of the metrics library as `autometrics` (and it must come from `crates.io` rather than git or another source). If not, the autometrics metrics will not appear in your exported metrics. From 957de34d0ab797626b1d23d646b145d328d9efa1 Mon Sep 17 00:00:00 2001 From: Mari Date: Tue, 12 Sep 2023 15:23:45 +0200 Subject: [PATCH 7/8] add documentation --- autometrics/src/otel_push_exporter.rs | 53 +++++++++++++++++++++------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/autometrics/src/otel_push_exporter.rs b/autometrics/src/otel_push_exporter.rs index 3f39043..be6226a 100644 --- a/autometrics/src/otel_push_exporter.rs +++ b/autometrics/src/otel_push_exporter.rs @@ -6,6 +6,7 @@ use opentelemetry_sdk::runtime; use std::ops::Deref; use std::time::Duration; +/// Newtype struct holding a [`MeterProvider`] with a custom `Drop` implementation to automatically clean up itself #[repr(transparent)] #[must_use = "Assign this to a unused variable instead: `let _meter = ...` (NOT `let _ = ...`), as else it will be dropped immediately - which will cause it to be shut down"] pub struct OtelMeterProvider(MeterProvider); @@ -25,15 +26,29 @@ impl Drop for OtelMeterProvider { } } +/// Initialize the OpenTelemetry push exporter using HTTP transport. +/// +/// # Interval and timeout +/// This function uses the environment variables `OTEL_METRIC_EXPORT_TIMEOUT` and `OTEL_METRIC_EXPORT_INTERVAL` +/// to configure the timeout and interval respectively. If you want to customize those +/// from within code, consider using [`init_http_with_timeout_period`]. #[cfg(feature = "otel-push-exporter-http")] pub fn init_http(url: impl Into) -> Result { - init_http_with_timeout_period( - url, - Duration::from_secs(OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), - Duration::from_secs(1), - ) + runtime() + .with_exporter( + opentelemetry_otlp::new_exporter() + .http() + .with_export_config(ExportConfig { + endpoint: url.into(), + protocol: Protocol::HttpBinary, + ..Default::default() + }), + ) + .build() + .map(OtelMeterProvider) } +/// Initialize the OpenTelemetry push exporter using HTTP transport with customized `timeout` and `period`. #[cfg(feature = "otel-push-exporter-http")] pub fn init_http_with_timeout_period( url: impl Into, @@ -56,15 +71,29 @@ pub fn init_http_with_timeout_period( .map(OtelMeterProvider) } +/// Initialize the OpenTelemetry push exporter using gRPC transport. +/// +/// # Interval and timeout +/// This function uses the environment variables `OTEL_METRIC_EXPORT_TIMEOUT` and `OTEL_METRIC_EXPORT_INTERVAL` +/// to configure the timeout and interval respectively. If you want to customize those +/// from within code, consider using [`init_grpc_with_timeout_period`]. #[cfg(feature = "otel-push-exporter-grpc")] pub fn init_grpc(url: impl Into) -> Result { - init_grpc_with_timeout_period( - url, - Duration::from_secs(OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), - Duration::from_secs(1), - ) + runtime() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(ExportConfig { + endpoint: url.into(), + protocol: Protocol::Grpc, + ..Default::default() + }), + ) + .build() + .map(OtelMeterProvider) } +/// Initialize the OpenTelemetry push exporter using gRPC transport with customized `timeout` and `period`. #[cfg(feature = "otel-push-exporter-grpc")] pub fn init_grpc_with_timeout_period( url: impl Into, @@ -89,10 +118,10 @@ pub fn init_grpc_with_timeout_period( #[cfg(all( feature = "otel-push-exporter-tokio", - not(any( + any( feature = "otel-push-exporter-tokio-current-thread", feature = "otel-push-exporter-async-std" - )) + ) ))] fn runtime() -> OtlpMetricPipeline { return opentelemetry_otlp::new_pipeline().metrics(opentelemetry_sdk::runtime::Tokio); From 9c870dd98a95119a60e58ff6b20d45503d14bf37 Mon Sep 17 00:00:00 2001 From: Mari Date: Tue, 12 Sep 2023 15:42:33 +0200 Subject: [PATCH 8/8] re-add the `not` i accidently added --- autometrics/src/otel_push_exporter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autometrics/src/otel_push_exporter.rs b/autometrics/src/otel_push_exporter.rs index be6226a..a475e7e 100644 --- a/autometrics/src/otel_push_exporter.rs +++ b/autometrics/src/otel_push_exporter.rs @@ -118,10 +118,10 @@ pub fn init_grpc_with_timeout_period( #[cfg(all( feature = "otel-push-exporter-tokio", - any( + not(any( feature = "otel-push-exporter-tokio-current-thread", feature = "otel-push-exporter-async-std" - ) + )) ))] fn runtime() -> OtlpMetricPipeline { return opentelemetry_otlp::new_pipeline().metrics(opentelemetry_sdk::runtime::Tokio);