From 66000d4fcdcdc12e0cda4c9452e13e99b0dd0904 Mon Sep 17 00:00:00 2001 From: Thiago Nobayashi Date: Tue, 29 Nov 2022 18:07:21 -0300 Subject: [PATCH 1/3] feat: create base config for observability PoC --- crates/topos-tce/Cargo.toml | 4 +-- crates/topos-tce/src/main.rs | 40 +++++++++++++++++++----- tools/.env | 3 ++ tools/docker-compose.yml | 49 ++++++++++++++++++++++-------- tools/otel-collector-config.yaml | 52 ++++++++++++++++++++++++++++++++ tools/prometheus.yaml | 6 ++++ 6 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 tools/.env create mode 100644 tools/otel-collector-config.yaml create mode 100644 tools/prometheus.yaml diff --git a/crates/topos-tce/Cargo.toml b/crates/topos-tce/Cargo.toml index cd3c1a573..ab3e91add 100644 --- a/crates/topos-tce/Cargo.toml +++ b/crates/topos-tce/Cargo.toml @@ -25,8 +25,8 @@ topos-tce-broadcast = { path = "../topos-tce-broadcast" } bincode = "1.3.3" clap = { version = "3.0.10", features = ["derive", "env"] } lazy_static = "1.4" -opentelemetry = { version = "0.18", default-features = false, features = ["trace", "rt-tokio"] } -opentelemetry-jaeger = { version = "0.17", features = ["rt-tokio"] } +opentelemetry = { version = "0.18", features = ["rt-tokio", "metrics"] } +opentelemetry-otlp = { version = " 0.11", features = ["tonic", "metrics"] } pretty_env_logger = "0.4.0" tracing-opentelemetry = "0.18.0" tracing-attributes = "0.1.23" diff --git a/crates/topos-tce/src/main.rs b/crates/topos-tce/src/main.rs index ae858aeea..87e97fb39 100644 --- a/crates/topos-tce/src/main.rs +++ b/crates/topos-tce/src/main.rs @@ -5,9 +5,15 @@ mod storage; use crate::app_context::AppContext; use crate::cli::AppArgs; use clap::Parser; +use opentelemetry::global; +use opentelemetry::runtime; +use opentelemetry::sdk::export::metrics::aggregation::cumulative_temporality_selector; +use opentelemetry::sdk::metrics::selectors; use opentelemetry::sdk::trace::{self, RandomIdGenerator, Sampler}; use opentelemetry::sdk::Resource; -use opentelemetry::{global, KeyValue}; +use opentelemetry::KeyValue; +use opentelemetry_otlp::{ExportConfig, Protocol, WithExportConfig}; +use std::time::Duration; use tce_store::{Store, StoreConfig}; use tokio::spawn; use topos_p2p::{utils::local_key_pair, Multiaddr}; @@ -25,12 +31,13 @@ async fn main() { let peer_id = key.public().to_peer_id(); tracing::Span::current().record("peer_id", &peer_id.to_string()); - let tracer = opentelemetry_jaeger::new_agent_pipeline() - .with_endpoint(args.jaeger_agent.clone()) - .with_service_name(args.jaeger_service_name.clone()) - .with_max_packet_size(1500) - .with_auto_split_batch(true) - .with_instrumentation_library_tags(false) + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint("http://otel-collector:4317"), + ) .with_trace_config( trace::config() .with_sampler(Sampler::AlwaysOn) @@ -47,6 +54,25 @@ async fn main() { .install_batch(opentelemetry::runtime::Tokio) .unwrap(); + let export_config = ExportConfig { + endpoint: "http://otel-collector:4317".to_string(), + timeout: Duration::from_secs(3), + protocol: Protocol::Grpc, + }; + + let _meter = opentelemetry_otlp::new_pipeline() + .metrics( + selectors::simple::inexpensive(), + cumulative_temporality_selector(), + runtime::Tokio, + ) + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(export_config), + ) + .build(); + #[cfg(feature = "log-json")] let formatting_layer = tracing_subscriber::fmt::layer().json(); diff --git a/tools/.env b/tools/.env new file mode 100644 index 000000000..c74e904a0 --- /dev/null +++ b/tools/.env @@ -0,0 +1,3 @@ +# TCE default args +TOOLCHAIN_VERSION=stable +GITHUB_TOKEN=${GITHUB_TOKEN} diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 363b19468..b59b1f0b1 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -4,21 +4,46 @@ services: container_name: jaeger restart: always ports: - - "6831/udp" - - "6832/udp" - - "16685" - "16686:16686" + - "14250:14250" + - "16685:16685" + + otel-collector: + image: otel/opentelemetry-collector-contrib-dev:latest + restart: always + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "55679:55679" # zpages extension + depends_on: + - jaeger + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + restart: always + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + boot: container_name: boot - image: ghcr.io/toposware/tce:pr-4 + image: tce:develop init: true build: context: ../ args: - - TOOLCHAIN_VERSION=stable - - GITHUB_TOKEN + - TOOLCHAIN_VERSION=${TOOLCHAIN_VERSION} + - GITHUB_TOKEN=${GITHUB_TOKEN} depends_on: - - jaeger + - otel-collector ports: - "9090" environment: @@ -36,15 +61,15 @@ services: - TCE_TRBP_DELIVERY_THRESHOLD=3 peer: - image: ghcr.io/toposware/tce:pr-4 + image: tce:develop init: true build: context: ../ args: - - TOOLCHAIN_VERSION=stable - - GITHUB_TOKEN + - TOOLCHAIN_VERSION=${TOOLCHAIN_VERSION} + - GITHUB_TOKEN=${GITHUB_TOKEN} depends_on: - - jaeger + - otel-collector - boot ports: - "9090" @@ -72,7 +97,7 @@ services: - RUST_LOG=debug - TARGET_NODES_PATH=/nodes.json depends_on: - - jaeger + - otel-collector - peer volumes: - ./peer_nodes.json:/nodes.json diff --git a/tools/otel-collector-config.yaml b/tools/otel-collector-config.yaml new file mode 100644 index 000000000..cb79731fa --- /dev/null +++ b/tools/otel-collector-config.yaml @@ -0,0 +1,52 @@ +receivers: + otlp: + protocols: + grpc: + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + const_labels: + label1: value1 + resource_to_telemetry_conversion: + enabled: true + + logging: + + jaeger: + endpoint: jaeger:14250 + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging, prometheus] + diff --git a/tools/prometheus.yaml b/tools/prometheus.yaml new file mode 100644 index 000000000..a8477547e --- /dev/null +++ b/tools/prometheus.yaml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['otel-collector:8889'] + - targets: ['otel-collector:8888'] From 8896762b023db2cf68d3770ed7c4ad13b3c1dc3e Mon Sep 17 00:00:00 2001 From: Thiago Nobayashi Date: Wed, 30 Nov 2022 17:48:51 -0300 Subject: [PATCH 2/3] fix: remove logging --- tools/otel-collector-config.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/otel-collector-config.yaml b/tools/otel-collector-config.yaml index cb79731fa..8d0522c08 100644 --- a/tools/otel-collector-config.yaml +++ b/tools/otel-collector-config.yaml @@ -21,8 +21,6 @@ exporters: resource_to_telemetry_conversion: enabled: true - logging: - jaeger: endpoint: jaeger:14250 tls: @@ -44,9 +42,9 @@ service: traces: receivers: [otlp] processors: [batch] - exporters: [logging, jaeger] + exporters: [jaeger] metrics: receivers: [otlp] processors: [batch] - exporters: [logging, prometheus] + exporters: [prometheus] From 451c094ad5100beda6819d34f6297b822ae4a4d6 Mon Sep 17 00:00:00 2001 From: Thiago Nobayashi Date: Wed, 28 Dec 2022 11:45:13 -0300 Subject: [PATCH 3/3] feat: temporary changes to test local observability stack --- crates/topos-tce/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/topos-tce/src/main.rs b/crates/topos-tce/src/main.rs index 87e97fb39..a3837194a 100644 --- a/crates/topos-tce/src/main.rs +++ b/crates/topos-tce/src/main.rs @@ -55,7 +55,7 @@ async fn main() { .unwrap(); let export_config = ExportConfig { - endpoint: "http://otel-collector:4317".to_string(), + endpoint: "http://otel-collector-opentelemetry-collector:4317".to_string(), timeout: Duration::from_secs(3), protocol: Protocol::Grpc, };