From 2c692d4339d5d942c6119138c39187d88ff18c80 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 16 Feb 2023 05:11:27 -0800 Subject: [PATCH] Add postgres receiver (#2564) --- docs/components.md | 1 + go.mod | 1 + go.sum | 2 + internal/components/components.go | 2 + internal/components/components_test.go | 1 + tests/receivers/postgresql/postgresql_test.go | 58 ++++++++++ .../testdata/all_metrics_config.yaml | 26 +++++ .../testdata/resource_metrics/all.yaml | 109 ++++++++++++++++++ .../postgresql/testdata/server/Dockerfile | 7 ++ .../testdata/server/initPostgresqlDB.sql | 26 +++++ 10 files changed, 233 insertions(+) create mode 100644 tests/receivers/postgresql/postgresql_test.go create mode 100644 tests/receivers/postgresql/testdata/all_metrics_config.yaml create mode 100644 tests/receivers/postgresql/testdata/resource_metrics/all.yaml create mode 100644 tests/receivers/postgresql/testdata/server/Dockerfile create mode 100644 tests/receivers/postgresql/testdata/server/initPostgresqlDB.sql diff --git a/docs/components.md b/docs/components.md index c90bf73ffb8..086f1086d40 100644 --- a/docs/components.md +++ b/docs/components.md @@ -32,6 +32,7 @@ The distribution offers support for the following components. | [mongodbatlas](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mongodbatlasreceiver) | [beta] | | [oracledb](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/oracledbreceiver) | [alpha] | | [otlp](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver) | [stable] | +| [postgres](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/postgresqlreceiver) | [beta] | | [prometheusexecreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusexecreceiver) | [deprecated] | | [prometheusreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) | [beta] | | [receiver_creator](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/receivercreator) | [beta] | diff --git a/go.mod b/go.mod index f641867d314..41c2c26ff90 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.70.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mongodbatlasreceiver v0.70.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/oracledbreceiver v0.70.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.70.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.70.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.70.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/receivercreator v0.70.0 diff --git a/go.sum b/go.sum index 35d07d1bdea..c3f3eaa5edb 100644 --- a/go.sum +++ b/go.sum @@ -1853,6 +1853,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mongodbatlasr github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mongodbatlasreceiver v0.70.0/go.mod h1:DmBckzGHUUi1qB2f7E7M9VQvtchzI1+DoXgKjIDJd0U= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/oracledbreceiver v0.70.0 h1:17Ug4A2BoXjE9SJt67hte4PWY1anarQtxMXm+c2QuIg= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/oracledbreceiver v0.70.0/go.mod h1:jINAWAwWVty1IxYZub4UB98Vsnoi2qfFK7r603zWkTg= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.70.0 h1:Ws4JdAGRDRVsF5+Gaod+m2VaMxNwF3cB77dKZFQ4BQs= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.70.0/go.mod h1:O8Xv/zy8aFzHdb7TmWmGgTRxrOw6fTgSNcUx0LT2J64= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.70.0 h1:O249Az9ujYbN8qDUpif9XGXGsXJah/AH4PAp8vEN9nU= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.70.0/go.mod h1:YxlNYCwYY3SQjPmJlH7t/SeJEcURlTPcYFw0z3kkQvk= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.70.0 h1:XNEvkCcP3B/EXATT3GN9SlgUgvso8Pge9iNR8L45hnw= diff --git a/internal/components/components.go b/internal/components/components.go index de436fcf1b6..f5e2305fcf8 100644 --- a/internal/components/components.go +++ b/internal/components/components.go @@ -61,6 +61,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mongodbatlasreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/oracledbreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/receivercreator" @@ -141,6 +142,7 @@ func Get() (otelcol.Factories, error) { mongodbatlasreceiver.NewFactory(), oracledbreceiver.NewFactory(), otlpreceiver.NewFactory(), + postgresqlreceiver.NewFactory(), prometheusexecreceiver.NewFactory(), prometheusreceiver.NewFactory(), receivercreator.NewFactory(), diff --git a/internal/components/components_test.go b/internal/components/components_test.go index 5e1554a206b..4cb417fcc58 100644 --- a/internal/components/components_test.go +++ b/internal/components/components_test.go @@ -60,6 +60,7 @@ func TestDefaultComponents(t *testing.T) { "mongodbatlas", "oracledb", "otlp", + "postgresql", "prometheus", "prometheus_exec", "prometheus_simple", diff --git a/tests/receivers/postgresql/postgresql_test.go b/tests/receivers/postgresql/postgresql_test.go new file mode 100644 index 00000000000..a0f232933f5 --- /dev/null +++ b/tests/receivers/postgresql/postgresql_test.go @@ -0,0 +1,58 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package tests + +import ( + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/signalfx/splunk-otel-collector/tests/testutils" +) + +var postgresqldb = []testutils.Container{testutils.NewContainer().WithContext( + path.Join(".", "testdata", "server"), +).WithName("postgresqldb").WithExposedPorts("15432:5432").WillWaitForLogs("database system is ready to accept connections")} + +// This test ensures the collector can connect to a PostgreSQL DB, and properly get metrics. It's not intended to +// test the receiver itself. +func TestPostgresqlDBIntegration(t *testing.T) { + tc := testutils.NewTestcase(t) + defer tc.PrintLogsOnFailure() + defer tc.ShutdownOTLPReceiverSink() + + _, stop := tc.Containers(postgresqldb...) + defer stop() + + _, shutdown := tc.SplunkOtelCollector( + "all_metrics_config.yaml", + func(collector testutils.Collector) testutils.Collector { + return collector.WithEnv(map[string]string{ + "POSTGRESQLDB_ENDPOINT": "localhost:15432", + "POSTGRESQLDB_USERNAME": "otelu", + "POSTGRESQLDB_PASSWORD": "otelp", + }) + }, + ) + defer shutdown() + + expectedResourceMetrics := tc.ResourceMetrics("all.yaml") + require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedResourceMetrics, 30*time.Second)) +} diff --git a/tests/receivers/postgresql/testdata/all_metrics_config.yaml b/tests/receivers/postgresql/testdata/all_metrics_config.yaml new file mode 100644 index 00000000000..aa69481de28 --- /dev/null +++ b/tests/receivers/postgresql/testdata/all_metrics_config.yaml @@ -0,0 +1,26 @@ +receivers: + postgresql: + endpoint: "${POSTGRESQLDB_ENDPOINT}" + username: "${POSTGRESQLDB_USERNAME}" + password: "${POSTGRESQLDB_PASSWORD}" + databases: + - otel + - otel2 + tls: + insecure: true +exporters: + otlp: + endpoint: "${OTLP_ENDPOINT}" + tls: + insecure: true + + +service: + telemetry: + logs: + level: "debug" + pipelines: + metrics: + receivers: + - postgresql + exporters: [otlp] diff --git a/tests/receivers/postgresql/testdata/resource_metrics/all.yaml b/tests/receivers/postgresql/testdata/resource_metrics/all.yaml new file mode 100644 index 00000000000..a26e58c4c8a --- /dev/null +++ b/tests/receivers/postgresql/testdata/resource_metrics/all.yaml @@ -0,0 +1,109 @@ +resource_metrics: + - attributes: + postgresql.database.name: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.commits + type: IntMonotonicCumulativeSum + - name: postgresql.db_size + type: IntNonmonotonicCumulativeSum + - name: postgresql.rollbacks + type: IntMonotonicCumulativeSum + - name: postgresql.table.count + type: IntNonmonotonicCumulativeSum + - attributes: + postgresql.database.name: + postgresql.index.name: + postgresql.table.name: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.index.scans + type: IntMonotonicCumulativeSum + - name: postgresql.index.size + type: IntGauge + - attributes: + postgresql.database.name: + postgresql.table.name: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: toast_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_hit + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: del + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: upd + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: hot_upd + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: dead + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: live + - name: postgresql.table.size + type: IntNonmonotonicCumulativeSum + - name: postgresql.table.vacuum.count + type: IntMonotonicCumulativeSum + - attributes: {} + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.bgwriter.buffers.allocated + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.buffers.writes + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.checkpoint.count + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.duration + type: DoubleMonotonicCumulativeSum + - name: postgresql.bgwriter.maxwritten + type: IntMonotonicCumulativeSum + - name: postgresql.connection.max + type: IntGauge + - name: postgresql.database.count + type: IntNonmonotonicCumulativeSum \ No newline at end of file diff --git a/tests/receivers/postgresql/testdata/server/Dockerfile b/tests/receivers/postgresql/testdata/server/Dockerfile new file mode 100644 index 00000000000..91f468d6712 --- /dev/null +++ b/tests/receivers/postgresql/testdata/server/Dockerfile @@ -0,0 +1,7 @@ +FROM postgres:15-alpine + +ENV POSTGRES_USER=root +ENV POSTGRES_PASSWORD=otel +ENV POSTGRES_DB=otel + +COPY initPostgresqlDB.sql /docker-entrypoint-initdb.d/ diff --git a/tests/receivers/postgresql/testdata/server/initPostgresqlDB.sql b/tests/receivers/postgresql/testdata/server/initPostgresqlDB.sql new file mode 100644 index 00000000000..d5b660692e1 --- /dev/null +++ b/tests/receivers/postgresql/testdata/server/initPostgresqlDB.sql @@ -0,0 +1,26 @@ +CREATE USER otelu WITH PASSWORD 'otelp'; +GRANT SELECT ON pg_stat_database TO otelu; + +CREATE TABLE table1 ( + id serial PRIMARY KEY +); +CREATE TABLE table2 ( + id serial PRIMARY KEY +); + +CREATE DATABASE otel2 OWNER otelu; +\c otel2 +CREATE TABLE test1 ( + id serial PRIMARY KEY +); +CREATE TABLE test2 ( + id serial PRIMARY KEY +); + +CREATE INDEX otelindex ON test1(id); +CREATE INDEX otel2index ON test2(id); + +-- Generating usage of index +INSERT INTO test2 (id) +VALUES(67); +SELECT * FROM test2;