[exporter/clickhouse] Add integration test (#30209)

**Description:** <Describe what has changed.> as #29626 (comment) mentioned, this PR adds an integration test for clickhouse exporter and it brings those benefits: - Verify tables are properly created in clickhouse after any breaking change(filter out bug like [this](#29573) before merge) - Verify if any bugs occur during data insertion like missed data or insert into an error column([30210](#30210)) .etc. - Compatibility testing for `clickhouse-server:23-alpine` and `clickhouse-server:22-alpine` close #29626 --------- Co-authored-by: Tyler Helmuth <[email protected]>
open-telemetry · Mar 18, 2024 · ce10c39 · ce10c39
1 parent 1cd7cf0
commit ce10c39
Show file tree

Hide file tree

Showing 11 changed files with 834 additions and 25 deletions.
diff --git a/.chloggen/add_integration_test_for_clickhous_exporter.yaml b/.chloggen/add_integration_test_for_clickhous_exporter.yaml
@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: clickhouseexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add integration test for clickhouse exporter
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [29626]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum
diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum
diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md
@@ -342,3 +342,7 @@ service:
       processors: [ batch ]
       exporters: [ clickhouse ]
 ```
+
+## Developer tips
+
+- Make sure integration tests pass after any change of sql
diff --git a/exporter/clickhouseexporter/exporter_logs_test.go b/exporter/clickhouseexporter/exporter_logs_test.go
@@ -121,6 +121,7 @@ func TestExporter_pushLogsData(t *testing.T) {
 	})
 }
 
+// nolint:unparam // not need to check this func
 func newTestLogsExporter(t *testing.T, dsn string, fns ...func(*Config)) *logsExporter {
 	exporter, err := newLogsExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn))
 	require.NoError(t, err)
@@ -151,10 +152,18 @@ func simpleLogs(count int) plog.Logs {
 	sl.Scope().SetName("io.opentelemetry.contrib.clickhouse")
 	sl.Scope().SetVersion("1.0.0")
 	sl.Scope().Attributes().PutStr("lib", "clickhouse")
+	timestamp := time.Unix(1703498029, 0)
 	for i := 0; i < count; i++ {
 		r := sl.LogRecords().AppendEmpty()
-		r.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		r.Attributes().PutStr(conventions.AttributeServiceName, "v")
+		r.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		r.SetObservedTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		r.SetSeverityNumber(plog.SeverityNumberError2)
+		r.SetSeverityText("error")
+		r.Body().SetStr("error message")
+		r.Attributes().PutStr(conventions.AttributeServiceNamespace, "default")
+		r.SetFlags(plog.DefaultLogRecordFlags)
+		r.SetTraceID([16]byte{1, 2, 3, byte(i)})
+		r.SetSpanID([8]byte{1, 2, 3, byte(i)})
 	}
 	return logs
 }

diff --git a/exporter/clickhouseexporter/exporter_metrics_test.go b/exporter/clickhouseexporter/exporter_metrics_test.go
@@ -29,7 +29,7 @@ func TestExporter_pushMetricsData(t *testing.T) {
 			}
 			return nil
 		})
-		exporter := newTestMetricsExporter(t)
+		exporter := newTestMetricsExporter(t, defaultEndpoint)
 		mustPushMetricsData(t, exporter, simpleMetrics(1))
 
 		require.Equal(t, int32(15), items.Load())
@@ -41,7 +41,7 @@ func TestExporter_pushMetricsData(t *testing.T) {
 			}
 			return nil
 		})
-		exporter := newTestMetricsExporter(t)
+		exporter := newTestMetricsExporter(t, defaultEndpoint)
 		err := exporter.pushMetricsData(context.TODO(), simpleMetrics(2))
 		require.Error(t, err)
 	})
@@ -93,7 +93,7 @@ func TestExporter_pushMetricsData(t *testing.T) {
 			}
 			return nil
 		})
-		exporter := newTestMetricsExporter(t)
+		exporter := newTestMetricsExporter(t, defaultEndpoint)
 		mustPushMetricsData(t, exporter, simpleMetrics(1))
 
 		require.Equal(t, int32(15), items.Load())
@@ -118,14 +118,14 @@ func TestExporter_pushMetricsData(t *testing.T) {
 			}
 			return nil
 		})
-		exporter := newTestMetricsExporter(t)
+		exporter := newTestMetricsExporter(t, defaultEndpoint)
 		mustPushMetricsData(t, exporter, simpleMetrics(1))
 	})
 }
 
 func Benchmark_pushMetricsData(b *testing.B) {
 	pm := simpleMetrics(1)
-	exporter := newTestMetricsExporter(&testing.T{})
+	exporter := newTestMetricsExporter(&testing.T{}, defaultEndpoint)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
@@ -148,6 +148,7 @@ func simpleMetrics(count int) pmetric.Metrics {
 	sm.Scope().SetDroppedAttributesCount(10)
 	sm.Scope().SetName("Scope name 1")
 	sm.Scope().SetVersion("Scope version 1")
+	timestamp := time.Unix(1703498029, 0)
 	for i := 0; i < count; i++ {
 		// gauge
 		m := sm.Metrics().AppendEmpty()
@@ -156,10 +157,12 @@ func simpleMetrics(count int) pmetric.Metrics {
 		m.SetDescription("This is a gauge metrics")
 		dp := m.SetEmptyGauge().DataPoints().AppendEmpty()
 		dp.SetIntValue(int64(i))
+		dp.SetFlags(pmetric.DefaultDataPointFlags)
 		dp.Attributes().PutStr("gauge_label_1", "1")
-		dp.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		dp.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		dp.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars := dp.Exemplars().AppendEmpty()
+		exemplars.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars.SetIntValue(54)
 		exemplars.FilteredAttributes().PutStr("key", "value")
 		exemplars.FilteredAttributes().PutStr("key2", "value2")
@@ -173,10 +176,12 @@ func simpleMetrics(count int) pmetric.Metrics {
 		m.SetDescription("This is a sum metrics")
 		dp = m.SetEmptySum().DataPoints().AppendEmpty()
 		dp.SetDoubleValue(11.234)
+		dp.SetFlags(pmetric.DefaultDataPointFlags)
 		dp.Attributes().PutStr("sum_label_1", "1")
-		dp.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		dp.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		dp.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars = dp.Exemplars().AppendEmpty()
+		exemplars.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars.SetIntValue(54)
 		exemplars.FilteredAttributes().PutStr("key", "value")
 		exemplars.FilteredAttributes().PutStr("key2", "value2")
@@ -189,17 +194,18 @@ func simpleMetrics(count int) pmetric.Metrics {
 		m.SetUnit("ms")
 		m.SetDescription("This is a histogram metrics")
 		dpHisto := m.SetEmptyHistogram().DataPoints().AppendEmpty()
-		dpHisto.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		dpHisto.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		dpHisto.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		dpHisto.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		dpHisto.SetCount(1)
 		dpHisto.SetSum(1)
 		dpHisto.Attributes().PutStr("key", "value")
-		dpHisto.Attributes().PutStr("key", "value")
+		dpHisto.Attributes().PutStr("key2", "value")
 		dpHisto.ExplicitBounds().FromRaw([]float64{0, 0, 0, 0, 0})
 		dpHisto.BucketCounts().FromRaw([]uint64{0, 0, 0, 1, 0})
 		dpHisto.SetMin(0)
 		dpHisto.SetMax(1)
 		exemplars = dpHisto.Exemplars().AppendEmpty()
+		exemplars.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars.SetDoubleValue(55.22)
 		exemplars.FilteredAttributes().PutStr("key", "value")
 		exemplars.FilteredAttributes().PutStr("key2", "value2")
@@ -212,21 +218,22 @@ func simpleMetrics(count int) pmetric.Metrics {
 		m.SetUnit("ms")
 		m.SetDescription("This is a exp histogram metrics")
 		dpExpHisto := m.SetEmptyExponentialHistogram().DataPoints().AppendEmpty()
-		dpExpHisto.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		dpExpHisto.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		dpExpHisto.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		dpExpHisto.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		dpExpHisto.SetSum(1)
 		dpExpHisto.SetMin(0)
 		dpExpHisto.SetMax(1)
 		dpExpHisto.SetZeroCount(0)
 		dpExpHisto.SetCount(1)
 		dpExpHisto.Attributes().PutStr("key", "value")
-		dpExpHisto.Attributes().PutStr("key", "value")
+		dpExpHisto.Attributes().PutStr("key2", "value")
 		dpExpHisto.Negative().SetOffset(1)
 		dpExpHisto.Negative().BucketCounts().FromRaw([]uint64{0, 0, 0, 1, 0})
 		dpExpHisto.Positive().SetOffset(1)
 		dpExpHisto.Positive().BucketCounts().FromRaw([]uint64{0, 0, 0, 1, 0})
 
 		exemplars = dpExpHisto.Exemplars().AppendEmpty()
+		exemplars.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		exemplars.SetIntValue(54)
 		exemplars.FilteredAttributes().PutStr("key", "value")
 		exemplars.FilteredAttributes().PutStr("key2", "value2")
@@ -239,10 +246,10 @@ func simpleMetrics(count int) pmetric.Metrics {
 		m.SetUnit("ms")
 		m.SetDescription("This is a summary metrics")
 		summary := m.SetEmptySummary().DataPoints().AppendEmpty()
-		summary.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		summary.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		summary.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		summary.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 		summary.Attributes().PutStr("key", "value")
-		summary.Attributes().PutStr("key2", "value2")
+		summary.Attributes().PutStr("key2", "value")
 		summary.SetCount(1)
 		summary.SetSum(1)
 		quantileValues := summary.QuantileValues().AppendEmpty()
@@ -475,8 +482,9 @@ func mustPushMetricsData(t *testing.T, exporter *metricsExporter, md pmetric.Met
 	require.NoError(t, err)
 }
 
-func newTestMetricsExporter(t *testing.T) *metricsExporter {
-	exporter, err := newMetricsExporter(zaptest.NewLogger(t), withTestExporterConfig()(defaultEndpoint))
+// nolint:unparam // not need to check this func
+func newTestMetricsExporter(t *testing.T, dsn string, fns ...func(*Config)) *metricsExporter {
+	exporter, err := newMetricsExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn))
 	require.NoError(t, err)
 	require.NoError(t, exporter.start(context.TODO(), nil))
 

diff --git a/exporter/clickhouseexporter/exporter_traces_test.go b/exporter/clickhouseexporter/exporter_traces_test.go
@@ -69,15 +69,28 @@ func simpleTraces(count int) ptrace.Traces {
 	ss.SetSchemaUrl("https://opentelemetry.io/schemas/1.7.0")
 	ss.Scope().SetDroppedAttributesCount(20)
 	ss.Scope().Attributes().PutStr("lib", "clickhouse")
+	timestamp := time.Unix(1703498029, 0)
 	for i := 0; i < count; i++ {
 		s := ss.Spans().AppendEmpty()
-		s.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now()))
-		s.SetEndTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		s.SetTraceID([16]byte{1, 2, 3, byte(i)})
+		s.SetSpanID([8]byte{1, 2, 3, byte(i)})
+		s.TraceState().FromRaw("trace state")
+		s.SetParentSpanID([8]byte{1, 2, 4, byte(i)})
+		s.SetName("call db")
+		s.SetKind(ptrace.SpanKindInternal)
+		s.SetStartTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		s.SetEndTimestamp(pcommon.NewTimestampFromTime(timestamp.Add(time.Minute)))
 		s.Attributes().PutStr(conventions.AttributeServiceName, "v")
+		s.Status().SetMessage("error")
+		s.Status().SetCode(ptrace.StatusCodeError)
 		event := s.Events().AppendEmpty()
 		event.SetName("event1")
-		event.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
+		event.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
+		event.Attributes().PutStr("level", "info")
 		link := s.Links().AppendEmpty()
+		link.SetTraceID([16]byte{1, 2, 5, byte(i)})
+		link.SetSpanID([8]byte{1, 2, 5, byte(i)})
+		link.TraceState().FromRaw("error")
 		link.Attributes().PutStr("k", "v")
 	}
 	return traces

diff --git a/exporter/clickhouseexporter/go.mod b/exporter/clickhouseexporter/go.mod
@@ -5,8 +5,10 @@ go 1.21
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.21.1
 	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/jmoiron/sqlx v1.3.5
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.96.0
 	github.com/stretchr/testify v1.9.0
+	github.com/testcontainers/testcontainers-go v0.28.0
 	go.opentelemetry.io/collector/component v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/config/configopaque v1.3.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/config/configretry v0.96.1-0.20240306115632-b2693620eff6
@@ -21,15 +23,28 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/containerd v1.7.12 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/docker v25.0.3+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -39,33 +54,53 @@ require (
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.19.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.1 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/collector v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/consumer v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/extension v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/receiver v0.96.1-0.20240306115632-b2693620eff6 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
 	google.golang.org/grpc v1.62.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect