From 2d6c6a92dbfc923a08659356d8ca62889a71ee23 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Sun, 18 Aug 2024 10:28:30 +0530 Subject: [PATCH 01/27] Added healthcheck to config.taml Signed-off-by: Wise-Wizard --- cmd/jaeger/config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/jaeger/config.yaml b/cmd/jaeger/config.yaml index dba80824d2d..09eb3e05429 100644 --- a/cmd/jaeger/config.yaml +++ b/cmd/jaeger/config.yaml @@ -7,7 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: - # health_check: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: # pprof: # endpoint: 0.0.0.0:1777 # zpages: From 2b2ca685a38e828b90956109255e1ab7e9f85be9 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Mon, 19 Aug 2024 09:23:26 +0530 Subject: [PATCH 02/27] Add health ext to all configs Signed-off-by: Wise-Wizard --- cmd/jaeger/config-badger.yaml | 7 ++++++- cmd/jaeger/config-cassandra.yaml | 7 ++++++- cmd/jaeger/config-elasticsearch.yaml | 7 ++++++- cmd/jaeger/config-opensearch.yaml | 7 ++++++- cmd/jaeger/config-remote-storage.yaml | 7 ++++++- cmd/jaeger/config.yaml | 2 +- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/jaeger/config-badger.yaml b/cmd/jaeger/config-badger.yaml index e919cd4411b..dcd7f1c0ec7 100644 --- a/cmd/jaeger/config-badger.yaml +++ b/cmd/jaeger/config-badger.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [otlp] @@ -7,6 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some_store trace_storage_archive: another_store diff --git a/cmd/jaeger/config-cassandra.yaml b/cmd/jaeger/config-cassandra.yaml index e72ae33f411..556441b9800 100644 --- a/cmd/jaeger/config-cassandra.yaml +++ b/cmd/jaeger/config-cassandra.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [otlp] @@ -7,6 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml index 917357ccce9..eac74a879eb 100644 --- a/cmd/jaeger/config-elasticsearch.yaml +++ b/cmd/jaeger/config-elasticsearch.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [otlp] @@ -7,6 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-opensearch.yaml b/cmd/jaeger/config-opensearch.yaml index 14973700aa9..3c80224d8b0 100644 --- a/cmd/jaeger/config-opensearch.yaml +++ b/cmd/jaeger/config-opensearch.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [otlp] @@ -7,6 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-remote-storage.yaml b/cmd/jaeger/config-remote-storage.yaml index a5e1b1528e1..5e4dc2d2aeb 100644 --- a/cmd/jaeger/config-remote-storage.yaml +++ b/cmd/jaeger/config-remote-storage.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [otlp] @@ -7,6 +7,11 @@ service: exporters: [jaeger_storage_exporter] extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some-storage ui_config: ./cmd/jaeger/config-ui.json diff --git a/cmd/jaeger/config.yaml b/cmd/jaeger/config.yaml index 09eb3e05429..cde8b3c4de2 100644 --- a/cmd/jaeger/config.yaml +++ b/cmd/jaeger/config.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query, remote_sampling] + extensions: [jaeger_storage, jaeger_query, remote_sampling, healthcheckv2] pipelines: traces: receivers: [otlp, jaeger, zipkin] From b31bf19679dd7b01474a58808ec877d413ca3b16 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Mon, 19 Aug 2024 10:03:39 +0530 Subject: [PATCH 03/27] Added e2e test Signed-off-by: Wise-Wizard --- .../internal/integration/e2e_integration.go | 16 +++++++++++++++- cmd/jaeger/internal/integration/kafka_test.go | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index ea7459c268f..ca1aa6ab946 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -5,6 +5,7 @@ package integration import ( "context" + "encoding/json" "fmt" "io" "net/http" @@ -109,7 +110,20 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { return false } defer resp.Body.Close() - return resp.StatusCode == http.StatusOK + var healthResponse struct { + Status string `json:"status"` + } + if err := json.NewDecoder(resp.Body).Decode(&healthResponse); err != nil { + t.Logf("Failed to decode JSON response: %v", err) + return false + } + + // Check if the status field in the JSON is "StatusOK" + if healthResponse.Status != "StatusOK" { + t.Logf("Received non-StatusOK status: %s", healthResponse.Status) + return false + } + return true }, 60*time.Second, 3*time.Second, "Jaeger-v2 did not start") t.Log("Jaeger-v2 is ready") t.Cleanup(func() { diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index 45038e335b6..9baf9e2a4b4 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -98,7 +98,7 @@ func TestKafkaStorage(t *testing.T) { collector := &E2EStorageIntegration{ SkipStorageCleaner: true, ConfigFile: createConfigWithEncoding(t, "../../collector-with-kafka.yaml", test.encoding, uniqueTopic), - HealthCheckEndpoint: "http://localhost:8888/metrics", + HealthCheckEndpoint: "http://localhost:13133/status", } collector.e2eInitialize(t, "kafka") t.Log("Collector initialized") From 12851ff75fddd02ee7d33b471594de44c93797ee Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Mon, 19 Aug 2024 10:34:09 +0530 Subject: [PATCH 04/27] Added config for other storage Signed-off-by: Wise-Wizard --- cmd/jaeger/internal/integration/e2e_integration.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index ca1aa6ab946..41bfe84242e 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -110,6 +110,10 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { return false } defer resp.Body.Close() + if s.HealthCheckEndpoint == "" { + return resp.StatusCode == http.StatusOK + } + var healthResponse struct { Status string `json:"status"` } From bd5b6145174ad812101287e2ac3b43e50cf5e435 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Mon, 19 Aug 2024 10:47:48 +0530 Subject: [PATCH 05/27] Add extension to Kafka Signed-off-by: Wise-Wizard --- cmd/jaeger/collector-with-kafka.yaml | 8 ++++++++ cmd/jaeger/ingester-remote-storage.yaml | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/jaeger/collector-with-kafka.yaml b/cmd/jaeger/collector-with-kafka.yaml index a248357b978..45c3542d286 100644 --- a/cmd/jaeger/collector-with-kafka.yaml +++ b/cmd/jaeger/collector-with-kafka.yaml @@ -1,4 +1,5 @@ service: + extensions: [healthcheckv2] pipelines: traces: receivers: [otlp, jaeger] @@ -10,6 +11,13 @@ service: metrics: level: detailed +extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: + receivers: otlp: protocols: diff --git a/cmd/jaeger/ingester-remote-storage.yaml b/cmd/jaeger/ingester-remote-storage.yaml index 2f200c6533a..1655dc864f2 100644 --- a/cmd/jaeger/ingester-remote-storage.yaml +++ b/cmd/jaeger/ingester-remote-storage.yaml @@ -1,5 +1,5 @@ service: - extensions: [jaeger_storage, jaeger_query] + extensions: [jaeger_storage, jaeger_query, healthcheckv2] pipelines: traces: receivers: [kafka] @@ -12,6 +12,11 @@ service: level: debug extensions: + healthcheckv2: + use_v2: true + http: + endpoint: "0.0.0.0:13133" + grpc: jaeger_query: trace_storage: some_storage From 8a9582a81bd44ef3064d13036e7ef2a99f767951 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Wed, 21 Aug 2024 15:38:06 +0530 Subject: [PATCH 06/27] Resolved Comments Signed-off-by: Wise-Wizard --- .../{collector-with-kafka.yaml => config-kafka-collector.yaml} | 0 ...{ingester-remote-storage.yaml => config-kafka-ingester.yaml} | 2 +- cmd/jaeger/internal/integration/kafka_test.go | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) rename cmd/jaeger/{collector-with-kafka.yaml => config-kafka-collector.yaml} (100%) rename cmd/jaeger/{ingester-remote-storage.yaml => config-kafka-ingester.yaml} (95%) diff --git a/cmd/jaeger/collector-with-kafka.yaml b/cmd/jaeger/config-kafka-collector.yaml similarity index 100% rename from cmd/jaeger/collector-with-kafka.yaml rename to cmd/jaeger/config-kafka-collector.yaml diff --git a/cmd/jaeger/ingester-remote-storage.yaml b/cmd/jaeger/config-kafka-ingester.yaml similarity index 95% rename from cmd/jaeger/ingester-remote-storage.yaml rename to cmd/jaeger/config-kafka-ingester.yaml index 1655dc864f2..d425bac541c 100644 --- a/cmd/jaeger/ingester-remote-storage.yaml +++ b/cmd/jaeger/config-kafka-ingester.yaml @@ -15,7 +15,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" + endpoint: "0.0.0.0:13134" grpc: jaeger_query: trace_storage: some_storage diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index 9baf9e2a4b4..8415f4d7adb 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -110,6 +110,7 @@ func TestKafkaStorage(t *testing.T) { GetDependenciesReturnsSource: true, SkipArchiveTest: true, }, + HealthCheckEndpoint: "http://localhost:13134/status", } ingester.e2eInitialize(t, "kafka") t.Log("Ingester initialized") From 7a010c5cfd056a7cad52b1670094abdaa9a26590 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Wed, 21 Aug 2024 15:47:05 +0530 Subject: [PATCH 07/27] Edited file path Signed-off-by: Wise-Wizard --- cmd/jaeger/internal/integration/kafka_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index 8415f4d7adb..c2dacf5abf4 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -97,14 +97,14 @@ func TestKafkaStorage(t *testing.T) { collector := &E2EStorageIntegration{ SkipStorageCleaner: true, - ConfigFile: createConfigWithEncoding(t, "../../collector-with-kafka.yaml", test.encoding, uniqueTopic), + ConfigFile: createConfigWithEncoding(t, "../../config-kafka-collector.yaml", test.encoding, uniqueTopic), HealthCheckEndpoint: "http://localhost:13133/status", } collector.e2eInitialize(t, "kafka") t.Log("Collector initialized") ingester := &E2EStorageIntegration{ - ConfigFile: createConfigWithEncoding(t, "../../ingester-remote-storage.yaml", test.encoding, uniqueTopic), + ConfigFile: createConfigWithEncoding(t, "../../config-kafka-ingester.yaml", test.encoding, uniqueTopic), StorageIntegration: integration.StorageIntegration{ CleanUp: purge, GetDependenciesReturnsSource: true, From fb3dc14981ec3b2159936d4a060eb3e23530f91e Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Wed, 21 Aug 2024 18:47:53 +0530 Subject: [PATCH 08/27] Test config Signed-off-by: Wise-Wizard --- cmd/jaeger/config-kafka-ingester.yaml | 4 ++-- cmd/jaeger/internal/integration/kafka_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/jaeger/config-kafka-ingester.yaml b/cmd/jaeger/config-kafka-ingester.yaml index d425bac541c..c076a86654d 100644 --- a/cmd/jaeger/config-kafka-ingester.yaml +++ b/cmd/jaeger/config-kafka-ingester.yaml @@ -15,7 +15,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13134" + endpoint: "0.0.0.0:13133" grpc: jaeger_query: trace_storage: some_storage @@ -26,7 +26,7 @@ extensions: memory: max_traces: 100000 -receivers: +receivers: kafka: brokers: - localhost:9092 diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index c2dacf5abf4..afa0a73f78a 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -110,7 +110,7 @@ func TestKafkaStorage(t *testing.T) { GetDependenciesReturnsSource: true, SkipArchiveTest: true, }, - HealthCheckEndpoint: "http://localhost:13134/status", + HealthCheckEndpoint: "http://localhost:13133/status", } ingester.e2eInitialize(t, "kafka") t.Log("Ingester initialized") From af87debd6404d63a4133f063f14f7bd6a75dccce Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Wed, 21 Aug 2024 19:45:53 +0530 Subject: [PATCH 09/27] Add different port Signed-off-by: Wise-Wizard --- cmd/jaeger/config-kafka-collector.yaml | 2 +- cmd/jaeger/config-kafka-ingester.yaml | 2 +- cmd/jaeger/internal/integration/kafka_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/jaeger/config-kafka-collector.yaml b/cmd/jaeger/config-kafka-collector.yaml index 45c3542d286..45630ff936c 100644 --- a/cmd/jaeger/config-kafka-collector.yaml +++ b/cmd/jaeger/config-kafka-collector.yaml @@ -15,7 +15,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" + endpoint: "0.0.0.0:14133" grpc: receivers: diff --git a/cmd/jaeger/config-kafka-ingester.yaml b/cmd/jaeger/config-kafka-ingester.yaml index c076a86654d..c3b21aac2bc 100644 --- a/cmd/jaeger/config-kafka-ingester.yaml +++ b/cmd/jaeger/config-kafka-ingester.yaml @@ -15,7 +15,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" + endpoint: "localhost:13133" grpc: jaeger_query: trace_storage: some_storage diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index afa0a73f78a..14ecafaf74c 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -110,7 +110,7 @@ func TestKafkaStorage(t *testing.T) { GetDependenciesReturnsSource: true, SkipArchiveTest: true, }, - HealthCheckEndpoint: "http://localhost:13133/status", + HealthCheckEndpoint: "http://localhost:14133/status", } ingester.e2eInitialize(t, "kafka") t.Log("Ingester initialized") From 1a2d577a18bee3c3e21de143d3cffc60f16fc22d Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Thu, 22 Aug 2024 00:08:27 +0530 Subject: [PATCH 10/27] Changed port to 13133 for all storages Signed-off-by: Wise-Wizard --- cmd/jaeger/internal/integration/e2e_integration.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 41bfe84242e..922d075a4d4 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -93,7 +93,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { // Wait for the binary to start and become ready to serve requests. healthCheckEndpoint := s.HealthCheckEndpoint if healthCheckEndpoint == "" { - healthCheckEndpoint = fmt.Sprintf("http://localhost:%d/", ports.QueryHTTP) + healthCheckEndpoint = fmt.Sprintf("http://localhost:13133/status") } require.Eventually(t, func() bool { t.Logf("Checking if Jaeger-v2 is available on %s", healthCheckEndpoint) @@ -110,10 +110,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { return false } defer resp.Body.Close() - if s.HealthCheckEndpoint == "" { - return resp.StatusCode == http.StatusOK - } - + var healthResponse struct { Status string `json:"status"` } From 294ba5aacde9226a1c791c89952c67b976763cd1 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 21 Aug 2024 17:22:34 -0400 Subject: [PATCH 11/27] refactor Signed-off-by: Yuri Shkuro --- .../internal/integration/e2e_integration.go | 79 ++++++++++--------- scripts/kafka-integration-test.sh | 2 +- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 922d075a4d4..3debcabeb0f 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -89,44 +89,6 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { } t.Logf("Running command: %v", cmd.Args) require.NoError(t, cmd.Start()) - - // Wait for the binary to start and become ready to serve requests. - healthCheckEndpoint := s.HealthCheckEndpoint - if healthCheckEndpoint == "" { - healthCheckEndpoint = fmt.Sprintf("http://localhost:13133/status") - } - require.Eventually(t, func() bool { - t.Logf("Checking if Jaeger-v2 is available on %s", healthCheckEndpoint) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthCheckEndpoint, nil) - if err != nil { - t.Logf("HTTP request creation failed: %v", err) - return false - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Logf("HTTP request failed: %v", err) - return false - } - defer resp.Body.Close() - - var healthResponse struct { - Status string `json:"status"` - } - if err := json.NewDecoder(resp.Body).Decode(&healthResponse); err != nil { - t.Logf("Failed to decode JSON response: %v", err) - return false - } - - // Check if the status field in the JSON is "StatusOK" - if healthResponse.Status != "StatusOK" { - t.Logf("Received non-StatusOK status: %s", healthResponse.Status) - return false - } - return true - }, 60*time.Second, 3*time.Second, "Jaeger-v2 did not start") - t.Log("Jaeger-v2 is ready") t.Cleanup(func() { if err := cmd.Process.Kill(); err != nil { t.Errorf("Failed to kill Jaeger-v2 process: %v", err) @@ -154,6 +116,11 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { } }) + // Wait for the binary to start and become ready to serve requests. + require.Eventually(t, func() bool { return s.doHealthCheckV2(t) }, + 60*time.Second, 3*time.Second, "Jaeger-v2 did not start") + t.Log("Jaeger-v2 is ready") + s.SpanWriter, err = createSpanWriter(logger, otlpPort) require.NoError(t, err) s.SpanReader, err = createSpanReader(logger, ports.QueryGRPC) @@ -165,6 +132,42 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { }) } +func (s *E2EStorageIntegration) doHealthCheckV2(t *testing.T) bool { + healthCheckEndpoint := s.HealthCheckEndpoint + if healthCheckEndpoint == "" { + healthCheckEndpoint = "http://localhost:13133/status" + } + t.Logf("Checking if Jaeger-v2 is available on %s", healthCheckEndpoint) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthCheckEndpoint, nil) + if err != nil { + t.Logf("HTTP request creation failed: %v", err) + return false + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Logf("HTTP request failed: %v", err) + return false + } + defer resp.Body.Close() + + var healthResponse struct { + Status string `json:"status"` + } + if err := json.NewDecoder(resp.Body).Decode(&healthResponse); err != nil { + t.Logf("Failed to decode JSON response: %v", err) + return false + } + + // Check if the status field in the JSON is "StatusOK" + if healthResponse.Status != "StatusOK" { + t.Logf("Received non-StatusOK status: %s", healthResponse.Status) + return false + } + return true +} + // e2eCleanUp closes the SpanReader and SpanWriter gRPC connection. // This function should be called after all the tests are finished. func (s *E2EStorageIntegration) e2eCleanUp(t *testing.T) { diff --git a/scripts/kafka-integration-test.sh b/scripts/kafka-integration-test.sh index eecaaf2092d..e4a1d375b01 100755 --- a/scripts/kafka-integration-test.sh +++ b/scripts/kafka-integration-test.sh @@ -45,7 +45,7 @@ setup_kafka() { } dump_logs() { - echo "::group::Kafka logs" + echo "::group::🚧 🚧 🚧 Kafka logs" docker compose -f "${compose_file}" logs echo "::endgroup::" } From 486144799398f86e6ed5f61a8bb2f563ebca09f3 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 21 Aug 2024 17:56:00 -0400 Subject: [PATCH 12/27] fix Signed-off-by: Yuri Shkuro --- .../internal/integration/e2e_integration.go | 22 ++++++++++++++++--- cmd/jaeger/internal/integration/kafka_test.go | 6 ++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 3debcabeb0f..24854bc1472 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -4,6 +4,7 @@ package integration import ( + "bytes" "context" "encoding/json" "fmt" @@ -12,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -117,7 +119,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { }) // Wait for the binary to start and become ready to serve requests. - require.Eventually(t, func() bool { return s.doHealthCheckV2(t) }, + require.Eventually(t, func() bool { return s.doHealthCheck(t) }, 60*time.Second, 3*time.Second, "Jaeger-v2 did not start") t.Log("Jaeger-v2 is ready") @@ -132,7 +134,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { }) } -func (s *E2EStorageIntegration) doHealthCheckV2(t *testing.T) bool { +func (s *E2EStorageIntegration) doHealthCheck(t *testing.T) bool { healthCheckEndpoint := s.HealthCheckEndpoint if healthCheckEndpoint == "" { healthCheckEndpoint = "http://localhost:13133/status" @@ -151,11 +153,25 @@ func (s *E2EStorageIntegration) doHealthCheckV2(t *testing.T) bool { return false } defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Logf("Failed to read HTTP response body: %v", err) + return false + } + if resp.StatusCode != http.StatusOK { + t.Logf("Failed to read receive OK HTTP response: %v", string(body)) + return false + } + // for backwards compatibility with other healthchecks + if !strings.HasSuffix(healthCheckEndpoint, "/status") { + t.Logf("OK HTTP from endpoint that is not healthcheckv2") + return false + } var healthResponse struct { Status string `json:"status"` } - if err := json.NewDecoder(resp.Body).Decode(&healthResponse); err != nil { + if err := json.NewDecoder(bytes.NewReader(body)).Decode(&healthResponse); err != nil { t.Logf("Failed to decode JSON response: %v", err) return false } diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index 14ecafaf74c..ffdf1abad9a 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -63,8 +63,7 @@ func createConfigWithEncoding(t *testing.T, configFile string, targetEncoding st err = os.WriteFile(tempFile, newData, 0o600) require.NoError(t, err, "Failed to write updated config file to: %s", tempFile) - t.Logf("Configuration file with encoding '%s' and topic '%s' written to: %s", targetEncoding, uniqueTopic, tempFile) - + t.Logf("Transformed configuration file %s to %s", configFile, tempFile) return tempFile } @@ -98,7 +97,7 @@ func TestKafkaStorage(t *testing.T) { collector := &E2EStorageIntegration{ SkipStorageCleaner: true, ConfigFile: createConfigWithEncoding(t, "../../config-kafka-collector.yaml", test.encoding, uniqueTopic), - HealthCheckEndpoint: "http://localhost:13133/status", + HealthCheckEndpoint: "http://localhost:14133/status", } collector.e2eInitialize(t, "kafka") t.Log("Collector initialized") @@ -110,7 +109,6 @@ func TestKafkaStorage(t *testing.T) { GetDependenciesReturnsSource: true, SkipArchiveTest: true, }, - HealthCheckEndpoint: "http://localhost:14133/status", } ingester.e2eInitialize(t, "kafka") t.Log("Ingester initialized") From b05472547f9abba20f76b44efe1f930d28a7f26e Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 21 Aug 2024 18:16:31 -0400 Subject: [PATCH 13/27] swap port, improve logging Signed-off-by: Yuri Shkuro --- cmd/jaeger/config-kafka-collector.yaml | 2 +- cmd/jaeger/config-kafka-ingester.yaml | 3 ++- .../internal/integration/e2e_integration.go | 26 +++++++++++-------- cmd/jaeger/internal/integration/kafka_test.go | 10 ++++--- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/cmd/jaeger/config-kafka-collector.yaml b/cmd/jaeger/config-kafka-collector.yaml index 45630ff936c..45c3542d286 100644 --- a/cmd/jaeger/config-kafka-collector.yaml +++ b/cmd/jaeger/config-kafka-collector.yaml @@ -15,7 +15,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:14133" + endpoint: "0.0.0.0:13133" grpc: receivers: diff --git a/cmd/jaeger/config-kafka-ingester.yaml b/cmd/jaeger/config-kafka-ingester.yaml index c3b21aac2bc..ead18fe0616 100644 --- a/cmd/jaeger/config-kafka-ingester.yaml +++ b/cmd/jaeger/config-kafka-ingester.yaml @@ -15,8 +15,9 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "localhost:13133" + endpoint: 0.0.0.0:14133 grpc: + endpoint: 0.0.0.0:14132 jaeger_query: trace_storage: some_storage diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 24854bc1472..5c2a30241bc 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -44,6 +44,7 @@ type E2EStorageIntegration struct { SkipStorageCleaner bool ConfigFile string + BinaryName string HealthCheckEndpoint string } @@ -52,16 +53,18 @@ type E2EStorageIntegration struct { // This function should be called before any of the tests start. func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { logger := zaptest.NewLogger(t, zaptest.WrapOptions(zap.AddCaller())) + if s.BinaryName == "" { + s.BinaryName = "jaeger-v2" + } configFile := s.ConfigFile if !s.SkipStorageCleaner { configFile = createStorageCleanerConfig(t, s.ConfigFile, storage) } - configFile, err := filepath.Abs(configFile) require.NoError(t, err, "Failed to get absolute path of the config file") require.FileExists(t, configFile, "Config file does not exist at the resolved path") - t.Logf("Starting Jaeger-v2 in the background with config file %s", configFile) + t.Logf("Starting %s in the background with config file %s", s.BinaryName, configFile) outFile, err := os.OpenFile( filepath.Join(t.TempDir(), "jaeger_output_logs.txt"), @@ -69,7 +72,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { os.ModePerm, ) require.NoError(t, err) - t.Logf("Writing the Jaeger-v2 output logs into %s", outFile.Name()) + t.Logf("Writing the %s output logs into %s", s.BinaryName, outFile.Name()) errFile, err := os.OpenFile( filepath.Join(t.TempDir(), "jaeger_error_logs.txt"), @@ -77,7 +80,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { os.ModePerm, ) require.NoError(t, err) - t.Logf("Writing the Jaeger-v2 error logs into %s", errFile.Name()) + t.Logf("Writing the %s error logs into %s", s.BinaryName, errFile.Name()) cmd := exec.Cmd{ Path: "./cmd/jaeger/jaeger", @@ -93,25 +96,25 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { require.NoError(t, cmd.Start()) t.Cleanup(func() { if err := cmd.Process.Kill(); err != nil { - t.Errorf("Failed to kill Jaeger-v2 process: %v", err) + t.Errorf("Failed to kill %s process: %v", s.BinaryName, err) } if t.Failed() { // A Github Actions special annotation to create a foldable section // in the Github runner output. // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines - fmt.Println("::group::🚧 🚧 🚧 Jaeger-v2 binary logs") + fmt.Printf("::group::🚧 🚧 🚧 %s binary logs\n", s.BinaryName) outLogs, err := os.ReadFile(outFile.Name()) if err != nil { t.Errorf("Failed to read output logs: %v", err) } else { - fmt.Printf("🚧 🚧 🚧 Jaeger-v2 output logs:\n%s", outLogs) + fmt.Printf("🚧 🚧 🚧 %s output logs:\n%s", s.BinaryName, outLogs) } errLogs, err := os.ReadFile(errFile.Name()) if err != nil { t.Errorf("Failed to read error logs: %v", err) } else { - fmt.Printf("🚧 🚧 🚧 Jaeger-v2 error logs:\n%s", errLogs) + fmt.Printf("🚧 🚧 🚧 5s error logs:\n%s", s.BinaryName, errLogs) } // End of Github Actions foldable section annotation. fmt.Println("::endgroup::") @@ -120,8 +123,8 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { // Wait for the binary to start and become ready to serve requests. require.Eventually(t, func() bool { return s.doHealthCheck(t) }, - 60*time.Second, 3*time.Second, "Jaeger-v2 did not start") - t.Log("Jaeger-v2 is ready") + 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) + t.Logf("%s is ready", s.BinaryName) s.SpanWriter, err = createSpanWriter(logger, otlpPort) require.NoError(t, err) @@ -139,7 +142,7 @@ func (s *E2EStorageIntegration) doHealthCheck(t *testing.T) bool { if healthCheckEndpoint == "" { healthCheckEndpoint = "http://localhost:13133/status" } - t.Logf("Checking if Jaeger-v2 is available on %s", healthCheckEndpoint) + t.Logf("Checking if %s is available on %s", s.BinaryName, healthCheckEndpoint) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthCheckEndpoint, nil) @@ -239,6 +242,7 @@ func createStorageCleanerConfig(t *testing.T, configFile string, storage string) err = os.WriteFile(tempFile, newData, 0o600) require.NoError(t, err) + t.Logf("Transformed configuration file %s to %s", configFile, tempFile) return tempFile } diff --git a/cmd/jaeger/internal/integration/kafka_test.go b/cmd/jaeger/internal/integration/kafka_test.go index ffdf1abad9a..0b98d4b3b6c 100644 --- a/cmd/jaeger/internal/integration/kafka_test.go +++ b/cmd/jaeger/internal/integration/kafka_test.go @@ -95,15 +95,17 @@ func TestKafkaStorage(t *testing.T) { // but the tests are run against the ingester. collector := &E2EStorageIntegration{ - SkipStorageCleaner: true, - ConfigFile: createConfigWithEncoding(t, "../../config-kafka-collector.yaml", test.encoding, uniqueTopic), - HealthCheckEndpoint: "http://localhost:14133/status", + BinaryName: "jaeger-v2-collector", + ConfigFile: createConfigWithEncoding(t, "../../config-kafka-collector.yaml", test.encoding, uniqueTopic), + SkipStorageCleaner: true, } collector.e2eInitialize(t, "kafka") t.Log("Collector initialized") ingester := &E2EStorageIntegration{ - ConfigFile: createConfigWithEncoding(t, "../../config-kafka-ingester.yaml", test.encoding, uniqueTopic), + BinaryName: "jaeger-v2-ingester", + ConfigFile: createConfigWithEncoding(t, "../../config-kafka-ingester.yaml", test.encoding, uniqueTopic), + HealthCheckEndpoint: "http://localhost:14133/status", StorageIntegration: integration.StorageIntegration{ CleanUp: purge, GetDependenciesReturnsSource: true, From 6ffc727b5f585a14c527298aeccc40c0b7059e1d Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Wed, 21 Aug 2024 18:27:52 -0400 Subject: [PATCH 14/27] fix-typo Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/e2e_integration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 5c2a30241bc..241ce1405c4 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -114,7 +114,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { if err != nil { t.Errorf("Failed to read error logs: %v", err) } else { - fmt.Printf("🚧 🚧 🚧 5s error logs:\n%s", s.BinaryName, errLogs) + fmt.Printf("🚧 🚧 🚧 %s error logs:\n%s", s.BinaryName, errLogs) } // End of Github Actions foldable section annotation. fmt.Println("::endgroup::") From e6bfa612a015eb4d0989aaf7372d9e3b7e32b62d Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Thu, 22 Aug 2024 14:30:34 +0530 Subject: [PATCH 15/27] Modified config file Signed-off-by: Wise-Wizard --- cmd/jaeger/config-remote-storage.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/jaeger/config-remote-storage.yaml b/cmd/jaeger/config-remote-storage.yaml index 5e4dc2d2aeb..f8cc71a1a96 100644 --- a/cmd/jaeger/config-remote-storage.yaml +++ b/cmd/jaeger/config-remote-storage.yaml @@ -10,8 +10,9 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" + endpoint: 0.0.0.0:13133 grpc: + endpoint: 0.0.0.0:13132 jaeger_query: trace_storage: some-storage ui_config: ./cmd/jaeger/config-ui.json From 06220d0c79ea40f0b0b6aa257cc5fb7eb2a7d231 Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Thu, 22 Aug 2024 16:18:35 +0530 Subject: [PATCH 16/27] Move HC Call Signed-off-by: Wise-Wizard --- cmd/jaeger/internal/integration/e2e_integration.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 241ce1405c4..7c279eb3fe3 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -121,16 +121,16 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { } }) - // Wait for the binary to start and become ready to serve requests. - require.Eventually(t, func() bool { return s.doHealthCheck(t) }, - 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) - t.Logf("%s is ready", s.BinaryName) - s.SpanWriter, err = createSpanWriter(logger, otlpPort) require.NoError(t, err) s.SpanReader, err = createSpanReader(logger, ports.QueryGRPC) require.NoError(t, err) + // Wait for the binary to start and become ready to serve requests. + require.Eventually(t, func() bool { return s.doHealthCheck(t) }, + 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) + t.Logf("%s is ready", s.BinaryName) + t.Cleanup(func() { // Call e2eCleanUp to close the SpanReader and SpanWriter gRPC connection. s.e2eCleanUp(t) From 383946d8745b8ad0af22a1cb45da437a40d306bf Mon Sep 17 00:00:00 2001 From: Wise-Wizard Date: Thu, 22 Aug 2024 17:09:07 +0530 Subject: [PATCH 17/27] Move HC Call back Signed-off-by: Wise-Wizard --- cmd/jaeger/internal/integration/e2e_integration.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 7c279eb3fe3..8f07d55a66b 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -120,17 +120,17 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { fmt.Println("::endgroup::") } }) + + // Wait for the binary to start and become ready to serve requests. + require.Eventually(t, func() bool { return s.doHealthCheck(t) }, + 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) + t.Logf("%s is ready", s.BinaryName) s.SpanWriter, err = createSpanWriter(logger, otlpPort) require.NoError(t, err) s.SpanReader, err = createSpanReader(logger, ports.QueryGRPC) require.NoError(t, err) - // Wait for the binary to start and become ready to serve requests. - require.Eventually(t, func() bool { return s.doHealthCheck(t) }, - 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) - t.Logf("%s is ready", s.BinaryName) - t.Cleanup(func() { // Call e2eCleanUp to close the SpanReader and SpanWriter gRPC connection. s.e2eCleanUp(t) From 95db3c2044fec313926ebd3fb0d101c4fdb48218 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 17:07:12 -0400 Subject: [PATCH 18/27] log found traces Signed-off-by: Yuri Shkuro --- .../internal/integration/span_reader.go | 3 --- plugin/storage/integration/integration.go | 23 ++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/cmd/jaeger/internal/integration/span_reader.go b/cmd/jaeger/internal/integration/span_reader.go index 3553551de25..abd98ecd6aa 100644 --- a/cmd/jaeger/internal/integration/span_reader.go +++ b/cmd/jaeger/internal/integration/span_reader.go @@ -96,7 +96,6 @@ func (r *spanReader) GetServices(ctx context.Context) ([]string, error) { if err != nil { return []string{}, err } - r.logger.Info(fmt.Sprintf("Received %d services", len(res.Services))) return res.Services, nil } @@ -109,8 +108,6 @@ func (r *spanReader) GetOperations(ctx context.Context, query spanstore.Operatio if err != nil { return operations, err } - r.logger.Info(fmt.Sprintf("Received %d operations", len(res.Operations))) - for _, operation := range res.Operations { operations = append(operations, spanstore.Operation{ Name: operation.Name, diff --git a/plugin/storage/integration/integration.go b/plugin/storage/integration/integration.go index ab0a89f8ee4..41215d93c27 100644 --- a/plugin/storage/integration/integration.go +++ b/plugin/storage/integration/integration.go @@ -146,7 +146,27 @@ func (s *StorageIntegration) testGetServices(t *testing.T) { actual, err = s.SpanReader.GetServices(context.Background()) require.NoError(t, err) sort.Strings(actual) - return assert.ObjectsAreEqualValues(expected, actual) + t.Logf("Retrieved services: %v", actual) + match := assert.ObjectsAreEqualValues(expected, actual) + if !match && len(actual) > len(expected) || true { + // If the storage backend returns more services than expected, let's log traces for those + t.Log("🛑 Found unexpected services!") + for _, service := range actual { + traces, err := s.SpanReader.FindTraces(context.Background(), &spanstore.TraceQueryParameters{ + ServiceName: service, + }) + if err != nil { + t.Log(err) + continue + } + for _, trace := range traces { + for _, span := range trace.Spans { + t.Logf("span: Service: %s, TraceID: %s, Operation: %s", service, span.TraceID, span.OperationName) + } + } + } + } + return match }) if !assert.True(t, found) { @@ -231,6 +251,7 @@ func (s *StorageIntegration) testGetOperations(t *testing.T) { sort.Slice(actual, func(i, j int) bool { return actual[i].Name < actual[j].Name }) + t.Logf("Retrieved operations: %v", actual) return assert.ObjectsAreEqualValues(expected, actual) }) From 6b1189fe0f4f16d93a38bb6ea8987bc4b21a8608 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 18:26:26 -0400 Subject: [PATCH 19/27] fix Signed-off-by: Yuri Shkuro --- plugin/storage/grpc/factory.go | 6 ++---- plugin/storage/integration/integration.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/storage/grpc/factory.go b/plugin/storage/grpc/factory.go index 2c53587da31..1806e18f5bb 100644 --- a/plugin/storage/grpc/factory.go +++ b/plugin/storage/grpc/factory.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/viper" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "github.com/jaegertracing/jaeger/pkg/metrics" @@ -32,7 +31,6 @@ var ( // interface comformance checks type Factory struct { metricsFactory metrics.Factory logger *zap.Logger - tracerProvider trace.TracerProvider // configV1 is used for backward compatibility. it will be removed in v2. // In the main initialization logic, only configV2 is used. @@ -76,14 +74,14 @@ func (f *Factory) InitFromViper(v *viper.Viper, logger *zap.Logger) { // Initialize implements storage.Factory func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) error { f.metricsFactory, f.logger = metricsFactory, logger - f.tracerProvider = otel.GetTracerProvider() + tracerProvider := otel.GetTracerProvider() if f.configV2 == nil { f.configV2 = f.configV1.TranslateToConfigV2() } var err error - f.services, err = f.configV2.Build(logger, f.tracerProvider) + f.services, err = f.configV2.Build(logger, tracerProvider) if err != nil { return fmt.Errorf("grpc storage builder failed to create a store: %w", err) } diff --git a/plugin/storage/integration/integration.go b/plugin/storage/integration/integration.go index 41215d93c27..f13abc58c01 100644 --- a/plugin/storage/integration/integration.go +++ b/plugin/storage/integration/integration.go @@ -148,7 +148,7 @@ func (s *StorageIntegration) testGetServices(t *testing.T) { sort.Strings(actual) t.Logf("Retrieved services: %v", actual) match := assert.ObjectsAreEqualValues(expected, actual) - if !match && len(actual) > len(expected) || true { + if !match && len(actual) > len(expected) { // If the storage backend returns more services than expected, let's log traces for those t.Log("🛑 Found unexpected services!") for _, service := range actual { From cfa986470e4ecb47a18cc24b8512c222a8cc1651 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 19:48:29 -0400 Subject: [PATCH 20/27] undo Signed-off-by: Yuri Shkuro --- cmd/jaeger/config-remote-storage.yaml | 2 -- plugin/storage/grpc/factory.go | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/jaeger/config-remote-storage.yaml b/cmd/jaeger/config-remote-storage.yaml index f8cc71a1a96..e7d980d5899 100644 --- a/cmd/jaeger/config-remote-storage.yaml +++ b/cmd/jaeger/config-remote-storage.yaml @@ -11,8 +11,6 @@ extensions: use_v2: true http: endpoint: 0.0.0.0:13133 - grpc: - endpoint: 0.0.0.0:13132 jaeger_query: trace_storage: some-storage ui_config: ./cmd/jaeger/config-ui.json diff --git a/plugin/storage/grpc/factory.go b/plugin/storage/grpc/factory.go index 1806e18f5bb..2c53587da31 100644 --- a/plugin/storage/grpc/factory.go +++ b/plugin/storage/grpc/factory.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/viper" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "github.com/jaegertracing/jaeger/pkg/metrics" @@ -31,6 +32,7 @@ var ( // interface comformance checks type Factory struct { metricsFactory metrics.Factory logger *zap.Logger + tracerProvider trace.TracerProvider // configV1 is used for backward compatibility. it will be removed in v2. // In the main initialization logic, only configV2 is used. @@ -74,14 +76,14 @@ func (f *Factory) InitFromViper(v *viper.Viper, logger *zap.Logger) { // Initialize implements storage.Factory func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) error { f.metricsFactory, f.logger = metricsFactory, logger - tracerProvider := otel.GetTracerProvider() + f.tracerProvider = otel.GetTracerProvider() if f.configV2 == nil { f.configV2 = f.configV1.TranslateToConfigV2() } var err error - f.services, err = f.configV2.Build(logger, tracerProvider) + f.services, err = f.configV2.Build(logger, f.tracerProvider) if err != nil { return fmt.Errorf("grpc storage builder failed to create a store: %w", err) } From 13bf5b21d66fcb06d4da385f4931ea29eb9f2214 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 19:49:13 -0400 Subject: [PATCH 21/27] simpler Signed-off-by: Yuri Shkuro --- plugin/storage/integration/integration.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugin/storage/integration/integration.go b/plugin/storage/integration/integration.go index f13abc58c01..09bb5114cf2 100644 --- a/plugin/storage/integration/integration.go +++ b/plugin/storage/integration/integration.go @@ -147,8 +147,7 @@ func (s *StorageIntegration) testGetServices(t *testing.T) { require.NoError(t, err) sort.Strings(actual) t.Logf("Retrieved services: %v", actual) - match := assert.ObjectsAreEqualValues(expected, actual) - if !match && len(actual) > len(expected) { + if len(actual) > len(expected) { // If the storage backend returns more services than expected, let's log traces for those t.Log("🛑 Found unexpected services!") for _, service := range actual { @@ -166,7 +165,7 @@ func (s *StorageIntegration) testGetServices(t *testing.T) { } } } - return match + return assert.ObjectsAreEqualValues(expected, actual) }) if !assert.True(t, found) { From 921d56466cc59190ee3647b34a2ac6d85eae3d1e Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 19:52:41 -0400 Subject: [PATCH 22/27] revert grpc to original health check Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/grpc_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/jaeger/internal/integration/grpc_test.go b/cmd/jaeger/internal/integration/grpc_test.go index cd12984bc62..4f7c1ceefd2 100644 --- a/cmd/jaeger/internal/integration/grpc_test.go +++ b/cmd/jaeger/internal/integration/grpc_test.go @@ -4,9 +4,11 @@ package integration import ( + "fmt" "testing" "github.com/jaegertracing/jaeger/plugin/storage/integration" + "github.com/jaegertracing/jaeger/ports" ) type GRPCStorageIntegration struct { @@ -30,6 +32,8 @@ func TestGRPCStorage(t *testing.T) { s := &GRPCStorageIntegration{ E2EStorageIntegration: E2EStorageIntegration{ ConfigFile: "../../config-remote-storage.yaml", + // TODO this should be removed in favor of default health check endpoint + HealthCheckEndpoint: fmt.Sprintf("http://localhost:%d/", ports.QueryHTTP), }, } s.CleanUp = s.cleanUp From e9f2438a1742ec5b081c42eec9c6e5f427eeaad6 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 19:59:24 -0400 Subject: [PATCH 23/27] fix Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/e2e_integration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 8f07d55a66b..8fb6a73d23a 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -120,7 +120,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { fmt.Println("::endgroup::") } }) - + // Wait for the binary to start and become ready to serve requests. require.Eventually(t, func() bool { return s.doHealthCheck(t) }, 60*time.Second, 3*time.Second, "%s did not start", s.BinaryName) @@ -168,7 +168,7 @@ func (s *E2EStorageIntegration) doHealthCheck(t *testing.T) bool { // for backwards compatibility with other healthchecks if !strings.HasSuffix(healthCheckEndpoint, "/status") { t.Logf("OK HTTP from endpoint that is not healthcheckv2") - return false + return true } var healthResponse struct { From b5a675ef6c267dd968d13b6becaa89c86180e61c Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 20:18:52 -0400 Subject: [PATCH 24/27] try to minimize configuration Signed-off-by: Yuri Shkuro --- cmd/jaeger/config-badger.yaml | 3 +-- cmd/jaeger/config-cassandra.yaml | 3 +-- cmd/jaeger/config-elasticsearch.yaml | 3 +-- cmd/jaeger/config-kafka-collector.yaml | 2 -- cmd/jaeger/config-kafka-ingester.yaml | 4 ++-- cmd/jaeger/config-opensearch.yaml | 3 +-- cmd/jaeger/config-remote-storage.yaml | 2 +- cmd/jaeger/config.yaml | 3 +-- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/jaeger/config-badger.yaml b/cmd/jaeger/config-badger.yaml index dcd7f1c0ec7..5ce7ce9dbfe 100644 --- a/cmd/jaeger/config-badger.yaml +++ b/cmd/jaeger/config-badger.yaml @@ -10,8 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: + jaeger_query: trace_storage: some_store trace_storage_archive: another_store diff --git a/cmd/jaeger/config-cassandra.yaml b/cmd/jaeger/config-cassandra.yaml index 556441b9800..00b03814494 100644 --- a/cmd/jaeger/config-cassandra.yaml +++ b/cmd/jaeger/config-cassandra.yaml @@ -10,8 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: + jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml index eac74a879eb..42cf78571c6 100644 --- a/cmd/jaeger/config-elasticsearch.yaml +++ b/cmd/jaeger/config-elasticsearch.yaml @@ -10,8 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: + jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-kafka-collector.yaml b/cmd/jaeger/config-kafka-collector.yaml index 45c3542d286..ac438f30022 100644 --- a/cmd/jaeger/config-kafka-collector.yaml +++ b/cmd/jaeger/config-kafka-collector.yaml @@ -15,8 +15,6 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: receivers: otlp: diff --git a/cmd/jaeger/config-kafka-ingester.yaml b/cmd/jaeger/config-kafka-ingester.yaml index ead18fe0616..9adfc308a9c 100644 --- a/cmd/jaeger/config-kafka-ingester.yaml +++ b/cmd/jaeger/config-kafka-ingester.yaml @@ -15,9 +15,9 @@ extensions: healthcheckv2: use_v2: true http: + # use different port to avoid conflict with collector endpoint: 0.0.0.0:14133 - grpc: - endpoint: 0.0.0.0:14132 + jaeger_query: trace_storage: some_storage diff --git a/cmd/jaeger/config-opensearch.yaml b/cmd/jaeger/config-opensearch.yaml index 3c80224d8b0..19be6e941c1 100644 --- a/cmd/jaeger/config-opensearch.yaml +++ b/cmd/jaeger/config-opensearch.yaml @@ -10,8 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: + jaeger_query: trace_storage: some_storage trace_storage_archive: another_storage diff --git a/cmd/jaeger/config-remote-storage.yaml b/cmd/jaeger/config-remote-storage.yaml index e7d980d5899..7c8b3dd7228 100644 --- a/cmd/jaeger/config-remote-storage.yaml +++ b/cmd/jaeger/config-remote-storage.yaml @@ -10,7 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: 0.0.0.0:13133 + jaeger_query: trace_storage: some-storage ui_config: ./cmd/jaeger/config-ui.json diff --git a/cmd/jaeger/config.yaml b/cmd/jaeger/config.yaml index cde8b3c4de2..f068e81ef6a 100644 --- a/cmd/jaeger/config.yaml +++ b/cmd/jaeger/config.yaml @@ -10,8 +10,7 @@ extensions: healthcheckv2: use_v2: true http: - endpoint: "0.0.0.0:13133" - grpc: + # pprof: # endpoint: 0.0.0.0:1777 # zpages: From 7a381e0ea67538b550575a933e3d1b1d2bbc3162 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 20:32:27 -0400 Subject: [PATCH 25/27] improve logging Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/e2e_integration.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 8fb6a73d23a..55f1517f9d7 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -162,7 +162,7 @@ func (s *E2EStorageIntegration) doHealthCheck(t *testing.T) bool { return false } if resp.StatusCode != http.StatusOK { - t.Logf("Failed to read receive OK HTTP response: %v", string(body)) + t.Logf("HTTP response not OK: %v", string(body)) return false } // for backwards compatibility with other healthchecks @@ -175,13 +175,13 @@ func (s *E2EStorageIntegration) doHealthCheck(t *testing.T) bool { Status string `json:"status"` } if err := json.NewDecoder(bytes.NewReader(body)).Decode(&healthResponse); err != nil { - t.Logf("Failed to decode JSON response: %v", err) + t.Logf("Failed to decode JSON response '%s': %v", string(body), err) return false } // Check if the status field in the JSON is "StatusOK" if healthResponse.Status != "StatusOK" { - t.Logf("Received non-StatusOK status: %s", healthResponse.Status) + t.Logf("Received non-K status %s: %s", healthResponse.Status, string(body)) return false } return true From a7c11e9d165af5c8388eba727099411930d17da3 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 20:32:50 -0400 Subject: [PATCH 26/27] undo grpc check override Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/grpc_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/jaeger/internal/integration/grpc_test.go b/cmd/jaeger/internal/integration/grpc_test.go index 4f7c1ceefd2..cd12984bc62 100644 --- a/cmd/jaeger/internal/integration/grpc_test.go +++ b/cmd/jaeger/internal/integration/grpc_test.go @@ -4,11 +4,9 @@ package integration import ( - "fmt" "testing" "github.com/jaegertracing/jaeger/plugin/storage/integration" - "github.com/jaegertracing/jaeger/ports" ) type GRPCStorageIntegration struct { @@ -32,8 +30,6 @@ func TestGRPCStorage(t *testing.T) { s := &GRPCStorageIntegration{ E2EStorageIntegration: E2EStorageIntegration{ ConfigFile: "../../config-remote-storage.yaml", - // TODO this should be removed in favor of default health check endpoint - HealthCheckEndpoint: fmt.Sprintf("http://localhost:%d/", ports.QueryHTTP), }, } s.CleanUp = s.cleanUp From b1d04313ff670bab84b9f276df016714caec2ee2 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 22 Aug 2024 20:38:13 -0400 Subject: [PATCH 27/27] redo override Signed-off-by: Yuri Shkuro --- cmd/jaeger/internal/integration/grpc_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/jaeger/internal/integration/grpc_test.go b/cmd/jaeger/internal/integration/grpc_test.go index cd12984bc62..4f7c1ceefd2 100644 --- a/cmd/jaeger/internal/integration/grpc_test.go +++ b/cmd/jaeger/internal/integration/grpc_test.go @@ -4,9 +4,11 @@ package integration import ( + "fmt" "testing" "github.com/jaegertracing/jaeger/plugin/storage/integration" + "github.com/jaegertracing/jaeger/ports" ) type GRPCStorageIntegration struct { @@ -30,6 +32,8 @@ func TestGRPCStorage(t *testing.T) { s := &GRPCStorageIntegration{ E2EStorageIntegration: E2EStorageIntegration{ ConfigFile: "../../config-remote-storage.yaml", + // TODO this should be removed in favor of default health check endpoint + HealthCheckEndpoint: fmt.Sprintf("http://localhost:%d/", ports.QueryHTTP), }, } s.CleanUp = s.cleanUp