From c6a63c0c85e27f6cd9fcf0026b2e1b624d4489bb Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 12:30:52 +0100 Subject: [PATCH 1/9] Added subject transforms to stream source config --- src/NATS.Client.JetStream/Models/StreamSource.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NATS.Client.JetStream/Models/StreamSource.cs b/src/NATS.Client.JetStream/Models/StreamSource.cs index 5c84a38da..5c19e7f4f 100644 --- a/src/NATS.Client.JetStream/Models/StreamSource.cs +++ b/src/NATS.Client.JetStream/Models/StreamSource.cs @@ -39,11 +39,11 @@ public record StreamSource public string FilterSubject { get; set; } = default!; /// - /// Map matching subjects according to this transform destination + /// Subject transforms to apply to matching messages /// - [System.Text.Json.Serialization.JsonPropertyName("subject_transform_dest")] + [System.Text.Json.Serialization.JsonPropertyName("subject_transforms")] [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingDefault)] - public string SubjectTransformDest { get; set; } = default!; + public System.Collections.Generic.ICollection SubjectTransforms { get; set; } = default!; [System.Text.Json.Serialization.JsonPropertyName("external")] [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingDefault)] From d925091b793017bc9b8338dd269733f94e311362 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 12:50:26 +0100 Subject: [PATCH 2/9] Fixed nats-cli download for tests --- .github/workflows/perf.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index e4c4c7413..85813440c 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -23,7 +23,9 @@ jobs: steps: - name: Install nats run: | - rel=$(curl -s https://api.github.com/repos/nats-io/natscli/releases/latest | jq -r .tag_name | sed s/v//) + # latest 0.1.1 doesn't have binaries + # rel=$(curl -s https://api.github.com/repos/nats-io/natscli/releases/latest | jq -r .tag_name | sed s/v//) + rel=0.1.0 wget https://github.com/nats-io/natscli/releases/download/v$rel/nats-$rel-linux-amd64.zip unzip nats-$rel-linux-amd64.zip sudo mv nats-$rel-linux-amd64/nats /usr/local/bin From e3ea6cdb7c4ad932d4b579eef823b2eb87888386 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 14:38:38 +0100 Subject: [PATCH 3/9] binaries.nats.dev isn't working for main --- .github/workflows/perf.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 85813440c..4d4469078 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -14,7 +14,7 @@ jobs: matrix: config: - branch: dev - - branch: main + #- branch: main runs-on: ubuntu-latest env: DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cee691711..e73a39c2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: matrix: config: - branch: dev - - branch: main + #- branch: main runs-on: ubuntu-latest env: DOTNET_CLI_TELEMETRY_OPTOUT: 1 From 3a2081205c3a3e4e49b51af2e7cbb1f235547a28 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 17:17:44 +0100 Subject: [PATCH 4/9] binaries.nats.dev should be working for main --- .github/workflows/perf.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 4d4469078..85813440c 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -14,7 +14,7 @@ jobs: matrix: config: - branch: dev - #- branch: main + - branch: main runs-on: ubuntu-latest env: DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e73a39c2d..cee691711 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: matrix: config: - branch: dev - #- branch: main + - branch: main runs-on: ubuntu-latest env: DOTNET_CLI_TELEMETRY_OPTOUT: 1 From dfe6a931c5891246067868a2809db70e2f245055 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 17:21:42 +0100 Subject: [PATCH 5/9] Add release/v2.9.23 to test matrix --- .github/workflows/perf.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 85813440c..d8daff967 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: config: - - branch: dev + - branch: release/v2.9.23 - branch: main runs-on: ubuntu-latest env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cee691711..bf1528c35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: config: - - branch: dev + - branch: release/v2.9.23 - branch: main runs-on: ubuntu-latest env: From 87a4a80ffcbb2ef07cc41cde3a5462119dafb569 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 17:29:39 +0100 Subject: [PATCH 6/9] Add latest to test matrix --- .github/workflows/perf.yml | 1 + .github/workflows/test.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index d8daff967..79ac4aaf8 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -14,6 +14,7 @@ jobs: matrix: config: - branch: release/v2.9.23 + - branch: latest - branch: main runs-on: ubuntu-latest env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf1528c35..eea223ad3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,7 @@ jobs: matrix: config: - branch: release/v2.9.23 + - branch: latest - branch: main runs-on: ubuntu-latest env: @@ -55,7 +56,8 @@ jobs: fail-fast: false matrix: config: - - branch: dev + - branch: release/v2.9.23 + - branch: latest - branch: main runs-on: windows-latest env: From 5e6390f6d462f01114e9106ed518b2e91820dbc4 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 21:46:52 +0100 Subject: [PATCH 7/9] Fixed potential test flapper --- .../ConsumerConsumeTest.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index 0dec6922d..feb325cbb 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -120,14 +120,28 @@ public async Task Consume_idle_heartbeat_test() await Retry.Until( "all pull requests are received", - () => proxy.ClientFrames.Count(f => f.Message.StartsWith("PUB $JS.API.CONSUMER.MSG.NEXT.s1.c1")) == 2); + () => proxy.ClientFrames.Count(f => f.Message.StartsWith("PUB $JS.API.CONSUMER.MSG.NEXT.s1.c1")) >= 2); var msgNextRequests = proxy .ClientFrames .Where(f => f.Message.StartsWith("PUB $JS.API.CONSUMER.MSG.NEXT.s1.c1")) .ToList(); - Assert.Equal(2, msgNextRequests.Count); + // In some cases we are receiving more than two requests which + // is possible if the tests are running in a slow container and taking + // more than the timeout? Looking at the test and the code I can't make + // sense of it, really, but I'm going to assume it's fine to receive 3 pull + // requests as well as 2 since test failure reported 3 and failed once. + if (msgNextRequests.Count > 2) + { + _output.WriteLine($"Pull request count more than expected: {msgNextRequests.Count}"); + foreach (var frame in msgNextRequests) + { + _output.WriteLine($"PULL REQUEST: {frame}"); + } + } + + Assert.True(msgNextRequests.Count is 2 or 3); // Pull requests foreach (var frame in msgNextRequests) From 80b3e035ba894fef0ef63124cd448c01c03e117a Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 21:59:51 +0100 Subject: [PATCH 8/9] Chasing test flapper --- tests/NATS.Client.Core.Tests/RequestReplyTest.cs | 4 +++- tests/NATS.Client.TestUtilities/NatsServer.cs | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/NATS.Client.Core.Tests/RequestReplyTest.cs b/tests/NATS.Client.Core.Tests/RequestReplyTest.cs index 8d0b081ae..699c7efc1 100644 --- a/tests/NATS.Client.Core.Tests/RequestReplyTest.cs +++ b/tests/NATS.Client.Core.Tests/RequestReplyTest.cs @@ -12,7 +12,9 @@ public class RequestReplyTest [Fact] public async Task Simple_request_reply_test() { - await using var server = NatsServer.Start(); + // Trace to hunt flapper! + await using var server = NatsServer.StartWithTrace(_output); + await using var nats = server.CreateClientConnection(); var sub = await nats.SubscribeAsync("foo"); diff --git a/tests/NATS.Client.TestUtilities/NatsServer.cs b/tests/NATS.Client.TestUtilities/NatsServer.cs index 40c43f96c..62a84a51b 100644 --- a/tests/NATS.Client.TestUtilities/NatsServer.cs +++ b/tests/NATS.Client.TestUtilities/NatsServer.cs @@ -151,7 +151,13 @@ public static NatsServer StartJS(ITestOutputHelper outputHelper, TransportType t public static NatsServer Start() => Start(new NullOutputHelper(), TransportType.Tcp); - public static NatsServer Start(ITestOutputHelper outputHelper) => Start(outputHelper, TransportType.Tcp); + public static NatsServer StartWithTrace(ITestOutputHelper outputHelper) + => Start( + outputHelper, + new NatsServerOptsBuilder() + .Trace() + .UseTransport(TransportType.Tcp) + .Build()); public static NatsServer Start(ITestOutputHelper outputHelper, TransportType transportType) => Start(outputHelper, new NatsServerOptsBuilder().UseTransport(transportType).Build()); From d85e87c462e91ac4b48d55167a8d02ba8f4fe511 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 27 Sep 2023 22:02:50 +0100 Subject: [PATCH 9/9] Chasing test flapper --- tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs | 5 +++-- tests/NATS.Client.TestUtilities/NatsServer.cs | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index feb325cbb..97fcf9106 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -79,7 +79,7 @@ await Retry.Until( public async Task Consume_idle_heartbeat_test() { var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - await using var server = NatsServer.StartJS(); + await using var server = NatsServer.StartJSWithTrace(_output); var (nats, proxy) = server.CreateProxiedClientConnection(); @@ -141,7 +141,8 @@ await Retry.Until( } } - Assert.True(msgNextRequests.Count is 2 or 3); + // Still fail and check traces if it happens again + Assert.True(msgNextRequests.Count is 2); // Pull requests foreach (var frame in msgNextRequests) diff --git a/tests/NATS.Client.TestUtilities/NatsServer.cs b/tests/NATS.Client.TestUtilities/NatsServer.cs index 62a84a51b..1805bd67b 100644 --- a/tests/NATS.Client.TestUtilities/NatsServer.cs +++ b/tests/NATS.Client.TestUtilities/NatsServer.cs @@ -142,6 +142,14 @@ public int ConnectionPort public static NatsServer StartJS() => StartJS(new NullOutputHelper(), TransportType.Tcp); + public static NatsServer StartJSWithTrace(ITestOutputHelper outputHelper) => Start( + outputHelper: outputHelper, + opts: new NatsServerOptsBuilder() + .UseTransport(TransportType.Tcp) + .Trace() + .UseJetStream() + .Build()); + public static NatsServer StartJS(ITestOutputHelper outputHelper, TransportType transportType) => Start( outputHelper: outputHelper, opts: new NatsServerOptsBuilder()