From 7f8575bf109e766a6b6855ddf46fa3e3af6a11b2 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 17 Nov 2023 06:59:06 -0800 Subject: [PATCH 1/8] Increase buffer size for pubsub server --- internal/eventbus/event_bus.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index 16aa33150..eb1b0528c 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -13,6 +13,8 @@ import ( "github.com/tendermint/tendermint/types" ) +var DefaultBufferCapacity = 1000 + // Subscription is a proxy interface for a pubsub Subscription. type Subscription interface { ID() string @@ -30,7 +32,7 @@ type EventBus struct { // NewDefault returns a new event bus with default options. func NewDefault(l log.Logger) *EventBus { logger := l.With("module", "eventbus") - pubsub := tmpubsub.NewServer(l, tmpubsub.BufferCapacity(0)) + pubsub := tmpubsub.NewServer(l, tmpubsub.BufferCapacity(DefaultBufferCapacity)) b := &EventBus{pubsub: pubsub} b.BaseService = *service.NewBaseService(logger, "EventBus", b) return b From 981e65253ab3e9a1b05c5277a5c51f0eedc63066 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 20 Nov 2023 10:06:31 -0800 Subject: [PATCH 2/8] Add more timeout for test failure --- internal/consensus/mempool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/consensus/mempool_test.go b/internal/consensus/mempool_test.go index 75f54a869..736590881 100644 --- a/internal/consensus/mempool_test.go +++ b/internal/consensus/mempool_test.go @@ -176,7 +176,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { headerEvent := msg.Data().(types.EventDataNewBlockHeader) n += headerEvent.NumTxs logger.Info("new transactions", "nTxs", headerEvent.NumTxs, "total", n) - case <-time.After(30 * time.Second): + case <-time.After(60 * time.Second): t.Fatal("Timed out waiting 30s to commit blocks with transactions") } } From 1effb3d0df9ce5592268e11cda89d87867952f19 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 10:10:16 -0800 Subject: [PATCH 3/8] Add more timeout --- .github/workflows/tests.yml | 2 +- Makefile | 2 +- internal/consensus/mempool_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 529e4d6b1..7a8e70e60 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,7 +99,7 @@ jobs: with: max_attempts: 2 retry_on: error - timeout_seconds: 30 + timeout_seconds: 60 command: | jobs=$(curl https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs) job_statuses=$(echo "$jobs" | jq -r '.jobs[] | .conclusion') diff --git a/Makefile b/Makefile index b4fc6c4e2..85a8b8126 100644 --- a/Makefile +++ b/Makefile @@ -347,4 +347,4 @@ $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. test-group-%:split-test-packages - cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=10m -race -covermode=atomic -coverprofile=$*.profile.out + cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=15m -race -covermode=atomic -coverprofile=$*.profile.out diff --git a/internal/consensus/mempool_test.go b/internal/consensus/mempool_test.go index 736590881..e634d4b22 100644 --- a/internal/consensus/mempool_test.go +++ b/internal/consensus/mempool_test.go @@ -177,7 +177,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { n += headerEvent.NumTxs logger.Info("new transactions", "nTxs", headerEvent.NumTxs, "total", n) case <-time.After(60 * time.Second): - t.Fatal("Timed out waiting 30s to commit blocks with transactions") + t.Fatal("Timed out waiting 60s to commit blocks with transactions") } } } From 06819929e5efcd48a5d8528e22ee1c0143c6ae8e Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 10:41:24 -0800 Subject: [PATCH 4/8] Fix test split scripts --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 85a8b8126..c1224f60d 100644 --- a/Makefile +++ b/Makefile @@ -345,6 +345,6 @@ $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) go list -f "{{ if (or .TestGoFiles .XTestGoFiles) }}{{ .ImportPath }}{{ end }}" ./... | sort > $@ split-test-packages:$(BUILDDIR)/packages.txt - split -d -n l/$(NUM_SPLIT) $< $<. + split -d -l $(NUM_SPLIT) $< $<. test-group-%:split-test-packages cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=15m -race -covermode=atomic -coverprofile=$*.profile.out From f989e293eb4db05dbe81e5d44198eef6249d0f28 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 10:43:51 -0800 Subject: [PATCH 5/8] Fix test split --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7a8e70e60..34bc0c413 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,8 @@ jobs: - name: Run Go Tests run: | NUM_SPLIT=20 - make test-group-${{matrix.part}} NUM_SPLIT=20 + make split-test-packages + make test-group-${{matrix.part}} - name: Upload coverage artifact uses: actions/upload-artifact@v3 From a5ed99e60120dec4397572ed42bd3e550129422b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 13:55:24 -0800 Subject: [PATCH 6/8] Fix unit test --- internal/consensus/state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index 3dbdf8453..f770d4998 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -2482,7 +2482,7 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { round++ // moving to the next round ensureNewRound(t, newRoundCh, height, round) - + time.Sleep(1000 * time.Millisecond) rs := cs1.GetRoundState() assert.Equal(t, rs.Step, cstypes.RoundStepPropose) // P0 does not prevote before timeoutPropose expires From 67b7153bdd69e241c6fd4043c1ccc0d46b601b4d Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 16:44:57 -0800 Subject: [PATCH 7/8] Unit test --- internal/consensus/reactor_test.go | 12 ++++++------ internal/eventbus/event_bus.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/consensus/reactor_test.go b/internal/consensus/reactor_test.go index 9220968d3..289349bd3 100644 --- a/internal/consensus/reactor_test.go +++ b/internal/consensus/reactor_test.go @@ -854,8 +854,8 @@ func TestReactorVotingPowerChange(t *testing.T) { func() bool { return previousTotalVotingPower != states[0].GetRoundState().LastValidators.TotalVotingPower() }, - 5*time.Second, - 300*time.Millisecond, + 30*time.Second, + 100*time.Millisecond, "expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), @@ -875,8 +875,8 @@ func TestReactorVotingPowerChange(t *testing.T) { func() bool { return previousTotalVotingPower != states[0].GetRoundState().LastValidators.TotalVotingPower() }, - 5*time.Second, - 300*time.Millisecond, + 30*time.Second, + 100*time.Millisecond, "expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), @@ -895,8 +895,8 @@ func TestReactorVotingPowerChange(t *testing.T) { func() bool { return previousTotalVotingPower != states[0].GetRoundState().LastValidators.TotalVotingPower() }, - 5*time.Second, - 300*time.Millisecond, + 30*time.Second, + 100*time.Millisecond, "expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index eb1b0528c..aa096aa17 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -13,7 +13,7 @@ import ( "github.com/tendermint/tendermint/types" ) -var DefaultBufferCapacity = 1000 +var DefaultBufferCapacity = 100 // Subscription is a proxy interface for a pubsub Subscription. type Subscription interface { From 26cd97a24d6fb0c8118ca8df376361888129774b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 21 Nov 2023 16:48:48 -0800 Subject: [PATCH 8/8] Unit test --- internal/consensus/state_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index f770d4998..200d792c4 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -2482,7 +2482,6 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { round++ // moving to the next round ensureNewRound(t, newRoundCh, height, round) - time.Sleep(1000 * time.Millisecond) rs := cs1.GetRoundState() assert.Equal(t, rs.Step, cstypes.RoundStepPropose) // P0 does not prevote before timeoutPropose expires