From 0419004b7b4f0bef4b2cd13da568362805598f55 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 6 Feb 2024 11:26:43 +0800 Subject: [PATCH] ci: putting tools tests in a separate chunk (#7800) close tikv/pd#7805 Signed-off-by: husharp --- .github/workflows/pd-tests.yaml | 2 +- scripts/ci-subtask.sh | 13 ++++++++---- tests/integrations/Makefile | 2 -- tools/Makefile | 10 ++++++++++ tools/pd-ctl/tests/config/config_test.go | 5 ++++- tools/pd-ctl/tests/health/health_test.go | 2 +- tools/pd-ctl/tests/hot/hot_test.go | 2 ++ .../tests/keyspace/keyspace_group_test.go | 8 ++++++++ tools/pd-ctl/tests/keyspace/keyspace_test.go | 8 +++++++- tools/pd-ctl/tests/label/label_test.go | 1 + tools/pd-ctl/tests/member/member_test.go | 1 + tools/pd-ctl/tests/operator/operator_test.go | 14 ++++++------- tools/pd-ctl/tests/region/region_test.go | 3 +++ .../pd-ctl/tests/scheduler/scheduler_test.go | 20 +++++++++---------- tools/pd-ctl/tests/store/store_test.go | 3 +++ tools/pd-ctl/tests/tso/tso_test.go | 14 ++++++++++++- .../tests/unsafe/unsafe_operation_test.go | 1 + 17 files changed, 81 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index 517a2c480e7..418fcfdfc10 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -45,7 +45,7 @@ jobs: env: WORKER_ID: ${{ matrix.worker_id }} WORKER_COUNT: 13 - JOB_COUNT: 10 # 11, 12, 13 are for other integrations jobs + JOB_COUNT: 9 # 10 is tools test, 11, 12, 13 are for other integrations jobs run: | make ci-test-job JOB_COUNT=$(($JOB_COUNT)) JOB_INDEX=$WORKER_ID mv covprofile covprofile_$WORKER_ID diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh index 40f2863e1f9..b9006dda503 100755 --- a/scripts/ci-subtask.sh +++ b/scripts/ci-subtask.sh @@ -1,13 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash # ./ci-subtask.sh ROOT_PATH=../../ -if [[ $2 -gt 10 ]]; then +if [[ $2 -gt 9 ]]; then + # run tools tests + if [[ $2 -eq 10 ]]; then + cd ./tools && make ci-test-job && cd .. && cat ./covprofile >> covprofile || exit 1 + exit + fi + + # Currently, we only have 3 integration tests, so we can hardcode the task index. integrations_dir=./tests/integrations integrations_tasks=($(find "$integrations_dir" -mindepth 1 -maxdepth 1 -type d)) - # Currently, we only have 3 integration tests, so we can hardcode the task index. for t in "${integrations_tasks[@]}"; do if [[ "$t" = "$integrations_dir/client" && $2 -eq 11 ]]; then cd ./client && make ci-test-job && cd .. && cat ./covprofile >> covprofile || exit 1 @@ -29,7 +35,6 @@ else [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30 [[ $1 == "github.com/tikv/pd/pkg/core" ]] && return 30 [[ $1 == "github.com/tikv/pd/tests/server/api" ]] && return 30 - [[ $1 == "github.com/tikv/pd/tools" ]] && return 30 [[ $1 =~ "pd/tests" ]] && return 5 return 1 } diff --git a/tests/integrations/Makefile b/tests/integrations/Makefile index ee0be1f4239..43b5945baca 100644 --- a/tests/integrations/Makefile +++ b/tests/integrations/Makefile @@ -42,8 +42,6 @@ install-tools: failpoint-enable: cd $(ROOT_PATH) && $(MAKE) failpoint-enable - go mod tidy failpoint-disable: cd $(ROOT_PATH) && $(MAKE) failpoint-disable - go mod tidy diff --git a/tools/Makefile b/tools/Makefile index d40fca2f8f4..052f8573b62 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -32,3 +32,13 @@ tidy: @ go mod tidy git diff go.mod go.sum | cat git diff --quiet go.mod go.sum + +ci-test-job: failpoint-enable + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { $(MAKE) failpoint-disable && exit 1; } + $(MAKE) failpoint-disable + +failpoint-enable: + cd $(ROOT_PATH) && $(MAKE) failpoint-enable + +failpoint-disable: + cd $(ROOT_PATH) && $(MAKE) failpoint-disable diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index 65ce3cce6d4..6776c9851b3 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -70,7 +70,8 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(configTestSuite)) } -func (suite *configTestSuite) SetupSuite() { +func (suite *configTestSuite) SetupTest() { + // use a new environment to avoid affecting other tests suite.env = pdTests.NewSchedulingTestEnvironment(suite.T()) } @@ -96,6 +97,7 @@ func (suite *configTestSuite) TearDownTest() { re.NoError(err) } suite.env.RunFuncInTwoModes(cleanFunc) + suite.env.Cleanup() } func (suite *configTestSuite) TestConfig() { @@ -877,6 +879,7 @@ func TestReplicationMode(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/health/health_test.go b/tools/pd-ctl/tests/health/health_test.go index 51bc47b5b17..9150a56c91b 100644 --- a/tools/pd-ctl/tests/health/health_test.go +++ b/tools/pd-ctl/tests/health/health_test.go @@ -33,6 +33,7 @@ func TestHealth(t *testing.T) { defer cancel() tc, err := pdTests.NewTestCluster(ctx, 3) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) tc.WaitLeader() @@ -40,7 +41,6 @@ func TestHealth(t *testing.T) { re.NoError(leaderServer.BootstrapCluster()) pdAddr := tc.GetConfig().GetClientURL() cmd := ctl.GetRootCmd() - defer tc.Destroy() client := tc.GetEtcdClient() members, err := cluster.GetMembers(client) diff --git a/tools/pd-ctl/tests/hot/hot_test.go b/tools/pd-ctl/tests/hot/hot_test.go index 7ec8315b6fd..9d8dbbd123a 100644 --- a/tools/pd-ctl/tests/hot/hot_test.go +++ b/tools/pd-ctl/tests/hot/hot_test.go @@ -405,6 +405,7 @@ func TestHistoryHotRegions(t *testing.T) { }, ) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() @@ -521,6 +522,7 @@ func TestBuckets(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, serverName string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 }) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index b9ad04a3efb..5d85f35dacf 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -41,6 +41,7 @@ func TestKeyspaceGroup(t *testing.T) { defer cancel() tc, err := pdTests.NewTestAPICluster(ctx, 1) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) tc.WaitLeader() @@ -103,6 +104,7 @@ func TestSplitKeyspaceGroup(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -157,6 +159,7 @@ func TestExternalAllocNodeWhenStart(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -196,6 +199,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -299,6 +303,7 @@ func TestMergeKeyspaceGroup(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -417,6 +422,7 @@ func TestKeyspaceGroupState(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -507,6 +513,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -594,6 +601,7 @@ func TestInPDMode(t *testing.T) { defer cancel() tc, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 62157ef968c..4c1fb2aadd5 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -51,6 +51,7 @@ func TestKeyspace(t *testing.T) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) + defer tc.Destroy() err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() @@ -115,6 +116,7 @@ func TestKeyspaceGroupUninitialized(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) tc, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer tc.Destroy() re.NoError(tc.RunInitialServers()) tc.WaitLeader() re.NoError(tc.GetLeaderServer().BootstrapCluster()) @@ -165,10 +167,14 @@ func (suite *keyspaceTestSuite) TearDownTest() { re := suite.Require() re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) +} + +func (suite *keyspaceTestSuite) TearDownSuite() { suite.cancel() + suite.cluster.Destroy() } -func (suite *keyspaceTestSuite) TestshowKeyspace() { +func (suite *keyspaceTestSuite) TestShowKeyspace() { re := suite.Require() keyspaceName := "DEFAULT" keyspaceID := uint32(0) diff --git a/tools/pd-ctl/tests/label/label_test.go b/tools/pd-ctl/tests/label/label_test.go index adfbc7f4ca1..9ba6f267ae1 100644 --- a/tools/pd-ctl/tests/label/label_test.go +++ b/tools/pd-ctl/tests/label/label_test.go @@ -36,6 +36,7 @@ func TestLabel(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, serverName string) { cfg.Replication.StrictlyMatchLabel = false }) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/member/member_test.go b/tools/pd-ctl/tests/member/member_test.go index d6888858a3b..7a2d5fc72c9 100644 --- a/tools/pd-ctl/tests/member/member_test.go +++ b/tools/pd-ctl/tests/member/member_test.go @@ -35,6 +35,7 @@ func TestMember(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 3) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/operator/operator_test.go b/tools/pd-ctl/tests/operator/operator_test.go index 335e6635948..5af73184076 100644 --- a/tools/pd-ctl/tests/operator/operator_test.go +++ b/tools/pd-ctl/tests/operator/operator_test.go @@ -232,25 +232,25 @@ func (suite *operatorTestSuite) checkOperator(cluster *pdTests.TestCluster) { }) } - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "2", "3") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "2", "3") re.NoError(err) re.Contains(string(output), "not supported") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "2", "follower", "3") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "2", "follower", "3") re.NoError(err) re.Contains(string(output), "not match") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-peer", "1", "2", "4") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-peer", "1", "2", "4") re.NoError(err) re.Contains(string(output), "is unhealthy") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "2", "leader", "4", "follower") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "2", "leader", "4", "follower") re.NoError(err) re.Contains(string(output), "is unhealthy") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "2", "follower", "leader", "3", "follower") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "2", "follower", "leader", "3", "follower") re.NoError(err) re.Contains(string(output), "invalid") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "leader", "2", "follower", "3") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "leader", "2", "follower", "3") re.NoError(err) re.Contains(string(output), "invalid") - output, err = tests.ExecuteCommand(cmd, "operator", "add", "transfer-region", "1", "2", "leader", "3", "follower") + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "add", "transfer-region", "1", "2", "leader", "3", "follower") re.NoError(err) re.Contains(string(output), "Success!") output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "operator", "remove", "1") diff --git a/tools/pd-ctl/tests/region/region_test.go b/tools/pd-ctl/tests/region/region_test.go index d14727db0ff..5df30520115 100644 --- a/tools/pd-ctl/tests/region/region_test.go +++ b/tools/pd-ctl/tests/region/region_test.go @@ -36,6 +36,7 @@ func TestRegionKeyFormat(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) + defer cluster.Destroy() re.NoError(err) err = cluster.RunInitialServers() re.NoError(err) @@ -62,6 +63,7 @@ func TestRegion(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() @@ -216,6 +218,7 @@ func TestRegionNoLeader(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index dcfa23f997f..18cfc91f2a4 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -50,7 +50,6 @@ func TestSchedulerTestSuite(t *testing.T) { func (suite *schedulerTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) - suite.env = pdTests.NewSchedulingTestEnvironment(suite.T()) suite.defaultSchedulers = []string{ "balance-leader-scheduler", "balance-region-scheduler", @@ -59,6 +58,11 @@ func (suite *schedulerTestSuite) SetupSuite() { } } +func (suite *schedulerTestSuite) SetupTest() { + // use a new environment to avoid affecting other tests + suite.env = pdTests.NewSchedulingTestEnvironment(suite.T()) +} + func (suite *schedulerTestSuite) TearDownSuite() { re := suite.Require() suite.env.Cleanup() @@ -91,13 +95,11 @@ func (suite *schedulerTestSuite) TearDownTest() { } } suite.env.RunFuncInTwoModes(cleanFunc) + suite.env.Cleanup() } func (suite *schedulerTestSuite) TestScheduler() { - // use a new environment to avoid affecting other tests - env := pdTests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkScheduler) - env.Cleanup() + suite.env.RunTestInTwoModes(suite.checkScheduler) } func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { @@ -483,7 +485,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { pdTests.MustPutStore(re, cluster, store) } re.Equal("5.2.0", leaderServer.GetClusterVersion().String()) - // After upgrading, we should not use query. + // After upgrading, we can use query. + expected1["write-leader-priorities"] = []any{"query", "byte"} checkHotSchedulerConfig(expected1) // cannot set qps as write-peer-priorities echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-peer-priorities", "query,byte"}, nil) @@ -636,10 +639,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { } func (suite *schedulerTestSuite) TestSchedulerDiagnostic() { - // use a new environment to avoid affecting other tests - env := pdTests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkSchedulerDiagnostic) - env.Cleanup() + suite.env.RunTestInTwoModes(suite.checkSchedulerDiagnostic) } func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *pdTests.TestCluster) { diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index 9cb5f795a24..b8cd5c13a79 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -44,6 +44,7 @@ func TestStore(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() @@ -492,6 +493,7 @@ func TestTombstoneStore(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() @@ -592,6 +594,7 @@ func TestStoreTLS(t *testing.T) { conf.InitialCluster = strings.ReplaceAll(conf.InitialCluster, "http", "https") }) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() diff --git a/tools/pd-ctl/tests/tso/tso_test.go b/tools/pd-ctl/tests/tso/tso_test.go index 77e1bc28e70..63816c40e7a 100644 --- a/tools/pd-ctl/tests/tso/tso_test.go +++ b/tools/pd-ctl/tests/tso/tso_test.go @@ -36,7 +36,7 @@ func TestTSO(t *testing.T) { // tso command ts := "395181938313123110" - args := []string{"-u", "127.0.0.1", "tso", ts} + args := []string{"tso", ts} output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) tsTime, err := strconv.ParseUint(ts, 10, 64) @@ -46,4 +46,16 @@ func TestTSO(t *testing.T) { physicalTime := time.Unix(int64(physical/1000), int64(physical%1000)*time.Millisecond.Nanoseconds()) str := fmt.Sprintln("system: ", physicalTime) + fmt.Sprintln("logic: ", logicalTime) re.Equal(string(output), str) + + // test with invalid address + args = []string{"-u", "127.0.0.1", "tso", ts} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + tsTime, err = strconv.ParseUint(ts, 10, 64) + re.NoError(err) + logicalTime = tsTime & logicalBits + physical = tsTime >> physicalShiftBits + physicalTime = time.Unix(int64(physical/1000), int64(physical%1000)*time.Millisecond.Nanoseconds()) + str = fmt.Sprintln("system: ", physicalTime) + fmt.Sprintln("logic: ", logicalTime) + re.Equal(string(output), str) } diff --git a/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go b/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go index 3b52de3d66a..26fb4fc46e4 100644 --- a/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go +++ b/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go @@ -30,6 +30,7 @@ func TestRemoveFailedStores(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader()