Skip to content

Commit

Permalink
Run Issue17529 100 times
Browse files Browse the repository at this point in the history 
Signed-off-by: Marek Siarkowicz <[email protected]>
  • Loading branch information
@serathius
serathius committed Jan 13, 2025
1 parent 9eb85ee commit 2fc3243
Showing 1 changed file with 161 additions and 167 deletions.
328 changes: 161 additions & 167 deletions tests/robustness/scenarios/scenarios.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,12 @@
package scenarios

import (
"path/filepath"
"testing"
"time"

"github.com/stretchr/testify/require"

"go.etcd.io/etcd/api/v3/version"
"go.etcd.io/etcd/client/pkg/v3/fileutil"
"go.etcd.io/etcd/server/v3/etcdserver"
"go.etcd.io/etcd/tests/v3/framework/e2e"
"go.etcd.io/etcd/tests/v3/robustness/client"
"go.etcd.io/etcd/tests/v3/robustness/failpoint"
"go.etcd.io/etcd/tests/v3/robustness/options"
"go.etcd.io/etcd/tests/v3/robustness/random"
"go.etcd.io/etcd/tests/v3/robustness/traffic"
)

Expand Down Expand Up @@ -71,175 +63,177 @@ type TestScenario struct {
}

func Exploratory(_ *testing.T) []TestScenario {
randomizableOptions := []e2e.EPClusterOption{
options.WithClusterOptionGroups(
options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)},
options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)},
options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}),
}

mixedVersionOptionChoices := []random.ChoiceWeight[options.ClusterOptions]{
// 60% with all members of current version
{Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60},
// 10% with 2 members of current version, 1 member last version, leader is current version
{Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
// 10% with 2 members of current version, 1 member last version, leader is last version
{Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
// 10% with 2 members of last version, 1 member current version, leader is last version
{Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
// 10% with 2 members of last version, 1 member current version, leader is current version
{Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
}
mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices))

baseOptions := []e2e.EPClusterOption{
options.WithSnapshotCount(50, 100, 1000),
options.WithSubsetOptions(randomizableOptions...),
e2e.WithGoFailEnabled(true),
// Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints.
options.WithCompactionBatchLimit(10, 100, 1000),
e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
}

if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries))
}
//randomizableOptions := []e2e.EPClusterOption{
// options.WithClusterOptionGroups(
// options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)},
// options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)},
// options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}),
//}
//
//mixedVersionOptionChoices := []random.ChoiceWeight[options.ClusterOptions]{
// // 60% with all members of current version
// {Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60},
// // 10% with 2 members of current version, 1 member last version, leader is current version
// {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
// // 10% with 2 members of current version, 1 member last version, leader is last version
// {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
// // 10% with 2 members of last version, 1 member current version, leader is last version
// {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10},
// // 10% with 2 members of last version, 1 member current version, leader is current version
// {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10},
//}
//mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices))
//
//baseOptions := []e2e.EPClusterOption{
// options.WithSnapshotCount(50, 100, 1000),
// options.WithSubsetOptions(randomizableOptions...),
// e2e.WithGoFailEnabled(true),
// // Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints.
// options.WithCompactionBatchLimit(10, 100, 1000),
// e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
//}
//
//if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
// baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries))
//}
scenarios := []TestScenario{}
for _, tp := range trafficProfiles {
name := filepath.Join(tp.Name, "ClusterOfSize1")
clusterOfSize1Options := baseOptions
clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
scenarios = append(scenarios, TestScenario{
Name: name,
Traffic: tp.Traffic,
Profile: tp.Profile,
Cluster: *e2e.NewConfig(clusterOfSize1Options...),
})
}

for _, tp := range trafficProfiles {
name := filepath.Join(tp.Name, "ClusterOfSize3")
clusterOfSize3Options := baseOptions
clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true))
clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true))
if fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption)
}
scenarios = append(scenarios, TestScenario{
Name: name,
Traffic: tp.Traffic,
Profile: tp.Profile,
Cluster: *e2e.NewConfig(clusterOfSize3Options...),
})
}
if e2e.BinPath.LazyFSAvailable() {
newScenarios := scenarios
for _, s := range scenarios {
// LazyFS increases the load on CPU, so we run it with more lightweight case.
if s.Profile.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 {
lazyfsCluster := s.Cluster
lazyfsCluster.LazyFSEnabled = true
newScenarios = append(newScenarios, TestScenario{
Name: filepath.Join(s.Name, "LazyFS"),
Failpoint: s.Failpoint,
Cluster: lazyfsCluster,
Traffic: s.Traffic,
Profile: s.Profile.WithoutCompaction(),
Watch: s.Watch,
})
}
}
scenarios = newScenarios
}
//for _, tp := range trafficProfiles {
// name := filepath.Join(tp.Name, "ClusterOfSize1")
// clusterOfSize1Options := baseOptions
// clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
// scenarios = append(scenarios, TestScenario{
// Name: name,
// Traffic: tp.Traffic,
// Profile: tp.Profile,
// Cluster: *e2e.NewConfig(clusterOfSize1Options...),
// })
//}
//
//for _, tp := range trafficProfiles {
// name := filepath.Join(tp.Name, "ClusterOfSize3")
// clusterOfSize3Options := baseOptions
// clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true))
// clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true))
// if fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
// clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption)
// }
// scenarios = append(scenarios, TestScenario{
// Name: name,
// Traffic: tp.Traffic,
// Profile: tp.Profile,
// Cluster: *e2e.NewConfig(clusterOfSize3Options...),
// })
//}
//if e2e.BinPath.LazyFSAvailable() {
// newScenarios := scenarios
// for _, s := range scenarios {
// // LazyFS increases the load on CPU, so we run it with more lightweight case.
// if s.Profile.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 {
// lazyfsCluster := s.Cluster
// lazyfsCluster.LazyFSEnabled = true
// newScenarios = append(newScenarios, TestScenario{
// Name: filepath.Join(s.Name, "LazyFS"),
// Failpoint: s.Failpoint,
// Cluster: lazyfsCluster,
// Traffic: s.Traffic,
// Profile: s.Profile.WithoutCompaction(),
// Watch: s.Watch,
// })
// }
// }
// scenarios = newScenarios
//}
return scenarios
}

func Regression(t *testing.T) []TestScenario {
v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd)
//v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
//require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd)

scenarios := []TestScenario{}
scenarios = append(scenarios, TestScenario{
Name: "Issue14370",
Failpoint: failpoint.RaftBeforeSavePanic,
Profile: traffic.LowTraffic,
Traffic: traffic.EtcdPutDeleteLease,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
e2e.WithGoFailEnabled(true),
),
})
scenarios = append(scenarios, TestScenario{
Name: "Issue14685",
Failpoint: failpoint.DefragBeforeCopyPanic,
Profile: traffic.LowTraffic,
Traffic: traffic.EtcdPutDeleteLease,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
e2e.WithGoFailEnabled(true),
),
})
scenarios = append(scenarios, TestScenario{
Name: "Issue13766",
Failpoint: failpoint.KillFailpoint,
Profile: traffic.HighTrafficProfile,
Traffic: traffic.EtcdPut,
Cluster: *e2e.NewConfig(
e2e.WithSnapshotCount(100),
),
})
scenarios = append(scenarios, TestScenario{
Name: "Issue15220",
Watch: client.WatchConfig{
RequestProgress: true,
},
Profile: traffic.LowTraffic,
Traffic: traffic.EtcdPutDeleteLease,
Failpoint: failpoint.KillFailpoint,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
),
})
scenarios = append(scenarios, TestScenario{
Name: "Issue17529",
Profile: traffic.HighTrafficProfile,
Traffic: traffic.Kubernetes,
Failpoint: failpoint.SleepBeforeSendWatchResponse,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
e2e.WithGoFailEnabled(true),
options.WithSnapshotCount(100),
),
})

scenarios = append(scenarios, TestScenario{
Name: "Issue17780",
Profile: traffic.LowTraffic.WithoutCompaction(),
Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic,
Traffic: traffic.Kubernetes,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
e2e.WithCompactionBatchLimit(300),
e2e.WithSnapshotCount(1000),
e2e.WithGoFailEnabled(true),
),
})
if v.Compare(version.V3_5) >= 0 {
opts := []e2e.EPClusterOption{
e2e.WithSnapshotCount(100),
e2e.WithPeerProxy(true),
e2e.WithIsPeerTLS(true),
}
if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
opts = append(opts, e2e.WithSnapshotCatchUpEntries(100))
}
//scenarios = append(scenarios, TestScenario{
// Name: "Issue14370",
// Failpoint: failpoint.RaftBeforeSavePanic,
// Profile: traffic.LowTraffic,
// Traffic: traffic.EtcdPutDeleteLease,
// Cluster: *e2e.NewConfig(
// e2e.WithClusterSize(1),
// e2e.WithGoFailEnabled(true),
// ),
//})
//scenarios = append(scenarios, TestScenario{
// Name: "Issue14685",
// Failpoint: failpoint.DefragBeforeCopyPanic,
// Profile: traffic.LowTraffic,
// Traffic: traffic.EtcdPutDeleteLease,
// Cluster: *e2e.NewConfig(
// e2e.WithClusterSize(1),
// e2e.WithGoFailEnabled(true),
// ),
//})
//scenarios = append(scenarios, TestScenario{
// Name: "Issue13766",
// Failpoint: failpoint.KillFailpoint,
// Profile: traffic.HighTrafficProfile,
// Traffic: traffic.EtcdPut,
// Cluster: *e2e.NewConfig(
// e2e.WithSnapshotCount(100),
// ),
//})
//scenarios = append(scenarios, TestScenario{
// Name: "Issue15220",
// Watch: client.WatchConfig{
// RequestProgress: true,
// },
// Profile: traffic.LowTraffic,
// Traffic: traffic.EtcdPutDeleteLease,
// Failpoint: failpoint.KillFailpoint,
// Cluster: *e2e.NewConfig(
// e2e.WithClusterSize(1),
// ),
//})
for i := 0; i < 100; i++ {
scenarios = append(scenarios, TestScenario{
Name: "Issue15271",
Failpoint: failpoint.BlackholeUntilSnapshot,
Name: "Issue17529",
Profile: traffic.HighTrafficProfile,
Traffic: traffic.EtcdPut,
Cluster: *e2e.NewConfig(opts...),
Traffic: traffic.Kubernetes,
Failpoint: failpoint.SleepBeforeSendWatchResponse,
Cluster: *e2e.NewConfig(
e2e.WithClusterSize(1),
e2e.WithGoFailEnabled(true),
options.WithSnapshotCount(100),
),
})
}

//scenarios = append(scenarios, TestScenario{
// Name: "Issue17780",
// Profile: traffic.LowTraffic.WithoutCompaction(),
// Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic,
// Traffic: traffic.Kubernetes,
// Cluster: *e2e.NewConfig(
// e2e.WithClusterSize(1),
// e2e.WithCompactionBatchLimit(300),
// e2e.WithSnapshotCount(1000),
// e2e.WithGoFailEnabled(true),
// ),
//})
//if v.Compare(version.V3_5) >= 0 {
// opts := []e2e.EPClusterOption{
// e2e.WithSnapshotCount(100),
// e2e.WithPeerProxy(true),
// e2e.WithIsPeerTLS(true),
// }
// if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
// opts = append(opts, e2e.WithSnapshotCatchUpEntries(100))
// }
// scenarios = append(scenarios, TestScenario{
// Name: "Issue15271",
// Failpoint: failpoint.BlackholeUntilSnapshot,
// Profile: traffic.HighTrafficProfile,
// Traffic: traffic.EtcdPut,
// Cluster: *e2e.NewConfig(opts...),
// })
//}
return scenarios
}

0 comments on commit 2fc3243

Please sign in to comment.