-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
asim: introduce rand testing framework with default-only settings
This patch lays the backbone of the randomized testing framework. Currently, it only supports default configuration for all options, implying that there is no randomization yet. Additionally, it refactors some of the existing structure in data_driven_test. Note that this should not change any existing behavior, and the main purpose is to make future commits cleaner. TestRandomized is a randomized testing framework designed to validate allocator by creating randomized configurations, generating corresponding allocator simulations, and validating assertions on the final state. Input of the framework (fields in the testSetting struct): 1. numIterations (int, default: 3): specifies number of test iterations to be run, each with different random configurations generated 2. duration (time.Duration, default: 30min): defined simulated duration of each iteration verbose (bool, default: false): enables detailed simulation information failing output 3. randSeed (int64, default: 42): sets seed value for random number generation 4. assertions ([]SimulationAssertion, default: conformanceAssertion with 0 under-replication, 0 over-replication, 0 violating, and 0 unavailable): defines criteria for validation assertions 5. randOptions: guides the aspect of the test configuration that should be randomized. This includes: - cluster (bool): indicates if the cluster configuration should be randomized - ranges (bool): indicates if the range configuration should be randomized - load (bool): indicates if the workload configuration should be randomized - staticSettings (bool): indicates if the simulation static settings should be randomized - staticEvents (bool): indicates if static events, including any delayed events to be applied during the simulation, should be randomized RandTestingFramework is initialized with a specified testSetting and maintained its state across all iterations. Each iteration in RandTestingFramework executes the following steps: 1. Generates a random configuration based on whether the aspect of the test configuration is set to be randomized in randOptions 2. Executes a simulation and store any assertion failures in a buffer Part of: #106311 Release note: None
- Loading branch information
Showing
7 changed files
with
437 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| // Copyright 2023 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package tests | ||
|
|
||
| import ( | ||
| "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/config" | ||
| "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/event" | ||
| "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/gen" | ||
| ) | ||
|
|
||
| // This file defines the default parameters for allocator simulator testing, | ||
| // including configurations for the cluster, ranges, load, static settings, | ||
| // static events, assertions, and plot settings. | ||
|
|
||
| const ( | ||
| defaultNodes = 3 | ||
| defaultStoresPerNode = 1 | ||
| ) | ||
|
|
||
| func defaultBasicClusterGen() gen.BasicCluster { | ||
| return gen.BasicCluster{ | ||
| Nodes: defaultNodes, | ||
| StoresPerNode: defaultStoresPerNode, | ||
| } | ||
| } | ||
|
|
||
| func defaultStaticSettingsGen() gen.StaticSettings { | ||
| return gen.StaticSettings{Settings: config.DefaultSimulationSettings()} | ||
| } | ||
|
|
||
| func defaultStaticEventsGen() gen.StaticEvents { | ||
| return gen.StaticEvents{DelayedEvents: event.DelayedEventList{}} | ||
| } | ||
|
|
||
| const defaultKeyspace = 200000 | ||
|
|
||
| const ( | ||
| defaultRwRatio, defaultRate = 0.0, 0.0 | ||
| defaultMinBlock, defaultMaxBlock = 1, 1 | ||
| defaultMinKey, defaultMaxKey = int64(1), int64(defaultKeyspace) | ||
| defaultSkewedAccess = false | ||
| ) | ||
|
|
||
| func defaultLoadGen() gen.BasicLoad { | ||
| return gen.BasicLoad{ | ||
| RWRatio: defaultRwRatio, | ||
| Rate: defaultRate, | ||
| SkewedAccess: defaultSkewedAccess, | ||
| MinBlockSize: defaultMinBlock, | ||
| MaxBlockSize: defaultMaxBlock, | ||
| MinKey: defaultMinKey, | ||
| MaxKey: defaultMaxKey, | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| defaultRanges = 1 | ||
| defaultPlacementType = gen.Uniform | ||
| defaultReplicationFactor = 1 | ||
| defaultBytes = 0 | ||
| ) | ||
|
|
||
| func defaultBasicRangesGen() gen.BasicRanges { | ||
| return gen.BasicRanges{ | ||
| BaseRanges: gen.BaseRanges{ | ||
| Ranges: defaultRanges, | ||
| KeySpace: defaultKeyspace, | ||
| ReplicationFactor: defaultReplicationFactor, | ||
| Bytes: defaultBytes, | ||
| }, | ||
| PlacementType: defaultPlacementType, | ||
| } | ||
| } | ||
|
|
||
| func defaultAssertions() []SimulationAssertion { | ||
| return []SimulationAssertion{ | ||
| conformanceAssertion{ | ||
| underreplicated: 0, | ||
| overreplicated: 0, | ||
| violating: 0, | ||
| unavailable: 0, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| defaultStat = "replicas" | ||
| defaultHeight, defaultWidth = 15, 80 | ||
| ) | ||
|
|
||
| type plotSettings struct { | ||
| stat string | ||
| height, width int | ||
| } | ||
|
|
||
| func defaultPlotSettings() plotSettings { | ||
| return plotSettings{ | ||
| stat: defaultStat, | ||
| height: defaultHeight, | ||
| width: defaultWidth, | ||
| } | ||
| } |
Oops, something went wrong.