From b8ad42c98de89cea7fab5e6bb4605bc2cd6a632b Mon Sep 17 00:00:00 2001 From: Henrik Johansson Date: Thu, 20 Jun 2019 11:47:49 +0200 Subject: [PATCH] gemini,schema: adds support for feature sets There is now a CLI toggle "--cql-features" that can enable or disable features that gemini uses. Three levels are deined so far: - "basic": only simple regular columns are used - "normal": MV and SI in addition to "basic" - "all": not used yet but will enable schema modifications --- CHANGELOG.md | 8 ++++++ cmd/gemini/root.go | 15 +++++++++++ schema.go | 66 +++++++++++++++++++++++++++++++--------------- 3 files changed, 68 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32be2fea..c2aac263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- CLI toggle `--cql-features` added to let the user select which type of CQL features + to use. The current levels are `basic`, `normal` and `all`. The `basic` level have only + regular columns and no indexes nor materialized views. `normal` adds these two constructs + and `all` currently the same as `normal` but will be used to differentiate more advanced + features in the future. + ## [1.1.0] - 2019-06-11 - Exponential backoff retry policy added with 5 retries between 1 and 10 seconds. diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go index 2c7cfda5..062478b9 100644 --- a/cmd/gemini/root.go +++ b/cmd/gemini/root.go @@ -48,6 +48,7 @@ var ( maxClusteringKeys int maxColumns int datasetSize string + cqlFeatures string ) const ( @@ -242,6 +243,7 @@ func createSchemaConfig() *gemini.SchemaConfig { MaxTupleParts: 2, MaxBlobLength: 20, MaxStringLength: 20, + CQLFeature: defaultConfig.CQLFeature, } default: return defaultConfig @@ -268,6 +270,7 @@ func createDefaultSchemaConfig() *gemini.SchemaConfig { MinBlobLength: MinBlobLength, MaxStringLength: MaxStringLength, MinStringLength: MinStringLength, + CQLFeature: getCQLFeature(cqlFeatures), } } @@ -308,6 +311,17 @@ func getCompactionStrategy(cs string) *gemini.CompactionStrategy { } } +func getCQLFeature(feature string) gemini.CQLFeature { + switch strings.ToLower(feature) { + case "all": + return gemini.CQL_FEATURE_ALL + case "normal": + return gemini.CQL_FEATURE_NORMAL + default: + return gemini.CQL_FEATURE_BASIC + } +} + func runJob(f testJob, schema *gemini.Schema, schemaConfig *gemini.SchemaConfig, s store.Store, mode string, out *os.File) { defer out.Sync() c := make(chan Status, 10000) @@ -517,6 +531,7 @@ func init() { rootCmd.Flags().IntVarP(&maxClusteringKeys, "max-clustering-keys", "", 4, "Maximum number of generated clustering keys") rootCmd.Flags().IntVarP(&maxColumns, "max-columns", "", 16, "Maximum number of generated columns") rootCmd.Flags().StringVarP(&datasetSize, "dataset-size", "", "large", "Specify the type of dataset size to use, small|large") + rootCmd.Flags().StringVarP(&cqlFeatures, "cql-features", "", "basic", "Specify the type of cql features to use, basic|normal|large") } func printSetup() error { diff --git a/schema.go b/schema.go index 0607343a..4902cb17 100644 --- a/schema.go +++ b/schema.go @@ -2,16 +2,22 @@ package gemini import ( "encoding/json" + "errors" "fmt" "math/rand" "strconv" "strings" - "github.com/pkg/errors" "github.com/scylladb/gocqlx/qb" ) +type CQLFeature int + const ( + CQL_FEATURE_BASIC CQLFeature = iota + 1 + CQL_FEATURE_NORMAL + CQL_FEATURE_ALL + KnownIssuesJsonWithTuples = "https://github.com/scylladb/scylla/issues/3708" ) @@ -26,6 +32,7 @@ type SchemaConfig struct { MaxStringLength int MinBlobLength int MinStringLength int + CQLFeature CQLFeature } type Keyspace struct { @@ -220,6 +227,38 @@ func GenSchema(sc *SchemaConfig) *Schema { for i := 0; i < numColumns; i++ { columns = append(columns, ColumnDef{Name: genColumnName("col", i), Type: genColumnType(numColumns, sc)}) } + var indexes []IndexDef + if sc.CQLFeature > CQL_FEATURE_BASIC { + indexes = createIndexes(numColumns, columns) + } + + var mvs []MaterializedView + if sc.CQLFeature > CQL_FEATURE_BASIC { + mvs = createMaterializedViews(partitionKeys, clusteringKeys, columns) + } + + table := Table{ + Name: "table1", + PartitionKeys: partitionKeys, + ClusteringKeys: clusteringKeys, + Columns: columns, + MaterializedViews: mvs, + Indexes: indexes, + KnownIssues: map[string]bool{ + KnownIssuesJsonWithTuples: true, + }, + } + if sc.CompactionStrategy == nil { + table.CompactionStrategy = randomCompactionStrategy() + } else { + table.CompactionStrategy = &(*sc.CompactionStrategy) + } + + builder.Table(&table) + return builder.Build() +} + +func createIndexes(numColumns int, columns []ColumnDef) []IndexDef { var indexes []IndexDef if numColumns > 0 { numIndexes := rand.Intn(numColumns) @@ -229,6 +268,10 @@ func GenSchema(sc *SchemaConfig) *Schema { } } } + return indexes +} + +func createMaterializedViews(partitionKeys []ColumnDef, clusteringKeys []ColumnDef, columns []ColumnDef) []MaterializedView { validMVColumn := func() (ColumnDef, error) { validCols := make([]ColumnDef, 0, len(columns)) for _, col := range columns { @@ -267,26 +310,7 @@ func GenSchema(sc *SchemaConfig) *Schema { } mvs = append(mvs, mv) } - - table := Table{ - Name: "table1", - PartitionKeys: partitionKeys, - ClusteringKeys: clusteringKeys, - Columns: columns, - MaterializedViews: mvs, - Indexes: indexes, - KnownIssues: map[string]bool{ - KnownIssuesJsonWithTuples: true, - }, - } - if sc.CompactionStrategy == nil { - table.CompactionStrategy = randomCompactionStrategy() - } else { - table.CompactionStrategy = &(*sc.CompactionStrategy) - } - - builder.Table(&table) - return builder.Build() + return mvs } func randomCompactionStrategy() *CompactionStrategy {