diff --git a/pkg/generators/statement_generator.go b/pkg/generators/statement_generator.go index 2fee7fcd..42e9011e 100644 --- a/pkg/generators/statement_generator.go +++ b/pkg/generators/statement_generator.go @@ -15,15 +15,9 @@ package generators import ( - "encoding/json" "fmt" - "math" "strings" - "github.com/pkg/errors" - "github.com/scylladb/gocqlx/v2/qb" - "golang.org/x/exp/rand" - "github.com/scylladb/gemini/pkg/builders" "github.com/scylladb/gemini/pkg/coltypes" "github.com/scylladb/gemini/pkg/testschema" @@ -31,794 +25,6 @@ import ( "github.com/scylladb/gemini/pkg/utils" ) -func GenMutateStmt(s *testschema.Schema, t *testschema.Table, g *Generator, r *rand.Rand, p *typedef.PartitionRangeConfig, deletes bool) (*typedef.Stmt, error) { - t.RLock() - defer t.RUnlock() - - valuesWithToken := g.Get() - if valuesWithToken == nil { - return nil, nil - } - useLWT := false - if p.UseLWT && r.Uint32()%10 == 0 { - useLWT = true - } - - if !deletes { - return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) - } - switch n := rand.Intn(1000); n { - case 10, 100: - return genDeleteRows(s, t, valuesWithToken, r, p) - default: - switch rand.Intn(2) { - case 0: - if t.KnownIssues[typedef.KnownIssuesJSONWithTuples] { - return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) - } - return genInsertJSONStmt(s, t, valuesWithToken, r, p) - default: - return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) - } - } -} - -func GenCheckStmt( - s *testschema.Schema, - table *testschema.Table, - g *Generator, - rnd *rand.Rand, - p *typedef.PartitionRangeConfig, -) *typedef.Stmt { - n := 0 - mvNum := -1 - maxClusteringRels := 0 - numQueryPKs := 0 - if len(table.MaterializedViews) > 0 && rnd.Int()%2 == 0 { - mvNum = utils.RandInt2(rnd, 0, len(table.MaterializedViews)) - } - - switch mvNum { - case -1: - if len(table.Indexes) > 0 { - n = rnd.Intn(5) - } else { - n = rnd.Intn(4) - } - switch n { - case 0: - return genSinglePartitionQuery(s, table, g) - case 1: - numQueryPKs = utils.RandInt2(rnd, 1, table.PartitionKeys.Len()) - multiplier := int(math.Pow(float64(numQueryPKs), float64(table.PartitionKeys.Len()))) - if multiplier > 100 { - numQueryPKs = 1 - } - return genMultiplePartitionQuery(s, table, g, numQueryPKs) - case 2: - maxClusteringRels = utils.RandInt2(rnd, 0, table.ClusteringKeys.Len()) - return genClusteringRangeQuery(s, table, g, rnd, p, maxClusteringRels) - case 3: - numQueryPKs = utils.RandInt2(rnd, 1, table.PartitionKeys.Len()) - multiplier := int(math.Pow(float64(numQueryPKs), float64(table.PartitionKeys.Len()))) - if multiplier > 100 { - numQueryPKs = 1 - } - maxClusteringRels = utils.RandInt2(rnd, 0, table.ClusteringKeys.Len()) - return genMultiplePartitionClusteringRangeQuery(s, table, g, rnd, p, numQueryPKs, maxClusteringRels) - case 4: - // Reducing the probability to hit these since they often take a long time to run - switch rnd.Intn(5) { - case 0: - idxCount := utils.RandInt2(rnd, 1, len(table.Indexes)) - return genSingleIndexQuery(s, table, g, rnd, p, idxCount) - default: - return genSinglePartitionQuery(s, table, g) - } - } - default: - n = rnd.Intn(4) - switch n { - case 0: - return genSinglePartitionQueryMv(s, table, g, rnd, p, mvNum) - case 1: - lenPartitionKeys := table.MaterializedViews[mvNum].PartitionKeys.Len() - numQueryPKs = utils.RandInt2(rnd, 1, lenPartitionKeys) - multiplier := int(math.Pow(float64(numQueryPKs), float64(lenPartitionKeys))) - if multiplier > 100 { - numQueryPKs = 1 - } - return genMultiplePartitionQueryMv(s, table, g, rnd, p, mvNum, numQueryPKs) - case 2: - lenClusteringKeys := table.MaterializedViews[mvNum].ClusteringKeys.Len() - maxClusteringRels = utils.RandInt2(rnd, 0, lenClusteringKeys) - return genClusteringRangeQueryMv(s, table, g, rnd, p, mvNum, maxClusteringRels) - case 3: - lenPartitionKeys := table.MaterializedViews[mvNum].PartitionKeys.Len() - numQueryPKs = utils.RandInt2(rnd, 1, lenPartitionKeys) - multiplier := int(math.Pow(float64(numQueryPKs), float64(lenPartitionKeys))) - if multiplier > 100 { - numQueryPKs = 1 - } - lenClusteringKeys := table.MaterializedViews[mvNum].ClusteringKeys.Len() - maxClusteringRels = utils.RandInt2(rnd, 0, lenClusteringKeys) - return genMultiplePartitionClusteringRangeQueryMv(s, table, g, rnd, p, mvNum, numQueryPKs, maxClusteringRels) - } - } - - return nil -} - -func genSinglePartitionQuery( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - valuesWithToken := g.GetOld() - if valuesWithToken == nil { - return nil - } - values := valuesWithToken.Value.Copy() - builder := qb.Select(s.Keyspace.Name + "." + t.Name) - typs := make([]typedef.Type, 0, 10) - for _, pk := range t.PartitionKeys { - builder = builder.Where(qb.Eq(pk.Name)) - typs = append(typs, pk.Type) - } - - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectStatementType, - }, - ValuesWithToken: valuesWithToken, - Values: values, - } -} - -func genSinglePartitionQueryMv( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - mvNum int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - valuesWithToken := g.GetOld() - if valuesWithToken == nil { - return nil - } - mv := t.MaterializedViews[mvNum] - builder := qb.Select(s.Keyspace.Name + "." + mv.Name) - typs := make([]typedef.Type, 0, 10) - for _, pk := range mv.PartitionKeys { - builder = builder.Where(qb.Eq(pk.Name)) - typs = append(typs, pk.Type) - } - - values := valuesWithToken.Value.Copy() - if mv.HaveNonPrimaryKey() { - var mvValues []interface{} - mvValues = append(mvValues, mv.NonPrimaryKey.Type.GenValue(r, p)...) - values = append(mvValues, values...) - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectStatementType, - }, - ValuesWithToken: valuesWithToken, - Values: values, - } -} - -func genMultiplePartitionQuery( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - numQueryPKs int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - var ( - values []interface{} - typs []typedef.Type - ) - builder := qb.Select(s.Keyspace.Name + "." + t.Name) - for i, pk := range t.PartitionKeys { - builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) - for j := 0; j < numQueryPKs; j++ { - vs := g.GetOld() - if vs == nil { - return nil - } - values = append(values, vs.Value[i]) - typs = append(typs, pk.Type) - } - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectStatementType, - }, - Values: values, - } -} - -func genMultiplePartitionQueryMv( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - mvNum, numQueryPKs int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - - var values []interface{} - var typs []typedef.Type - - mv := t.MaterializedViews[mvNum] - builder := qb.Select(s.Keyspace.Name + "." + mv.Name) - switch mv.HaveNonPrimaryKey() { - case true: - for i, pk := range mv.PartitionKeys { - builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) - for j := 0; j < numQueryPKs; j++ { - vs := g.GetOld() - if vs == nil { - return nil - } - if i == 0 { - values = appendValue(pk.Type, r, p, values) - typs = append(typs, pk.Type) - } else { - values = append(values, vs.Value[i-1]) - typs = append(typs, pk.Type) - } - } - } - case false: - for i, pk := range mv.PartitionKeys { - builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) - for j := 0; j < numQueryPKs; j++ { - vs := g.GetOld() - if vs == nil { - return nil - } - values = append(values, vs.Value[i]) - typs = append(typs, pk.Type) - } - } - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectStatementType, - }, - Values: values, - } -} - -func genClusteringRangeQuery( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - maxClusteringRels int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - vs := g.GetOld() - if vs == nil { - return nil - } - var allTypes []typedef.Type - values := vs.Value.Copy() - builder := qb.Select(s.Keyspace.Name + "." + t.Name) - for _, pk := range t.PartitionKeys { - builder = builder.Where(qb.Eq(pk.Name)) - allTypes = append(allTypes, pk.Type) - } - clusteringKeys := t.ClusteringKeys - if len(clusteringKeys) > 0 { - for i := 0; i < maxClusteringRels; i++ { - ck := clusteringKeys[i] - builder = builder.Where(qb.Eq(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - allTypes = append(allTypes, ck.Type) - } - ck := clusteringKeys[maxClusteringRels] - builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - values = append(values, ck.Type.GenValue(r, p)...) - allTypes = append(allTypes, ck.Type, ck.Type) - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - QueryType: typedef.SelectRangeStatementType, - Types: allTypes, - }, - Values: values, - } -} - -func genClusteringRangeQueryMv( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - mvNum, maxClusteringRels int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - vs := g.GetOld() - if vs == nil { - return nil - } - values := vs.Value.Copy() - mv := t.MaterializedViews[mvNum] - if mv.HaveNonPrimaryKey() { - mvValues := append([]interface{}{}, mv.NonPrimaryKey.Type.GenValue(r, p)...) - values = append(mvValues, values...) - } - builder := qb.Select(s.Keyspace.Name + "." + mv.Name) - - var allTypes []typedef.Type - for _, pk := range mv.PartitionKeys { - builder = builder.Where(qb.Eq(pk.Name)) - allTypes = append(allTypes, pk.Type) - } - - clusteringKeys := mv.ClusteringKeys - if len(clusteringKeys) > 0 { - for i := 0; i < maxClusteringRels; i++ { - ck := clusteringKeys[i] - builder = builder.Where(qb.Eq(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - allTypes = append(allTypes, ck.Type) - } - ck := clusteringKeys[maxClusteringRels] - builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) - values = append(values, t.ClusteringKeys[maxClusteringRels].Type.GenValue(r, p)...) - values = append(values, t.ClusteringKeys[maxClusteringRels].Type.GenValue(r, p)...) - allTypes = append(allTypes, ck.Type, ck.Type) - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - QueryType: typedef.SelectRangeStatementType, - Types: allTypes, - }, - Values: values, - } -} - -func genMultiplePartitionClusteringRangeQuery( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - numQueryPKs, maxClusteringRels int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - - clusteringKeys := t.ClusteringKeys - pkValues := t.PartitionKeysLenValues() - valuesCount := pkValues*numQueryPKs + clusteringKeys[:maxClusteringRels].LenValues() + clusteringKeys[maxClusteringRels].Type.LenValue()*2 - values := make(typedef.Values, pkValues*numQueryPKs, valuesCount) - typs := make(typedef.Types, pkValues*numQueryPKs, valuesCount) - builder := qb.Select(s.Keyspace.Name + "." + t.Name) - - for _, pk := range t.PartitionKeys { - builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) - } - - for j := 0; j < numQueryPKs; j++ { - vs := g.GetOld() - if vs == nil { - return nil - } - for id := range vs.Value { - idx := id*numQueryPKs + j - typs[idx] = t.PartitionKeys[id].Type - values[idx] = vs.Value[id] - } - } - - if len(clusteringKeys) > 0 { - for i := 0; i < maxClusteringRels; i++ { - ck := clusteringKeys[i] - builder = builder.Where(qb.Eq(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - typs = append(typs, ck.Type) - } - ck := clusteringKeys[maxClusteringRels] - builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - values = append(values, ck.Type.GenValue(r, p)...) - typs = append(typs, ck.Type, ck.Type) - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectRangeStatementType, - }, - Values: values, - } -} - -func genMultiplePartitionClusteringRangeQueryMv( - s *testschema.Schema, - t *testschema.Table, - g GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - mvNum, numQueryPKs, maxClusteringRels int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - - mv := t.MaterializedViews[mvNum] - clusteringKeys := mv.ClusteringKeys - pkValues := mv.PartitionKeysLenValues() - valuesCount := pkValues*numQueryPKs + clusteringKeys[:maxClusteringRels].LenValues() + clusteringKeys[maxClusteringRels].Type.LenValue()*2 - mvKey := mv.NonPrimaryKey - - var ( - mvKeyLen int - baseID int - ) - if mvKey != nil { - mvKeyLen = mvKey.Type.LenValue() - baseID = 1 - valuesCount += mv.PartitionKeys.LenValues() * numQueryPKs - } - values := make(typedef.Values, pkValues*numQueryPKs, valuesCount) - typs := make(typedef.Types, pkValues*numQueryPKs, valuesCount) - builder := qb.Select(s.Keyspace.Name + "." + mv.Name) - - for _, pk := range mv.PartitionKeys { - builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) - } - - if mvKey != nil { - // Fill values for Materialized view primary key - for j := 0; j < numQueryPKs; j++ { - typs[j] = mvKey.Type - copy(values[j*mvKeyLen:], mvKey.Type.GenValue(r, p)) - } - } - - for j := 0; j < numQueryPKs; j++ { - vs := g.GetOld() - if vs == nil { - return nil - } - for id := range vs.Value { - idx := (baseID+id)*numQueryPKs + j - typs[idx] = mv.PartitionKeys[baseID+id].Type - values[idx] = vs.Value[id] - } - } - - if len(clusteringKeys) > 0 { - for i := 0; i < maxClusteringRels; i++ { - ck := clusteringKeys[i] - builder = builder.Where(qb.Eq(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - typs = append(typs, ck.Type) - } - ck := clusteringKeys[maxClusteringRels] - builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) - values = append(values, ck.Type.GenValue(r, p)...) - values = append(values, ck.Type.GenValue(r, p)...) - typs = append(typs, ck.Type, ck.Type) - } - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectRangeStatementType, - }, - Values: values, - } -} - -func genSingleIndexQuery( - s *testschema.Schema, - t *testschema.Table, - _ GeneratorInterface, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - idxCount int, -) *typedef.Stmt { - t.RLock() - defer t.RUnlock() - - var ( - values []interface{} - typs []typedef.Type - ) - - builder := qb.Select(s.Keyspace.Name + "." + t.Name) - builder.AllowFiltering() - for i := 0; i < idxCount; i++ { - builder = builder.Where(qb.Eq(t.Indexes[i].Column)) - values = append(values, t.Columns[t.Indexes[i].ColumnIdx].Type.GenValue(r, p)...) - typs = append(typs, t.Columns[t.Indexes[i].ColumnIdx].Type) - } - - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: typs, - QueryType: typedef.SelectByIndexStatementType, - }, - Values: values, - } -} - -func genInsertOrUpdateStmt( - s *testschema.Schema, - t *testschema.Table, - valuesWithToken *typedef.ValueWithToken, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - useLWT bool, -) (*typedef.Stmt, error) { - if t.IsCounterTable() { - return genUpdateStmt(s, t, valuesWithToken, r, p) - } - return genInsertStmt(s, t, valuesWithToken, r, p, useLWT) -} - -func genUpdateStmt(_ *testschema.Schema, t *testschema.Table, valuesWithToken *typedef.ValueWithToken, r *rand.Rand, p *typedef.PartitionRangeConfig) (*typedef.Stmt, error) { - stmtCache := t.GetQueryCache(typedef.CacheUpdate) - nonCounters := t.Columns.NonCounters() - values := make(typedef.Values, 0, t.PartitionKeys.LenValues()+t.ClusteringKeys.LenValues()+nonCounters.LenValues()) - for _, cdef := range nonCounters { - values = appendValue(cdef.Type, r, p, values) - } - values = values.CopyFrom(valuesWithToken.Value) - for _, ck := range t.ClusteringKeys { - values = appendValue(ck.Type, r, p, values) - } - return &typedef.Stmt{ - StmtCache: stmtCache, - ValuesWithToken: valuesWithToken, - Values: values, - }, nil -} - -func genInsertStmt( - _ *testschema.Schema, - t *testschema.Table, - valuesWithToken *typedef.ValueWithToken, - r *rand.Rand, - p *typedef.PartitionRangeConfig, - useLWT bool, -) (*typedef.Stmt, error) { - values := make(typedef.Values, 0, t.PartitionKeys.LenValues()+t.ClusteringKeys.LenValues()+t.Columns.LenValues()) - values = values.CopyFrom(valuesWithToken.Value) - for _, ck := range t.ClusteringKeys { - values = append(values, ck.Type.GenValue(r, p)...) - } - for _, col := range t.Columns { - values = append(values, col.Type.GenValue(r, p)...) - } - cacheType := typedef.CacheInsert - if useLWT { - cacheType = typedef.CacheInsertIfNotExists - } - stmtCache := t.GetQueryCache(cacheType) - return &typedef.Stmt{ - StmtCache: stmtCache, - ValuesWithToken: valuesWithToken, - Values: values, - }, nil -} - -func genInsertJSONStmt( - s *testschema.Schema, - table *testschema.Table, - valuesWithToken *typedef.ValueWithToken, - r *rand.Rand, - p *typedef.PartitionRangeConfig, -) (*typedef.Stmt, error) { - var v string - var ok bool - if table.IsCounterTable() { - return nil, nil - } - vs := valuesWithToken.Value.Copy() - values := make(map[string]interface{}) - for i, pk := range table.PartitionKeys { - switch t := pk.Type.(type) { - case coltypes.SimpleType: - if t != coltypes.TYPE_BLOB { - values[pk.Name] = vs[i] - continue - } - v, ok = vs[i].(string) - if ok { - values[pk.Name] = "0x" + v - } - case *coltypes.TupleType: - tupVals := make([]interface{}, len(t.Types)) - for j := 0; j < len(t.Types); j++ { - if t.Types[j] == coltypes.TYPE_BLOB { - v, ok = vs[i+j].(string) - if ok { - v = "0x" + v - } - vs[i+j] = v - } - tupVals[i] = vs[i+j] - i++ - } - values[pk.Name] = tupVals - default: - panic(fmt.Sprintf("unknown type: %s", t.Name())) - } - } - values = table.ClusteringKeys.ToJSONMap(values, r, p) - values = table.Columns.ToJSONMap(values, r, p) - - jsonString, err := json.Marshal(values) - if err != nil { - return nil, err - } - - builder := qb.Insert(s.Keyspace.Name + "." + table.Name).Json() - return &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: builder, - Types: []typedef.Type{coltypes.TYPE_TEXT}, - QueryType: typedef.InsertStatement, - }, - ValuesWithToken: valuesWithToken, - Values: []interface{}{string(jsonString)}, - }, nil -} - -func genDeleteRows(_ *testschema.Schema, t *testschema.Table, valuesWithToken *typedef.ValueWithToken, r *rand.Rand, p *typedef.PartitionRangeConfig) (*typedef.Stmt, error) { - stmtCache := t.GetQueryCache(typedef.CacheDelete) - values := valuesWithToken.Value.Copy() - if len(t.ClusteringKeys) > 0 { - ck := t.ClusteringKeys[0] - values = appendValue(ck.Type, r, p, values) - values = appendValue(ck.Type, r, p, values) - } - return &typedef.Stmt{ - StmtCache: stmtCache, - ValuesWithToken: valuesWithToken, - Values: values, - }, nil -} - -func GenDDLStmt(s *testschema.Schema, t *testschema.Table, r *rand.Rand, _ *typedef.PartitionRangeConfig, sc *typedef.SchemaConfig) (*typedef.Stmts, error) { - maxVariant := 1 - if len(t.Columns) > 0 { - maxVariant = 2 - } - switch n := r.Intn(maxVariant + 2); n { - // case 0: // Alter column not supported in Cassandra from 3.0.11 - // return t.alterColumn(s.Keyspace.Name) - case 2: - colNum := r.Intn(len(t.Columns)) - return genDropColumnStmt(t, s.Keyspace.Name, colNum) - default: - column := testschema.ColumnDef{Name: GenColumnName("col", len(t.Columns)+1), Type: GenColumnType(len(t.Columns)+1, sc)} - return genAddColumnStmt(t, s.Keyspace.Name, &column) - } -} - -func appendValue(columnType typedef.Type, r *rand.Rand, p *typedef.PartitionRangeConfig, values []interface{}) []interface{} { - return append(values, columnType.GenValue(r, p)...) -} - -func genAddColumnStmt(t *testschema.Table, keyspace string, column *testschema.ColumnDef) (*typedef.Stmts, error) { - var stmts []*typedef.Stmt - if c, ok := column.Type.(*coltypes.UDTType); ok { - createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s);" - var typs []string - for name, typ := range c.Types { - typs = append(typs, name+" "+typ.CQLDef()) - } - stmt := fmt.Sprintf(createType, keyspace, c.TypeName, strings.Join(typs, ",")) - stmts = append(stmts, &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: &builders.AlterTableBuilder{ - Stmt: stmt, - }, - }, - }) - } - stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ADD " + column.Name + " " + column.Type.CQLDef() - stmts = append(stmts, &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: &builders.AlterTableBuilder{ - Stmt: stmt, - }, - }, - }) - return &typedef.Stmts{ - List: stmts, - PostStmtHook: func() { - t.Columns = append(t.Columns, column) - t.ResetQueryCache() - }, - }, nil -} - -//nolint:unused -func alterColumn(t *testschema.Table, keyspace string) ([]*typedef.Stmt, func(), error) { - var stmts []*typedef.Stmt - idx := rand.Intn(len(t.Columns)) - column := t.Columns[idx] - oldType, isSimpleType := column.Type.(coltypes.SimpleType) - if !isSimpleType { - return nil, func() {}, errors.Errorf("complex type=%s cannot be altered", column.Name) - } - compatTypes := coltypes.CompatibleColumnTypes[oldType] - if len(compatTypes) == 0 { - return nil, func() {}, errors.Errorf("simple type=%s has no compatible coltypes so it cannot be altered", column.Name) - } - newType := compatTypes.Random() - newColumn := testschema.ColumnDef{Name: column.Name, Type: newType} - stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ALTER " + column.Name + " TYPE " + column.Type.CQLDef() - stmts = append(stmts, &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: &builders.AlterTableBuilder{ - Stmt: stmt, - }, - QueryType: typedef.AlterColumnStatementType, - }, - }) - return stmts, func() { - t.Columns[idx] = &newColumn - t.ResetQueryCache() - }, nil -} - -func genDropColumnStmt(t *testschema.Table, keyspace string, colNum int) (*typedef.Stmts, error) { - var stmts []*typedef.Stmt - - column := t.Columns[colNum] - stmt := "ALTER TABLE " + keyspace + "." + t.Name + " DROP " + column.Name - stmts = append(stmts, &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: &builders.AlterTableBuilder{ - Stmt: stmt, - }, - QueryType: typedef.DropColumnStatementType, - }, - }) - return &typedef.Stmts{ - List: stmts, - PostStmtHook: func() { - t.Columns = t.Columns.Remove(colNum) - t.ResetQueryCache() - }, - }, nil -} - func GenSchema(sc typedef.SchemaConfig) *testschema.Schema { builder := builders.NewSchemaBuilder() keyspace := typedef.Keyspace{ diff --git a/pkg/jobs/gen_check_stmt.go b/pkg/jobs/gen_check_stmt.go new file mode 100644 index 00000000..59a71e24 --- /dev/null +++ b/pkg/jobs/gen_check_stmt.go @@ -0,0 +1,538 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jobs + +import ( + "math" + + "github.com/scylladb/gocqlx/v2/qb" + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/generators" + "github.com/scylladb/gemini/pkg/testschema" + "github.com/scylladb/gemini/pkg/typedef" + "github.com/scylladb/gemini/pkg/utils" +) + +func GenCheckStmt( + s *testschema.Schema, + table *testschema.Table, + g generators.GeneratorInterface, + rnd *rand.Rand, + p *typedef.PartitionRangeConfig, +) *typedef.Stmt { + n := 0 + mvNum := -1 + maxClusteringRels := 0 + numQueryPKs := 0 + if len(table.MaterializedViews) > 0 && rnd.Int()%2 == 0 { + mvNum = utils.RandInt2(rnd, 0, len(table.MaterializedViews)) + } + + switch mvNum { + case -1: + if len(table.Indexes) > 0 { + n = rnd.Intn(5) + } else { + n = rnd.Intn(4) + } + switch n { + case 0: + return genSinglePartitionQuery(s, table, g) + case 1: + numQueryPKs = utils.RandInt2(rnd, 1, table.PartitionKeys.Len()) + multiplier := int(math.Pow(float64(numQueryPKs), float64(table.PartitionKeys.Len()))) + if multiplier > 100 { + numQueryPKs = 1 + } + return genMultiplePartitionQuery(s, table, g, numQueryPKs) + case 2: + maxClusteringRels = utils.RandInt2(rnd, 0, table.ClusteringKeys.Len()) + return genClusteringRangeQuery(s, table, g, rnd, p, maxClusteringRels) + case 3: + numQueryPKs = utils.RandInt2(rnd, 1, table.PartitionKeys.Len()) + multiplier := int(math.Pow(float64(numQueryPKs), float64(table.PartitionKeys.Len()))) + if multiplier > 100 { + numQueryPKs = 1 + } + maxClusteringRels = utils.RandInt2(rnd, 0, table.ClusteringKeys.Len()) + return genMultiplePartitionClusteringRangeQuery(s, table, g, rnd, p, numQueryPKs, maxClusteringRels) + case 4: + // Reducing the probability to hit these since they often take a long time to run + switch rnd.Intn(5) { + case 0: + idxCount := utils.RandInt2(rnd, 1, len(table.Indexes)) + return genSingleIndexQuery(s, table, g, rnd, p, idxCount) + default: + return genSinglePartitionQuery(s, table, g) + } + } + default: + n = rnd.Intn(4) + switch n { + case 0: + return genSinglePartitionQueryMv(s, table, g, rnd, p, mvNum) + case 1: + lenPartitionKeys := table.MaterializedViews[mvNum].PartitionKeys.Len() + numQueryPKs = utils.RandInt2(rnd, 1, lenPartitionKeys) + multiplier := int(math.Pow(float64(numQueryPKs), float64(lenPartitionKeys))) + if multiplier > 100 { + numQueryPKs = 1 + } + return genMultiplePartitionQueryMv(s, table, g, rnd, p, mvNum, numQueryPKs) + case 2: + lenClusteringKeys := table.MaterializedViews[mvNum].ClusteringKeys.Len() + maxClusteringRels = utils.RandInt2(rnd, 0, lenClusteringKeys) + return genClusteringRangeQueryMv(s, table, g, rnd, p, mvNum, maxClusteringRels) + case 3: + lenPartitionKeys := table.MaterializedViews[mvNum].PartitionKeys.Len() + numQueryPKs = utils.RandInt2(rnd, 1, lenPartitionKeys) + multiplier := int(math.Pow(float64(numQueryPKs), float64(lenPartitionKeys))) + if multiplier > 100 { + numQueryPKs = 1 + } + lenClusteringKeys := table.MaterializedViews[mvNum].ClusteringKeys.Len() + maxClusteringRels = utils.RandInt2(rnd, 0, lenClusteringKeys) + return genMultiplePartitionClusteringRangeQueryMv(s, table, g, rnd, p, mvNum, numQueryPKs, maxClusteringRels) + } + } + + return nil +} + +func genSinglePartitionQuery( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + valuesWithToken := g.GetOld() + if valuesWithToken == nil { + return nil + } + values := valuesWithToken.Value.Copy() + builder := qb.Select(s.Keyspace.Name + "." + t.Name) + typs := make([]typedef.Type, 0, 10) + for _, pk := range t.PartitionKeys { + builder = builder.Where(qb.Eq(pk.Name)) + typs = append(typs, pk.Type) + } + + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectStatementType, + }, + ValuesWithToken: valuesWithToken, + Values: values, + } +} + +func genSinglePartitionQueryMv( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + mvNum int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + valuesWithToken := g.GetOld() + if valuesWithToken == nil { + return nil + } + mv := t.MaterializedViews[mvNum] + builder := qb.Select(s.Keyspace.Name + "." + mv.Name) + typs := make([]typedef.Type, 0, 10) + for _, pk := range mv.PartitionKeys { + builder = builder.Where(qb.Eq(pk.Name)) + typs = append(typs, pk.Type) + } + + values := valuesWithToken.Value.Copy() + if mv.HaveNonPrimaryKey() { + var mvValues []interface{} + mvValues = append(mvValues, mv.NonPrimaryKey.Type.GenValue(r, p)...) + values = append(mvValues, values...) + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectStatementType, + }, + ValuesWithToken: valuesWithToken, + Values: values, + } +} + +func genMultiplePartitionQuery( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + numQueryPKs int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + var ( + values []interface{} + typs []typedef.Type + ) + builder := qb.Select(s.Keyspace.Name + "." + t.Name) + for i, pk := range t.PartitionKeys { + builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) + for j := 0; j < numQueryPKs; j++ { + vs := g.GetOld() + if vs == nil { + return nil + } + values = append(values, vs.Value[i]) + typs = append(typs, pk.Type) + } + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectStatementType, + }, + Values: values, + } +} + +func genMultiplePartitionQueryMv( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + mvNum, numQueryPKs int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + + var values []interface{} + var typs []typedef.Type + + mv := t.MaterializedViews[mvNum] + builder := qb.Select(s.Keyspace.Name + "." + mv.Name) + switch mv.HaveNonPrimaryKey() { + case true: + for i, pk := range mv.PartitionKeys { + builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) + for j := 0; j < numQueryPKs; j++ { + vs := g.GetOld() + if vs == nil { + return nil + } + if i == 0 { + values = appendValue(pk.Type, r, p, values) + typs = append(typs, pk.Type) + } else { + values = append(values, vs.Value[i-1]) + typs = append(typs, pk.Type) + } + } + } + case false: + for i, pk := range mv.PartitionKeys { + builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) + for j := 0; j < numQueryPKs; j++ { + vs := g.GetOld() + if vs == nil { + return nil + } + values = append(values, vs.Value[i]) + typs = append(typs, pk.Type) + } + } + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectStatementType, + }, + Values: values, + } +} + +func genClusteringRangeQuery( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + maxClusteringRels int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + vs := g.GetOld() + if vs == nil { + return nil + } + var allTypes []typedef.Type + values := vs.Value.Copy() + builder := qb.Select(s.Keyspace.Name + "." + t.Name) + for _, pk := range t.PartitionKeys { + builder = builder.Where(qb.Eq(pk.Name)) + allTypes = append(allTypes, pk.Type) + } + clusteringKeys := t.ClusteringKeys + if len(clusteringKeys) > 0 { + for i := 0; i < maxClusteringRels; i++ { + ck := clusteringKeys[i] + builder = builder.Where(qb.Eq(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + allTypes = append(allTypes, ck.Type) + } + ck := clusteringKeys[maxClusteringRels] + builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + values = append(values, ck.Type.GenValue(r, p)...) + allTypes = append(allTypes, ck.Type, ck.Type) + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + QueryType: typedef.SelectRangeStatementType, + Types: allTypes, + }, + Values: values, + } +} + +func genClusteringRangeQueryMv( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + mvNum, maxClusteringRels int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + vs := g.GetOld() + if vs == nil { + return nil + } + values := vs.Value.Copy() + mv := t.MaterializedViews[mvNum] + if mv.HaveNonPrimaryKey() { + mvValues := append([]interface{}{}, mv.NonPrimaryKey.Type.GenValue(r, p)...) + values = append(mvValues, values...) + } + builder := qb.Select(s.Keyspace.Name + "." + mv.Name) + + var allTypes []typedef.Type + for _, pk := range mv.PartitionKeys { + builder = builder.Where(qb.Eq(pk.Name)) + allTypes = append(allTypes, pk.Type) + } + + clusteringKeys := mv.ClusteringKeys + if len(clusteringKeys) > 0 { + for i := 0; i < maxClusteringRels; i++ { + ck := clusteringKeys[i] + builder = builder.Where(qb.Eq(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + allTypes = append(allTypes, ck.Type) + } + ck := clusteringKeys[maxClusteringRels] + builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) + values = append(values, t.ClusteringKeys[maxClusteringRels].Type.GenValue(r, p)...) + values = append(values, t.ClusteringKeys[maxClusteringRels].Type.GenValue(r, p)...) + allTypes = append(allTypes, ck.Type, ck.Type) + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + QueryType: typedef.SelectRangeStatementType, + Types: allTypes, + }, + Values: values, + } +} + +func genMultiplePartitionClusteringRangeQuery( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + numQueryPKs, maxClusteringRels int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + + clusteringKeys := t.ClusteringKeys + pkValues := t.PartitionKeysLenValues() + valuesCount := pkValues*numQueryPKs + clusteringKeys[:maxClusteringRels].LenValues() + clusteringKeys[maxClusteringRels].Type.LenValue()*2 + values := make(typedef.Values, pkValues*numQueryPKs, valuesCount) + typs := make(typedef.Types, pkValues*numQueryPKs, valuesCount) + builder := qb.Select(s.Keyspace.Name + "." + t.Name) + + for _, pk := range t.PartitionKeys { + builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) + } + + for j := 0; j < numQueryPKs; j++ { + vs := g.GetOld() + if vs == nil { + return nil + } + for id := range vs.Value { + idx := id*numQueryPKs + j + typs[idx] = t.PartitionKeys[id].Type + values[idx] = vs.Value[id] + } + } + + if len(clusteringKeys) > 0 { + for i := 0; i < maxClusteringRels; i++ { + ck := clusteringKeys[i] + builder = builder.Where(qb.Eq(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + typs = append(typs, ck.Type) + } + ck := clusteringKeys[maxClusteringRels] + builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + values = append(values, ck.Type.GenValue(r, p)...) + typs = append(typs, ck.Type, ck.Type) + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectRangeStatementType, + }, + Values: values, + } +} + +func genMultiplePartitionClusteringRangeQueryMv( + s *testschema.Schema, + t *testschema.Table, + g generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + mvNum, numQueryPKs, maxClusteringRels int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + + mv := t.MaterializedViews[mvNum] + clusteringKeys := mv.ClusteringKeys + pkValues := mv.PartitionKeysLenValues() + valuesCount := pkValues*numQueryPKs + clusteringKeys[:maxClusteringRels].LenValues() + clusteringKeys[maxClusteringRels].Type.LenValue()*2 + mvKey := mv.NonPrimaryKey + + var ( + mvKeyLen int + baseID int + ) + if mvKey != nil { + mvKeyLen = mvKey.Type.LenValue() + baseID = 1 + valuesCount += mv.PartitionKeys.LenValues() * numQueryPKs + } + values := make(typedef.Values, pkValues*numQueryPKs, valuesCount) + typs := make(typedef.Types, pkValues*numQueryPKs, valuesCount) + builder := qb.Select(s.Keyspace.Name + "." + mv.Name) + + for _, pk := range mv.PartitionKeys { + builder = builder.Where(qb.InTuple(pk.Name, numQueryPKs)) + } + + if mvKey != nil { + // Fill values for Materialized view primary key + for j := 0; j < numQueryPKs; j++ { + typs[j] = mvKey.Type + copy(values[j*mvKeyLen:], mvKey.Type.GenValue(r, p)) + } + } + + for j := 0; j < numQueryPKs; j++ { + vs := g.GetOld() + if vs == nil { + return nil + } + for id := range vs.Value { + idx := (baseID+id)*numQueryPKs + j + typs[idx] = mv.PartitionKeys[baseID+id].Type + values[idx] = vs.Value[id] + } + } + + if len(clusteringKeys) > 0 { + for i := 0; i < maxClusteringRels; i++ { + ck := clusteringKeys[i] + builder = builder.Where(qb.Eq(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + typs = append(typs, ck.Type) + } + ck := clusteringKeys[maxClusteringRels] + builder = builder.Where(qb.Gt(ck.Name)).Where(qb.Lt(ck.Name)) + values = append(values, ck.Type.GenValue(r, p)...) + values = append(values, ck.Type.GenValue(r, p)...) + typs = append(typs, ck.Type, ck.Type) + } + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectRangeStatementType, + }, + Values: values, + } +} + +func genSingleIndexQuery( + s *testschema.Schema, + t *testschema.Table, + _ generators.GeneratorInterface, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + idxCount int, +) *typedef.Stmt { + t.RLock() + defer t.RUnlock() + + var ( + values []interface{} + typs []typedef.Type + ) + + builder := qb.Select(s.Keyspace.Name + "." + t.Name) + builder.AllowFiltering() + for i := 0; i < idxCount; i++ { + builder = builder.Where(qb.Eq(t.Indexes[i].Column)) + values = append(values, t.Columns[t.Indexes[i].ColumnIdx].Type.GenValue(r, p)...) + typs = append(typs, t.Columns[t.Indexes[i].ColumnIdx].Type) + } + + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: typs, + QueryType: typedef.SelectByIndexStatementType, + }, + Values: values, + } +} diff --git a/pkg/generators/schema_check_stmt_test.go b/pkg/jobs/gen_check_stmt_test.go similarity index 99% rename from pkg/generators/schema_check_stmt_test.go rename to pkg/jobs/gen_check_stmt_test.go index 770eed13..7e9c1c0b 100644 --- a/pkg/generators/schema_check_stmt_test.go +++ b/pkg/jobs/gen_check_stmt_test.go @@ -13,7 +13,7 @@ // limitations under the License. //nolint:thelper -package generators +package jobs import ( "path" diff --git a/pkg/generators/suite_const_test.go b/pkg/jobs/gen_const_test.go similarity index 99% rename from pkg/generators/suite_const_test.go rename to pkg/jobs/gen_const_test.go index caa4d19c..d9b68526 100644 --- a/pkg/generators/suite_const_test.go +++ b/pkg/jobs/gen_const_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package generators +package jobs import ( "flag" diff --git a/pkg/jobs/gen_ddl_stmt.go b/pkg/jobs/gen_ddl_stmt.go new file mode 100644 index 00000000..0200bf5f --- /dev/null +++ b/pkg/jobs/gen_ddl_stmt.go @@ -0,0 +1,136 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jobs + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + "golang.org/x/exp/rand" + + "github.com/scylladb/gemini/pkg/builders" + "github.com/scylladb/gemini/pkg/coltypes" + "github.com/scylladb/gemini/pkg/generators" + "github.com/scylladb/gemini/pkg/testschema" + "github.com/scylladb/gemini/pkg/typedef" +) + +func GenDDLStmt(s *testschema.Schema, t *testschema.Table, r *rand.Rand, _ *typedef.PartitionRangeConfig, sc *typedef.SchemaConfig) (*typedef.Stmts, error) { + maxVariant := 1 + if len(t.Columns) > 0 { + maxVariant = 2 + } + switch n := r.Intn(maxVariant + 2); n { + // case 0: // Alter column not supported in Cassandra from 3.0.11 + // return t.alterColumn(s.Keyspace.Name) + case 2: + colNum := r.Intn(len(t.Columns)) + return genDropColumnStmt(t, s.Keyspace.Name, colNum) + default: + column := testschema.ColumnDef{Name: generators.GenColumnName("col", len(t.Columns)+1), Type: generators.GenColumnType(len(t.Columns)+1, sc)} + return genAddColumnStmt(t, s.Keyspace.Name, &column) + } +} + +func appendValue(columnType typedef.Type, r *rand.Rand, p *typedef.PartitionRangeConfig, values []interface{}) []interface{} { + return append(values, columnType.GenValue(r, p)...) +} + +func genAddColumnStmt(t *testschema.Table, keyspace string, column *testschema.ColumnDef) (*typedef.Stmts, error) { + var stmts []*typedef.Stmt + if c, ok := column.Type.(*coltypes.UDTType); ok { + createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s);" + var typs []string + for name, typ := range c.Types { + typs = append(typs, name+" "+typ.CQLDef()) + } + stmt := fmt.Sprintf(createType, keyspace, c.TypeName, strings.Join(typs, ",")) + stmts = append(stmts, &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: &builders.AlterTableBuilder{ + Stmt: stmt, + }, + }, + }) + } + stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ADD " + column.Name + " " + column.Type.CQLDef() + stmts = append(stmts, &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: &builders.AlterTableBuilder{ + Stmt: stmt, + }, + }, + }) + return &typedef.Stmts{ + List: stmts, + PostStmtHook: func() { + t.Columns = append(t.Columns, column) + t.ResetQueryCache() + }, + }, nil +} + +//nolint:unused +func alterColumn(t *testschema.Table, keyspace string) ([]*typedef.Stmt, func(), error) { + var stmts []*typedef.Stmt + idx := rand.Intn(len(t.Columns)) + column := t.Columns[idx] + oldType, isSimpleType := column.Type.(coltypes.SimpleType) + if !isSimpleType { + return nil, func() {}, errors.Errorf("complex type=%s cannot be altered", column.Name) + } + compatTypes := coltypes.CompatibleColumnTypes[oldType] + if len(compatTypes) == 0 { + return nil, func() {}, errors.Errorf("simple type=%s has no compatible coltypes so it cannot be altered", column.Name) + } + newType := compatTypes.Random() + newColumn := testschema.ColumnDef{Name: column.Name, Type: newType} + stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ALTER " + column.Name + " TYPE " + column.Type.CQLDef() + stmts = append(stmts, &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: &builders.AlterTableBuilder{ + Stmt: stmt, + }, + QueryType: typedef.AlterColumnStatementType, + }, + }) + return stmts, func() { + t.Columns[idx] = &newColumn + t.ResetQueryCache() + }, nil +} + +func genDropColumnStmt(t *testschema.Table, keyspace string, colNum int) (*typedef.Stmts, error) { + var stmts []*typedef.Stmt + + column := t.Columns[colNum] + stmt := "ALTER TABLE " + keyspace + "." + t.Name + " DROP " + column.Name + stmts = append(stmts, &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: &builders.AlterTableBuilder{ + Stmt: stmt, + }, + QueryType: typedef.DropColumnStatementType, + }, + }) + return &typedef.Stmts{ + List: stmts, + PostStmtHook: func() { + t.Columns = t.Columns.Remove(colNum) + t.ResetQueryCache() + }, + }, nil +} diff --git a/pkg/generators/schema_ddl_stmt_test.go b/pkg/jobs/gen_ddl_stmt_test.go similarity index 99% rename from pkg/generators/schema_ddl_stmt_test.go rename to pkg/jobs/gen_ddl_stmt_test.go index 1a7e290c..dcadc989 100644 --- a/pkg/generators/schema_ddl_stmt_test.go +++ b/pkg/jobs/gen_ddl_stmt_test.go @@ -13,7 +13,7 @@ // limitations under the License. //nolint:thelper -package generators +package jobs import ( "path" diff --git a/pkg/jobs/gen_mutate_stmt.go b/pkg/jobs/gen_mutate_stmt.go new file mode 100644 index 00000000..c0afd53c --- /dev/null +++ b/pkg/jobs/gen_mutate_stmt.go @@ -0,0 +1,199 @@ +// Copyright 2019 ScyllaDB +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jobs + +import ( + "encoding/json" + "fmt" + + "golang.org/x/exp/rand" + + "github.com/scylladb/gocqlx/v2/qb" + + "github.com/scylladb/gemini/pkg/coltypes" + "github.com/scylladb/gemini/pkg/generators" + "github.com/scylladb/gemini/pkg/testschema" + "github.com/scylladb/gemini/pkg/typedef" +) + +func GenMutateStmt(s *testschema.Schema, t *testschema.Table, g generators.GeneratorInterface, r *rand.Rand, p *typedef.PartitionRangeConfig, deletes bool) (*typedef.Stmt, error) { + t.RLock() + defer t.RUnlock() + + valuesWithToken := g.Get() + if valuesWithToken == nil { + return nil, nil + } + useLWT := false + if p.UseLWT && r.Uint32()%10 == 0 { + useLWT = true + } + + if !deletes { + return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) + } + switch n := rand.Intn(1000); n { + case 10, 100: + return genDeleteRows(s, t, valuesWithToken, r, p) + default: + switch rand.Intn(2) { + case 0: + if t.KnownIssues[typedef.KnownIssuesJSONWithTuples] { + return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) + } + return genInsertJSONStmt(s, t, valuesWithToken, r, p) + default: + return genInsertOrUpdateStmt(s, t, valuesWithToken, r, p, useLWT) + } + } +} + +func genInsertOrUpdateStmt( + s *testschema.Schema, + t *testschema.Table, + valuesWithToken *typedef.ValueWithToken, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + useLWT bool, +) (*typedef.Stmt, error) { + if t.IsCounterTable() { + return genUpdateStmt(s, t, valuesWithToken, r, p) + } + return genInsertStmt(s, t, valuesWithToken, r, p, useLWT) +} + +func genUpdateStmt(_ *testschema.Schema, t *testschema.Table, valuesWithToken *typedef.ValueWithToken, r *rand.Rand, p *typedef.PartitionRangeConfig) (*typedef.Stmt, error) { + stmtCache := t.GetQueryCache(typedef.CacheUpdate) + nonCounters := t.Columns.NonCounters() + values := make(typedef.Values, 0, t.PartitionKeys.LenValues()+t.ClusteringKeys.LenValues()+nonCounters.LenValues()) + for _, cdef := range nonCounters { + values = appendValue(cdef.Type, r, p, values) + } + values = values.CopyFrom(valuesWithToken.Value) + for _, ck := range t.ClusteringKeys { + values = appendValue(ck.Type, r, p, values) + } + return &typedef.Stmt{ + StmtCache: stmtCache, + ValuesWithToken: valuesWithToken, + Values: values, + }, nil +} + +func genInsertStmt( + _ *testschema.Schema, + t *testschema.Table, + valuesWithToken *typedef.ValueWithToken, + r *rand.Rand, + p *typedef.PartitionRangeConfig, + useLWT bool, +) (*typedef.Stmt, error) { + values := make(typedef.Values, 0, t.PartitionKeys.LenValues()+t.ClusteringKeys.LenValues()+t.Columns.LenValues()) + values = values.CopyFrom(valuesWithToken.Value) + for _, ck := range t.ClusteringKeys { + values = append(values, ck.Type.GenValue(r, p)...) + } + for _, col := range t.Columns { + values = append(values, col.Type.GenValue(r, p)...) + } + cacheType := typedef.CacheInsert + if useLWT { + cacheType = typedef.CacheInsertIfNotExists + } + stmtCache := t.GetQueryCache(cacheType) + return &typedef.Stmt{ + StmtCache: stmtCache, + ValuesWithToken: valuesWithToken, + Values: values, + }, nil +} + +func genInsertJSONStmt( + s *testschema.Schema, + table *testschema.Table, + valuesWithToken *typedef.ValueWithToken, + r *rand.Rand, + p *typedef.PartitionRangeConfig, +) (*typedef.Stmt, error) { + var v string + var ok bool + if table.IsCounterTable() { + return nil, nil + } + vs := valuesWithToken.Value.Copy() + values := make(map[string]interface{}) + for i, pk := range table.PartitionKeys { + switch t := pk.Type.(type) { + case coltypes.SimpleType: + if t != coltypes.TYPE_BLOB { + values[pk.Name] = vs[i] + continue + } + v, ok = vs[i].(string) + if ok { + values[pk.Name] = "0x" + v + } + case *coltypes.TupleType: + tupVals := make([]interface{}, len(t.Types)) + for j := 0; j < len(t.Types); j++ { + if t.Types[j] == coltypes.TYPE_BLOB { + v, ok = vs[i+j].(string) + if ok { + v = "0x" + v + } + vs[i+j] = v + } + tupVals[i] = vs[i+j] + i++ + } + values[pk.Name] = tupVals + default: + panic(fmt.Sprintf("unknown type: %s", t.Name())) + } + } + values = table.ClusteringKeys.ToJSONMap(values, r, p) + values = table.Columns.ToJSONMap(values, r, p) + + jsonString, err := json.Marshal(values) + if err != nil { + return nil, err + } + + builder := qb.Insert(s.Keyspace.Name + "." + table.Name).Json() + return &typedef.Stmt{ + StmtCache: &typedef.StmtCache{ + Query: builder, + Types: []typedef.Type{coltypes.TYPE_TEXT}, + QueryType: typedef.InsertStatement, + }, + ValuesWithToken: valuesWithToken, + Values: []interface{}{string(jsonString)}, + }, nil +} + +func genDeleteRows(_ *testschema.Schema, t *testschema.Table, valuesWithToken *typedef.ValueWithToken, r *rand.Rand, p *typedef.PartitionRangeConfig) (*typedef.Stmt, error) { + stmtCache := t.GetQueryCache(typedef.CacheDelete) + values := valuesWithToken.Value.Copy() + if len(t.ClusteringKeys) > 0 { + ck := t.ClusteringKeys[0] + values = appendValue(ck.Type, r, p, values) + values = appendValue(ck.Type, r, p, values) + } + return &typedef.Stmt{ + StmtCache: stmtCache, + ValuesWithToken: valuesWithToken, + Values: values, + }, nil +} diff --git a/pkg/generators/schema_mutation_stmt_test.go b/pkg/jobs/gen_mutate_stmt_test.go similarity index 99% rename from pkg/generators/schema_mutation_stmt_test.go rename to pkg/jobs/gen_mutate_stmt_test.go index 1131abfa..e509c7e6 100644 --- a/pkg/generators/schema_mutation_stmt_test.go +++ b/pkg/jobs/gen_mutate_stmt_test.go @@ -13,7 +13,7 @@ // limitations under the License. //nolint:thelper -package generators +package jobs import ( "path" diff --git a/pkg/generators/suite_utils_test.go b/pkg/jobs/gen_utils_test.go similarity index 99% rename from pkg/generators/suite_utils_test.go rename to pkg/jobs/gen_utils_test.go index 75f6cd83..101630c1 100644 --- a/pkg/generators/suite_utils_test.go +++ b/pkg/jobs/gen_utils_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package generators +package jobs import ( "encoding/json" @@ -28,6 +28,7 @@ import ( "github.com/scylladb/gemini/pkg/builders" "github.com/scylladb/gemini/pkg/coltypes" + "github.com/scylladb/gemini/pkg/generators" "github.com/scylladb/gemini/pkg/replication" "github.com/scylladb/gemini/pkg/routingkey" "github.com/scylladb/gemini/pkg/tableopts" @@ -454,7 +455,7 @@ func getFromOptions(t testInterface, table *testschema.Table, option, optionsNum case "addSt": funcOpts.addType = testschema.ColumnDef{ Type: createColumnSimpleType(t, optionsNum), - Name: GenColumnName("col", len(table.Columns)+1), + Name: generators.GenColumnName("col", len(table.Columns)+1), } } diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 08cb45ea..8fbfee3d 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -235,7 +235,7 @@ func validationJob( return nil case hb := <-pump: time.Sleep(hb) - stmt := generators.GenCheckStmt(schema, table, g, r, p) + stmt := GenCheckStmt(schema, table, g, r, p) if stmt == nil { logger.Info("Validation. No statement generated from GenCheckStmt.") continue @@ -323,7 +323,7 @@ func ddl( } table.Lock() defer table.Unlock() - ddlStmts, err := generators.GenDDLStmt(schema, table, r, p, sc) + ddlStmts, err := GenDDLStmt(schema, table, r, p, sc) if err != nil { logger.Error("Failed! Mutation statement generation failed", zap.Error(err)) globalStatus.WriteErrors.Add(1) @@ -370,7 +370,7 @@ func mutation( deletes bool, logger *zap.Logger, ) error { - mutateStmt, err := generators.GenMutateStmt(schema, table, g, r, p, deletes) + mutateStmt, err := GenMutateStmt(schema, table, g, r, p, deletes) if err != nil { logger.Error("Failed! Mutation statement generation failed", zap.Error(err)) globalStatus.WriteErrors.Add(1) diff --git a/pkg/generators/test_expected_data/check/clustering_range.json b/pkg/jobs/test_expected_data/check/clustering_range.json similarity index 100% rename from pkg/generators/test_expected_data/check/clustering_range.json rename to pkg/jobs/test_expected_data/check/clustering_range.json diff --git a/pkg/generators/test_expected_data/check/clustering_range_mv.json b/pkg/jobs/test_expected_data/check/clustering_range_mv.json similarity index 100% rename from pkg/generators/test_expected_data/check/clustering_range_mv.json rename to pkg/jobs/test_expected_data/check/clustering_range_mv.json diff --git a/pkg/generators/test_expected_data/check/multiple_partition.json b/pkg/jobs/test_expected_data/check/multiple_partition.json similarity index 100% rename from pkg/generators/test_expected_data/check/multiple_partition.json rename to pkg/jobs/test_expected_data/check/multiple_partition.json diff --git a/pkg/generators/test_expected_data/check/multiple_partition_clustering_range.json b/pkg/jobs/test_expected_data/check/multiple_partition_clustering_range.json similarity index 100% rename from pkg/generators/test_expected_data/check/multiple_partition_clustering_range.json rename to pkg/jobs/test_expected_data/check/multiple_partition_clustering_range.json diff --git a/pkg/generators/test_expected_data/check/multiple_partition_clustering_range_mv.json b/pkg/jobs/test_expected_data/check/multiple_partition_clustering_range_mv.json similarity index 100% rename from pkg/generators/test_expected_data/check/multiple_partition_clustering_range_mv.json rename to pkg/jobs/test_expected_data/check/multiple_partition_clustering_range_mv.json diff --git a/pkg/generators/test_expected_data/check/multiple_partition_mv.json b/pkg/jobs/test_expected_data/check/multiple_partition_mv.json similarity index 100% rename from pkg/generators/test_expected_data/check/multiple_partition_mv.json rename to pkg/jobs/test_expected_data/check/multiple_partition_mv.json diff --git a/pkg/generators/test_expected_data/check/single_index.json b/pkg/jobs/test_expected_data/check/single_index.json similarity index 100% rename from pkg/generators/test_expected_data/check/single_index.json rename to pkg/jobs/test_expected_data/check/single_index.json diff --git a/pkg/generators/test_expected_data/check/single_partition.json b/pkg/jobs/test_expected_data/check/single_partition.json similarity index 100% rename from pkg/generators/test_expected_data/check/single_partition.json rename to pkg/jobs/test_expected_data/check/single_partition.json diff --git a/pkg/generators/test_expected_data/check/single_partition_mv.json b/pkg/jobs/test_expected_data/check/single_partition_mv.json similarity index 100% rename from pkg/generators/test_expected_data/check/single_partition_mv.json rename to pkg/jobs/test_expected_data/check/single_partition_mv.json diff --git a/pkg/generators/test_expected_data/ddl/add_column.json b/pkg/jobs/test_expected_data/ddl/add_column.json similarity index 100% rename from pkg/generators/test_expected_data/ddl/add_column.json rename to pkg/jobs/test_expected_data/ddl/add_column.json diff --git a/pkg/generators/test_expected_data/ddl/drop_column.json b/pkg/jobs/test_expected_data/ddl/drop_column.json similarity index 100% rename from pkg/generators/test_expected_data/ddl/drop_column.json rename to pkg/jobs/test_expected_data/ddl/drop_column.json diff --git a/pkg/generators/test_expected_data/mutate/delete.json b/pkg/jobs/test_expected_data/mutate/delete.json similarity index 100% rename from pkg/generators/test_expected_data/mutate/delete.json rename to pkg/jobs/test_expected_data/mutate/delete.json diff --git a/pkg/generators/test_expected_data/mutate/insert.json b/pkg/jobs/test_expected_data/mutate/insert.json similarity index 100% rename from pkg/generators/test_expected_data/mutate/insert.json rename to pkg/jobs/test_expected_data/mutate/insert.json diff --git a/pkg/generators/test_expected_data/mutate/insert_j.json b/pkg/jobs/test_expected_data/mutate/insert_j.json similarity index 100% rename from pkg/generators/test_expected_data/mutate/insert_j.json rename to pkg/jobs/test_expected_data/mutate/insert_j.json diff --git a/pkg/generators/test_expected_data/mutate/update.json b/pkg/jobs/test_expected_data/mutate/update.json similarity index 100% rename from pkg/generators/test_expected_data/mutate/update.json rename to pkg/jobs/test_expected_data/mutate/update.json diff --git a/pkg/generators/test_generator.go b/pkg/jobs/test_generator.go similarity index 99% rename from pkg/generators/test_generator.go rename to pkg/jobs/test_generator.go index a0675aff..002fd505 100644 --- a/pkg/generators/test_generator.go +++ b/pkg/jobs/test_generator.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package generators +package jobs import ( "fmt"