Skip to content

Commit

Permalink
Merge 'schema: Single secondary index queries supported' from Henrik
Browse files Browse the repository at this point in the history 
"The main issue here is that we need to sort the rows when we validate
 them, which is possibly a Scylla bug.  Not sure if the possible row
 count is to big to do this in memory reliably but that's the way it is
 done now.

 Fixes: #6"

* origin/secondary_indexes:
  schema: Single secondary index queries supported.
  • Loading branch information
@penberg
penberg committed Mar 21, 2019
2 parents 9263e77 + efd7d83 commit 387d9d2
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 34 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

### Added

- Support for queries using secondary indexes added.
- Switched to using the upstream Go driver https://github.com/scylladb/gocql instead
of the regular driver. The goal is performance gains by using that shard awareness
feature as well as providing proper more real testing of the driver.
Expand Down
2 changes: 1 addition & 1 deletion cmd/gemini/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func validationJob(schema *gemini.Schema, table gemini.Table, s *gemini.Session,
if verbose {
fmt.Printf("%s (values=%v)\n", checkQuery, checkValues)
}
err := s.Check(checkQuery, checkValues...)
err := s.Check(table, checkQuery, checkValues...)
if err == nil {
testStatus.ReadOps++
} else {
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.1 // indirect
github.com/mattn/go-isatty v0.0.6 // indirect
github.com/scylladb/go-set v1.0.2
github.com/segmentio/ksuid v1.0.2
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 h1:vF83LI8tAakwEwvWZtrIEx7pOySacl2TOxx6eXk4ePo=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
Expand All @@ -30,6 +31,9 @@ github.com/mattn/go-isatty v0.0.6 h1:SrwhHcpV4nWrMGdNcC2kXpMfcBVYGDuTArqyhocJgvA
github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
github.com/scylladb/gocql v1.0.1 h1:LVWuLOTllhzKNh4QzPEe/gbsTVww1Li1xTHLv/vaTvY=
github.com/scylladb/gocql v1.0.1/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc=
github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU=
Expand Down
97 changes: 76 additions & 21 deletions schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,17 @@ type ColumnDef struct {
Type string
}

type IndexDef struct {
Name string
Column ColumnDef
}

type Table struct {
Name string `json:"name"`
PartitionKeys []ColumnDef `json:"partition_keys"`
ClusteringKeys []ColumnDef `json:"clustering_keys"`
Columns []ColumnDef `json:"columns"`
Indexes []IndexDef `json:"indexes"`
}

type Stmt struct {
Expand Down Expand Up @@ -63,6 +69,10 @@ func genColumnDef(prefix string, idx int) ColumnDef {
}
}

func genIndexName(prefix string, idx int) string {
return fmt.Sprintf("%s_idx", genColumnName(prefix, idx))
}

const (
MaxPartitionKeys = 2
MaxClusteringKeys = 4
Expand All @@ -75,26 +85,34 @@ func GenSchema() *Schema {
Name: "ks1",
}
builder.Keyspace(keyspace)
partitionKeys := []ColumnDef{}
var partitionKeys []ColumnDef
numPartitionKeys := rand.Intn(MaxPartitionKeys-1) + 1
for i := 0; i < numPartitionKeys; i++ {
partitionKeys = append(partitionKeys, ColumnDef{Name: genColumnName("pk", i), Type: genColumnType()})
}
clusteringKeys := []ColumnDef{}
var clusteringKeys []ColumnDef
numClusteringKeys := rand.Intn(MaxClusteringKeys)
for i := 0; i < numClusteringKeys; i++ {
clusteringKeys = append(clusteringKeys, ColumnDef{Name: genColumnName("ck", i), Type: genColumnType()})
}
columns := []ColumnDef{}
var columns []ColumnDef
numColumns := rand.Intn(MaxColumns)
for i := 0; i < numColumns; i++ {
columns = append(columns, ColumnDef{Name: genColumnName("col", i), Type: genColumnType()})
}
var indexes []IndexDef
if numColumns > 0 {
numIndexes := rand.Intn(numColumns)
for i := 0; i < numIndexes; i++ {
indexes = append(indexes, IndexDef{Name: genIndexName("col", i), Column: columns[i]})
}
}
table := Table{
Name: "table1",
PartitionKeys: partitionKeys,
ClusteringKeys: clusteringKeys,
Columns: columns,
Indexes: indexes,
}
builder.Table(table)
return builder.Build()
Expand Down Expand Up @@ -159,9 +177,11 @@ func (s *Schema) GetCreateSchema() []string {
stmts := []string{createKeyspace}

for _, t := range s.Tables {
partitionKeys := []string{}
clusteringKeys := []string{}
columns := []string{}
var (
partitionKeys []string
clusteringKeys []string
columns []string
)
for _, pk := range t.PartitionKeys {
partitionKeys = append(partitionKeys, pk.Name)
columns = append(columns, fmt.Sprintf("%s %s", pk.Name, pk.Type))
Expand All @@ -181,13 +201,18 @@ func (s *Schema) GetCreateSchema() []string {
strings.Join(partitionKeys, ","), strings.Join(clusteringKeys, ","))
}
stmts = append(stmts, createTable)
for _, idef := range t.Indexes {
stmts = append(stmts, fmt.Sprintf("CREATE INDEX %s ON %s.%s (%s)", idef.Name, s.Keyspace.Name, t.Name, idef.Column.Name))
}
}
return stmts
}

func (s *Schema) GenInsertStmt(t Table, p *PartitionRange) *Stmt {
columns := []string{}
placeholders := []string{}
var (
columns []string
placeholders []string
)
values := make([]interface{}, 0)
for _, pk := range t.PartitionKeys {
columns = append(columns, pk.Name)
Expand All @@ -214,8 +239,10 @@ func (s *Schema) GenInsertStmt(t Table, p *PartitionRange) *Stmt {
}

func (s *Schema) GenDeleteRows(t Table, p *PartitionRange) *Stmt {
relations := []string{}
values := make([]interface{}, 0)
var (
relations []string
values []interface{}
)
for _, pk := range t.PartitionKeys {
relations = append(relations, fmt.Sprintf("%s = ?", pk.Name))
values = genValue(pk.Type, p, values)
Expand All @@ -242,11 +269,16 @@ func (s *Schema) GenMutateStmt(t Table, p *PartitionRange) *Stmt {
default:
return s.GenInsertStmt(t, p)
}
return nil
}

func (s *Schema) GenCheckStmt(t Table, p *PartitionRange) *Stmt {
switch n := rand.Intn(4); n {
var n int
if len(t.Indexes) > 0 {
n = rand.Intn(5)
} else {
n = rand.Intn(4)
}
switch n {
case 0:
return s.genSinglePartitionQuery(t, p)
case 1:
Expand All @@ -255,12 +287,14 @@ func (s *Schema) GenCheckStmt(t Table, p *PartitionRange) *Stmt {
return s.genClusteringRangeQuery(t, p)
case 3:
return s.genMultiplePartitionClusteringRangeQuery(t, p)
case 4:
return s.genSingleIndexQuery(t, p)
}
return nil
}

func (s *Schema) genSinglePartitionQuery(t Table, p *PartitionRange) *Stmt {
relations := []string{}
var relations []string
values := make([]interface{}, 0)
for _, pk := range t.PartitionKeys {
relations = append(relations, fmt.Sprintf("%s = ?", pk.Name))
Expand All @@ -276,10 +310,12 @@ func (s *Schema) genSinglePartitionQuery(t Table, p *PartitionRange) *Stmt {
}

func (s *Schema) genMultiplePartitionQuery(t Table, p *PartitionRange) *Stmt {
relations := []string{}
values := make([]interface{}, 0)
var (
relations []string
pkNames []string
values []interface{}
)
pkNum := rand.Intn(10)
pkNames := []string{}
for _, pk := range t.PartitionKeys {
pkNames = append(pkNames, pk.Name)
relations = append(relations, fmt.Sprintf("%s IN (%s)", pk.Name, strings.TrimRight(strings.Repeat("?,", pkNum), ",")))
Expand All @@ -297,8 +333,10 @@ func (s *Schema) genMultiplePartitionQuery(t Table, p *PartitionRange) *Stmt {
}

func (s *Schema) genClusteringRangeQuery(t Table, p *PartitionRange) *Stmt {
relations := []string{}
values := make([]interface{}, 0)
var (
relations []string
values []interface{}
)
for _, pk := range t.PartitionKeys {
relations = append(relations, fmt.Sprintf("%s = ?", pk.Name))
values = genValue(pk.Type, p, values)
Expand All @@ -317,10 +355,12 @@ func (s *Schema) genClusteringRangeQuery(t Table, p *PartitionRange) *Stmt {
}

func (s *Schema) genMultiplePartitionClusteringRangeQuery(t Table, p *PartitionRange) *Stmt {
relations := []string{}
var (
relations []string
pkNames []string
values []interface{}
)
pkNum := rand.Intn(10)
pkNames := []string{}
values := make([]interface{}, 0)
for _, pk := range t.PartitionKeys {
pkNames = append(pkNames, pk.Name)
relations = append(relations, fmt.Sprintf("%s IN (%s)", pk.Name, strings.TrimRight(strings.Repeat("?,", pkNum), ",")))
Expand All @@ -341,6 +381,21 @@ func (s *Schema) genMultiplePartitionClusteringRangeQuery(t Table, p *PartitionR
}
}

func (s *Schema) genSingleIndexQuery(t Table, p *PartitionRange) *Stmt {
if len(t.Indexes) == 0 {
return nil
}
idx := rand.Intn(len(t.Indexes))
query := fmt.Sprintf("SELECT * FROM %s.%s WHERE %s=?", s.Keyspace.Name, t.Name, t.Indexes[idx].Column.Name)
values := genValue(t.Indexes[idx].Column.Type, p, nil)
return &Stmt{
Query: query,
Values: func() []interface{} {
return values
},
}
}

type SchemaBuilder interface {
Keyspace(Keyspace) SchemaBuilder
Table(Table) SchemaBuilder
Expand Down
Loading

0 comments on commit 387d9d2

Please sign in to comment.