Skip to content

Commit

Permalink
workload: fixture create can make table subset
Browse files Browse the repository at this point in the history
The new --only-tables flag allows specifying a subset of the tables in a
workload for creation. This is useful for benchmarking the workload
program itself, as well as creating partial backups of very large
datasets.

Release note: None
  • Loading branch information
jordanlewis committed Mar 5, 2018
1 parent 17576e1 commit 15be838
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 0 deletions.
15 changes: 15 additions & 0 deletions pkg/cmd/workload/fixtures.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ var fixturesMakeCSVServerURL = fixturesMakeCmd.PersistentFlags().String(
`csv-server`, ``,
`Skip saving CSVs to cloud storage, instead get them from a 'csv-server' running at this url`)

var fixturesMakeOnlyTable = fixturesMakeCmd.PersistentFlags().String(
`only-tables`, ``,
`Only load the tables with the given comma-separated names`)

// gcs-bucket-override and gcs-prefix-override are exposed for testing.
var gcsBucketOverride, gcsPrefixOverride *string

Expand Down Expand Up @@ -162,6 +166,17 @@ func fixturesMake(gen workload.Generator, urls []string, dbName string) error {
if err != nil {
return err
}
if *fixturesMakeOnlyTable != "" {
tableNames := strings.Split(*fixturesMakeOnlyTable, ",")
if len(tableNames) == 0 {
return errors.New("no table names specified")
}
filter := make(map[string]struct{}, len(tableNames))
for _, tableName := range tableNames {
filter[tableName] = struct{}{}
}
gen = workload.NewFilteringGenerator(gen, filter)
}
fixture, err := workloadccl.MakeFixture(ctx, sqlDB, gcs, config(), gen)
if err != nil {
return err
Expand Down
26 changes: 26 additions & 0 deletions pkg/workload/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,32 @@ type QueryLoad struct {
ResultHist string
}

type filteringGenerator struct {
gen Generator
filter map[string]struct{}
}

func NewFilteringGenerator(gen Generator, filter map[string]struct{}) Generator {
return filteringGenerator{
gen: gen,
filter: filter,
}
}

func (f filteringGenerator) Meta() Meta {
return f.gen.Meta()
}

func (f filteringGenerator) Tables() []Table {
ret := make([]Table, 0)
for _, t := range f.gen.Tables() {
if _, ok := f.filter[t.Name]; ok {
ret = append(ret, t)
}
}
return ret
}

var registered = make(map[string]Meta)

// Register is a hook for init-time registration of Generator implementations.
Expand Down

0 comments on commit 15be838

Please sign in to comment.