Skip to content

Commit

Permalink
Merge pull request #7623 from paperstreet/family
Browse files Browse the repository at this point in the history
sql: turn on column family heuristics
  • Loading branch information
danhhz authored Jul 8, 2016
2 parents 97f4679 + 0ac6d56 commit a2c2244
Show file tree
Hide file tree
Showing 16 changed files with 97 additions and 90 deletions.
2 changes: 1 addition & 1 deletion acceptance/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func testDockerSuccess(t *testing.T, name string, cmd []string) {

const (
// NB: postgresTestTag is grepped for in circle-deps.sh, so don't rename it.
postgresTestTag = "20160623-2125"
postgresTestTag = "20160705-1326"
// Iterating against a locally built version of the docker image can be done
// by changing postgresTestImage to the hash of the container.
postgresTestImage = "cockroachdb/postgres-test:" + postgresTestTag
Expand Down
15 changes: 4 additions & 11 deletions cli/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,17 +96,10 @@ CREATE TABLE t (
o BOOL NULL,
e DECIMAL NULL,
tz TIMESTAMP WITH TIME ZONE NULL,
FAMILY "primary" (rowid),
FAMILY fam_1_i (i),
FAMILY fam_2_f (f),
FAMILY fam_3_s (s),
FAMILY fam_4_b (b),
FAMILY fam_5_d (d),
FAMILY fam_6_t (t),
FAMILY fam_7_n (n),
FAMILY fam_8_o (o),
FAMILY fam_9_e (e),
FAMILY fam_10_tz (tz)
FAMILY "primary" (i, f, d, t, n, o, tz, rowid),
FAMILY fam_1_s (s),
FAMILY fam_2_b (b),
FAMILY fam_3_e (e)
);
INSERT INTO t VALUES
Expand Down
3 changes: 1 addition & 2 deletions sql/bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1174,8 +1174,7 @@ func runBenchmarkWideTable(b *testing.B, db *gosql.DB, count int) {
const schema = `CREATE TABLE bench.widetable (
f1 INT, f2 INT, f3 INT, f4 INT, f5 INT, f6 INT, f7 INT, f8 INT, f9 INT, f10 INT,
f11 INT, f12 INT, f13 INT, f14 INT, f15 INT, f16 INT, f17 INT, f18 INT, f19 INT, f20 INT,
PRIMARY KEY (f1, f2, f3),
FAMILY (f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17, f18, f19, f20)
PRIMARY KEY (f1, f2, f3)
)`
if _, err := db.Exec(schema); err != nil {
b.Fatal(err)
Expand Down
18 changes: 9 additions & 9 deletions sql/schema_changer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -555,11 +555,11 @@ CREATE UNIQUE INDEX vidx ON t.test (v);
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID)))
tableEnd := tablePrefix.PrefixEnd()
// number of keys == 4 * number of rows; 3 columns and 1 index entry for
// each row.
// number of keys == 3 * number of rows; 2 column families and 1 index entry
// for each row.
if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if e := 4 * (maxValue + 1); len(kvs) != e {
} else if e := 3 * (maxValue + 1); len(kvs) != e {
t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}

Expand All @@ -573,7 +573,7 @@ CREATE UNIQUE INDEX vidx ON t.test (v);
kvDB,
"ALTER TABLE t.test ADD COLUMN x DECIMAL DEFAULT (DECIMAL '1.4')",
maxValue,
5,
4,
backfillNotification)

// Drop column.
Expand All @@ -584,7 +584,7 @@ CREATE UNIQUE INDEX vidx ON t.test (v);
kvDB,
"ALTER TABLE t.test DROP pi",
maxValue,
4,
3,
backfillNotification)

// Add index.
Expand All @@ -595,7 +595,7 @@ CREATE UNIQUE INDEX vidx ON t.test (v);
kvDB,
"CREATE UNIQUE INDEX foo ON t.test (v)",
maxValue,
5,
4,
backfillNotification)

// Drop index.
Expand All @@ -606,7 +606,7 @@ CREATE UNIQUE INDEX vidx ON t.test (v);
kvDB,
"DROP INDEX t.test@vidx",
maxValue,
4,
3,
backfillNotification)

// Verify that the index foo over v is consistent, and that column x has
Expand Down Expand Up @@ -857,7 +857,7 @@ CREATE TABLE t.test (k INT PRIMARY KEY, v INT);
tableEnd := tablePrefix.PrefixEnd()
if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if e := 2*(maxValue+2) + numGarbageValues; len(kvs) != e {
} else if e := 1*(maxValue+2) + numGarbageValues; len(kvs) != e {
t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}

Expand All @@ -877,7 +877,7 @@ CREATE TABLE t.test (k INT PRIMARY KEY, v INT);
numGarbageValues = 0
if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if e := 2*(maxValue+2) + numGarbageValues; len(kvs) != e {
} else if e := 1*(maxValue+2) + numGarbageValues; len(kvs) != e {
t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}
}
Expand Down
26 changes: 8 additions & 18 deletions sql/show_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,8 @@ func TestShowCreateTable(t *testing.T) {
s STRING NULL,
v FLOAT NOT NULL,
t TIMESTAMP NULL DEFAULT NOW(),
FAMILY "primary" (rowid),
FAMILY fam_1_i (i),
FAMILY fam_2_s (s),
FAMILY fam_3_v (v),
FAMILY fam_4_t (t),
FAMILY "primary" (i, v, t, rowid),
FAMILY fam_1_s (s),
CHECK (i > 0)
)`,
},
Expand All @@ -75,11 +72,8 @@ func TestShowCreateTable(t *testing.T) {
s STRING NULL,
v FLOAT NOT NULL,
t TIMESTAMP NULL DEFAULT NOW(),
FAMILY "primary" (rowid),
FAMILY fam_1_i (i),
FAMILY fam_2_s (s),
FAMILY fam_3_v (v),
FAMILY fam_4_t (t),
FAMILY "primary" (i, v, t, rowid),
FAMILY fam_1_s (s),
CHECK (i > 0)
)`,
},
Expand All @@ -92,9 +86,8 @@ func TestShowCreateTable(t *testing.T) {
expect: `CREATE TABLE %s (
i INT NULL,
s STRING NULL,
FAMILY "primary" (rowid),
FAMILY fam_1_i (i),
FAMILY fam_2_s (s),
FAMILY "primary" (i, rowid),
FAMILY fam_1_s (s),
CONSTRAINT ck CHECK (i > 0)
)`,
},
Expand All @@ -121,11 +114,8 @@ func TestShowCreateTable(t *testing.T) {
d DATE NULL,
INDEX idx_if (f, i) STORING (s, d),
UNIQUE INDEX %[1]s_d_key (d),
FAMILY "primary" (rowid),
FAMILY fam_1_i (i),
FAMILY fam_2_f (f),
FAMILY fam_3_s (s),
FAMILY fam_4_d (d)
FAMILY "primary" (i, f, d, rowid),
FAMILY fam_1_s (s)
)`,
},
{
Expand Down
36 changes: 27 additions & 9 deletions sql/sqlbase/structured.go
Original file line number Diff line number Diff line change
Expand Up @@ -456,19 +456,37 @@ func (desc *TableDescriptor) AllocateIDs() error {
return
}
if _, ok := primaryIndexColIDs[col.ID]; ok {
// Primary index columns are required to be assigned to family 0.
desc.Families[0].ColumnNames = append(desc.Families[0].ColumnNames, col.Name)
desc.Families[0].ColumnIDs = append(desc.Families[0].ColumnIDs, col.ID)
return
}
// TODO(dan): This assigns families such that the encoding is exactly the
// same as before column families. A followup commit will implement the
// heuristics described in the families rfc.
familyID := FamilyID(col.ID)
desc.Families = append(desc.Families, ColumnFamilyDescriptor{
ID: familyID,
ColumnNames: []string{col.Name},
ColumnIDs: []ColumnID{col.ID},
})
var familyID FamilyID
if desc.ParentID == keys.SystemDatabaseID {
// TODO(dan): This assigns families such that the encoding is exactly the
// same as before column families. It's used for all system tables because
// reads of them don't go through the normal sql layer, which is where the
// knowledge of families lives. Fix that and remove this workaround.
familyID = FamilyID(col.ID)
desc.Families = append(desc.Families, ColumnFamilyDescriptor{
ID: familyID,
ColumnNames: []string{col.Name},
ColumnIDs: []ColumnID{col.ID},
})
} else {
idx, ok := fitColumnToFamily(*desc, *col)
if !ok {
idx = len(desc.Families)
desc.Families = append(desc.Families, ColumnFamilyDescriptor{
ID: desc.NextFamilyID,
ColumnNames: []string{},
ColumnIDs: []ColumnID{},
})
}
familyID = desc.Families[idx].ID
desc.Families[idx].ColumnNames = append(desc.Families[idx].ColumnNames, col.Name)
desc.Families[idx].ColumnIDs = append(desc.Families[idx].ColumnIDs, col.ID)
}
if familyID >= desc.NextFamilyID {
desc.NextFamilyID = familyID + 1
}
Expand Down
13 changes: 4 additions & 9 deletions sql/sqlbase/structured_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,14 +76,9 @@ func TestAllocateIDs(t *testing.T) {
Families: []ColumnFamilyDescriptor{
{
ID: 0, Name: "primary",
ColumnNames: []string{"a", "b"},
ColumnIDs: []ColumnID{1, 2},
},
{
ID: 3, Name: "fam_3_c",
ColumnNames: []string{"c"},
ColumnIDs: []ColumnID{3},
DefaultColumnID: ColumnID(3),
ColumnNames: []string{"a", "b", "c"},
ColumnIDs: []ColumnID{1, 2, 3},
DefaultColumnID: 3,
},
},
PrimaryIndex: IndexDescriptor{
Expand All @@ -101,7 +96,7 @@ func TestAllocateIDs(t *testing.T) {
},
Privileges: NewDefaultPrivilegeDescriptor(),
NextColumnID: 4,
NextFamilyID: 4,
NextFamilyID: 1,
NextIndexID: 4,
NextMutationID: 1,
FormatVersion: FamilyFormatVersion,
Expand Down
9 changes: 7 additions & 2 deletions sql/testdata/aggregate
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,10 @@ CREATE TABLE xyz (
y INT,
z FLOAT,
INDEX xy (x, y),
INDEX zyx (z, y, x)
INDEX zyx (z, y, x),
FAMILY (x),
FAMILY (y),
FAMILY (z)
)

statement ok
Expand Down Expand Up @@ -705,7 +708,9 @@ false false
statement OK
CREATE TABLE ab (
a INT PRIMARY KEY,
b INT
b INT,
FAMILY (a),
FAMILY (b)
)

statement OK
Expand Down
10 changes: 3 additions & 7 deletions sql/testdata/alter_table
Original file line number Diff line number Diff line change
Expand Up @@ -94,13 +94,9 @@ query ITTT colnames
EXPLAIN (DEBUG) SELECT * FROM t
----
RowIdx Key Value Disposition
0 /t/primary/1 NULL ROW
1 /t/primary/2 NULL PARTIAL
1 /t/primary/2/b 1 PARTIAL
1 /t/primary/2/c 1 ROW
2 /t/primary/3 NULL PARTIAL
2 /t/primary/3/b 2 PARTIAL
2 /t/primary/3/c 1 ROW
0 /t/primary/1 NULL ROW
1 /t/primary/2/b/c /1/1 ROW
2 /t/primary/3/b/c /2/1 ROW

statement ok
ALTER TABLE t DROP b, DROP c
Expand Down
4 changes: 3 additions & 1 deletion sql/testdata/create_index
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
statement ok
CREATE TABLE t (
a INT PRIMARY KEY,
b INT
b INT,
FAMILY (a),
FAMILY (b)
)

statement ok
Expand Down
5 changes: 4 additions & 1 deletion sql/testdata/datetime
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ CREATE TABLE t (
b DATE,
c INTERVAL,
UNIQUE (b),
UNIQUE (c)
UNIQUE (c),
FAMILY (a),
FAMILY (b),
FAMILY (c)
)

statement ok
Expand Down
10 changes: 5 additions & 5 deletions sql/testdata/family
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ SELECT * from abcd WHERE a > 1

# Check the descriptor bookkeeping
statement ok
ALTER TABLE abcd ADD COLUMN f INT
ALTER TABLE abcd ADD COLUMN f DECIMAL

query TT
SHOW CREATE TABLE abcd
Expand All @@ -91,11 +91,11 @@ abcd CREATE TABLE abcd (
c INT NULL,
d INT NULL,
e STRING NULL,
f INT NULL,
f DECIMAL NULL,
CONSTRAINT "primary" PRIMARY KEY (a),
FAMILY f1 (a, b, e),
FAMILY fam_1_c_d (c, d),
FAMILY fam_6_f (f)
FAMILY fam_2_f (f)
)

statement ok
Expand All @@ -107,10 +107,10 @@ SHOW CREATE TABLE abcd
abcd CREATE TABLE abcd (
a INT NOT NULL,
b INT NULL,
f INT NULL,
f DECIMAL NULL,
CONSTRAINT "primary" PRIMARY KEY (a),
FAMILY f1 (a, b),
FAMILY fam_6_f (f)
FAMILY fam_2_f (f)
)

statement error unknown family \"foo\"
Expand Down
6 changes: 5 additions & 1 deletion sql/testdata/select_non_covering_index
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ CREATE TABLE t (
c INT,
d INT,
INDEX b (b),
UNIQUE INDEX c (c)
UNIQUE INDEX c (c),
FAMILY (a),
FAMILY (b),
FAMILY (c),
FAMILY (d)
)

statement ok
Expand Down
6 changes: 5 additions & 1 deletion sql/testdata/select_non_covering_index_filtering
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ CREATE TABLE t (
b INT,
c INT,
s STRING,
INDEX bc (b, c)
INDEX bc (b, c),
FAMILY (a),
FAMILY (b),
FAMILY (c),
FAMILY (s)
)

statement ok
Expand Down
20 changes: 8 additions & 12 deletions sql/testdata/table
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,10 @@ test.users CREATE TABLE "test.users"
CONSTRAINT "primary" PRIMARY KEY (id),
INDEX foo (name),
UNIQUE INDEX bar (id, name),
FAMILY "primary" (id),
FAMILY fam_2_name (name),
FAMILY fam_3_title (title),
FAMILY fam_4_nickname (nickname),
FAMILY fam_5_username (username),
FAMILY fam_6_email (email),
FAMILY "primary" (id, username, email),
FAMILY fam_1_name (name),
FAMILY fam_2_title (title),
FAMILY fam_3_nickname (nickname),
CHECK (LENGTH(nickname) < LENGTH(name)),
CHECK (LENGTH(nickname) < 10)
)
Expand Down Expand Up @@ -225,12 +223,10 @@ test.named_constraints CREATE TABLE "test.named_constraints" (
INDEX foo (name),
UNIQUE INDEX uq2 (username),
UNIQUE INDEX bar (id, name),
FAMILY "primary" (id),
FAMILY fam_2_name (name),
FAMILY fam_3_title (title),
FAMILY fam_4_nickname (nickname),
FAMILY fam_5_username (username),
FAMILY fam_6_email (email),
FAMILY "primary" (id, username, email),
FAMILY fam_1_name (name),
FAMILY fam_2_title (title),
FAMILY fam_3_nickname (nickname),
CONSTRAINT ck2 CHECK (LENGTH(nickname) < LENGTH(name)),
CONSTRAINT ck1 CHECK (LENGTH(nickname) < 10)
)
Expand Down
Loading

0 comments on commit a2c2244

Please sign in to comment.