Skip to content

Commit

Permalink
opt: increase cost for table descriptor fetch during virtual scan
Browse files Browse the repository at this point in the history 
This commit bumps the cost of each virtual scan to 25*randIOCostFactor
from its previous value of 10*randIOCostFactor. This new value threads
the needle so that a lookup join will still be chosen if the predicate
is very selective, but the plan for the PGJDBC query identified in cockroachdb#55140
no longer includes lookup joins.

Fixes cockroachdb#55140

Release note (performance improvement): Adjusted the cost model in
the optimizer so that the optimizer is less likely to plan a lookup
join into a virtual table. Performing a lookup join into a virtual
table is expensive, so this change will generally result in better
performance for queries involving joins with virtual tables.
  • Loading branch information
@rytaft
rytaft committed Nov 5, 2020
1 parent c534b40 commit 0830131
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 73 deletions.
88 changes: 50 additions & 38 deletions pkg/sql/opt/exec/execbuilder/testdata/join
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,23 +431,23 @@ vectorized: false
│ render 12: relname
│
└── • hash join (inner)
│ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname, generate_series, oid, relname, relnamespace, oid, nspname, objid, refobjid, oid, relname, relkind)
│ columns: (attrelid, attname, attnum, attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname, generate_series, oid, relname, relnamespace, oid, nspname, objid, refobjid, oid, relname, relkind)
│ estimated row count: 110908 (missing stats)
│ equality: (oid) = (objid)
│
├── • hash join (inner)
│ │ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname, generate_series, oid, relname, relnamespace, oid, nspname)
│ │ columns: (attrelid, attname, attnum, attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname, generate_series, oid, relname, relnamespace, oid, nspname)
│ │ estimated row count: 114302 (missing stats)
│ │ equality: (relnamespace) = (oid)
│ │
│ ├── • hash join (inner)
│ │ │ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname, generate_series, oid, relname, relnamespace)
│ │ │ columns: (attrelid, attname, attnum, attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname, generate_series, oid, relname, relnamespace)
│ │ │ estimated row count: 11557 (missing stats)
│ │ │ equality: (attrelid) = (oid)
│ │ │ pred: attnum = confkey[generate_series]
│ │ │
│ │ ├── • hash join (inner)
│ │ │ │ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname, generate_series)
│ │ │ │ columns: (attrelid, attname, attnum, attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname, generate_series)
│ │ │ │ estimated row count: 3502 (missing stats)
│ │ │ │ equality: (attrelid) = (confrelid)
│ │ │ │
Expand All @@ -457,47 +457,59 @@ vectorized: false
│ │ │ │ table: pg_attribute@primary
│ │ │ │
│ │ │ └── • cross join (inner)
│ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname, generate_series)
│ │ │ │ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname, generate_series)
│ │ │ │ estimated row count: 354 (missing stats)
│ │ │ │ pred: attnum = conkey[generate_series]
│ │ │ │
│ │ │ ├── • hash join (inner)
│ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum, oid, nspname)
│ │ │ ├── • merge join (inner)
│ │ │ │ │ columns: (attrelid, attname, attnum, oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname)
│ │ │ │ │ estimated row count: 107 (missing stats)
│ │ │ │ │ equality: (relnamespace) = (oid)
│ │ │ │ │ equality: (attrelid) = (oid)
│ │ │ │ │ merge ordering: +"(attrelid=oid)"
│ │ │ │ │
│ │ │ │ ├── • virtual table lookup join (inner)
│ │ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, attrelid, attname, attnum)
│ │ │ │ │ │ estimated row count: 105 (missing stats)
│ │ │ │ │ │ table: pg_attribute@pg_attribute_attrelid_idx
│ │ │ │ │ │ equality: (oid) = (attrelid)
│ │ │ │ │ │
│ │ │ │ │ └── • virtual table lookup join (inner)
│ │ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey)
│ │ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ │ table: pg_constraint@pg_constraint_conrelid_idx
│ │ │ │ │ │ equality: (oid) = (conrelid)
│ │ │ │ │ │ pred: contype = 'f'
│ │ │ │ │ │
│ │ │ │ │ └── • filter
│ │ │ │ │ │ columns: (oid, relname, relnamespace)
│ │ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ │ filter: relname = 'orders'
│ │ │ │ │ │
│ │ │ │ │ └── • virtual table
│ │ │ │ │ columns: (oid, relname, relnamespace)
│ │ │ │ │ estimated row count: 1000 (missing stats)
│ │ │ │ │ table: pg_class@primary
│ │ │ │ ├── • virtual table
│ │ │ │ │ columns: (attrelid, attname, attnum)
│ │ │ │ │ ordering: +attrelid
│ │ │ │ │ estimated row count: 1000 (missing stats)
│ │ │ │ │ table: pg_attribute@pg_attribute_attrelid_idx
│ │ │ │ │
│ │ │ │ └── • filter
│ │ │ │ │ columns: (oid, nspname)
│ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ filter: nspname = 'public'
│ │ │ │ └── • sort
│ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname)
│ │ │ │ │ ordering: +oid
│ │ │ │ │ estimated row count: 11 (missing stats)
│ │ │ │ │ order: +oid
│ │ │ │ │
│ │ │ │ └── • virtual table
│ │ │ │ columns: (oid, nspname)
│ │ │ │ estimated row count: 1000 (missing stats)
│ │ │ │ table: pg_namespace@primary
│ │ │ │ └── • hash join (inner)
│ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey, oid, nspname)
│ │ │ │ │ estimated row count: 11 (missing stats)
│ │ │ │ │ equality: (relnamespace) = (oid)
│ │ │ │ │
│ │ │ │ ├── • virtual table lookup join (inner)
│ │ │ │ │ │ columns: (oid, relname, relnamespace, oid, conname, contype, condeferrable, condeferred, conrelid, confrelid, confupdtype, confdeltype, conkey, confkey)
│ │ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ │ table: pg_constraint@pg_constraint_conrelid_idx
│ │ │ │ │ │ equality: (oid) = (conrelid)
│ │ │ │ │ │ pred: contype = 'f'
│ │ │ │ │ │
│ │ │ │ │ └── • filter
│ │ │ │ │ │ columns: (oid, relname, relnamespace)
│ │ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ │ filter: relname = 'orders'
│ │ │ │ │ │
│ │ │ │ │ └── • virtual table
│ │ │ │ │ columns: (oid, relname, relnamespace)
│ │ │ │ │ estimated row count: 1000 (missing stats)
│ │ │ │ │ table: pg_class@primary
│ │ │ │ │
│ │ │ │ └── • filter
│ │ │ │ │ columns: (oid, nspname)
│ │ │ │ │ estimated row count: 10 (missing stats)
│ │ │ │ │ filter: nspname = 'public'
│ │ │ │ │
│ │ │ │ └── • virtual table
│ │ │ │ columns: (oid, nspname)
│ │ │ │ estimated row count: 1000 (missing stats)
│ │ │ │ table: pg_namespace@primary
│ │ │ │
│ │ │ └── • project set
│ │ │ │ columns: (generate_series)
Expand Down
19 changes: 12 additions & 7 deletions pkg/sql/opt/exec/execbuilder/testdata/virtual
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,20 +164,25 @@ vectorized: false
│
└── • render
│
└── • virtual table lookup join
│ table: pg_class@pg_class_oid_idx
│ equality: (confrelid) = (oid)
└── • merge join
│ equality: (oid) = (confrelid)
│
├── • virtual table
│ table: pg_class@pg_class_oid_idx
│
└── • virtual table lookup join
│ table: pg_class@pg_class_oid_idx
│ equality: (conrelid) = (oid)
│ pred: oid = 'b'::REGCLASS
│
└── • filter
│ filter: (conrelid = 'b'::REGCLASS) AND (contype = 'f')
└── • sort
│ order: +confrelid
│
└── • virtual table
table: pg_constraint@primary
└── • filter
│ filter: (conrelid = 'b'::REGCLASS) AND (contype = 'f')
│
└── • virtual table
table: pg_constraint@primary

# Test that limits are respected.
query T
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/opt/xform/coster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ const (

// virtualScanTableDescriptorFetchCost is the cost to retrieve the table
// descriptors when performing a virtual table scan.
virtualScanTableDescriptorFetchCost = 10 * randIOCostFactor
virtualScanTableDescriptorFetchCost = 25 * randIOCostFactor

// Input rows to a join are processed in batches of this size.
// See joinreader.go.
Expand Down
20 changes: 10 additions & 10 deletions pkg/sql/opt/xform/testdata/coster/join
View file
Original file line number Diff line number Diff line change
Expand Up @@ -857,32 +857,32 @@ WHERE
project
├── columns: attname:3!null atttypid:4!null typbasetype:50 typtype:32
├── stats: [rows=198]
├── cost: 2724.37877
├── cost: 2844.37877
└── inner-join (merge)
├── columns: attname:3!null atttypid:4!null oid:26!null typtype:32 typbasetype:50
├── left ordering: +26
├── right ordering: +4
├── stats: [rows=198, distinct(4)=17.2927193, null(4)=0, distinct(26)=17.2927193, null(26)=0]
├── cost: 2722.38877
├── cost: 2842.38877
├── fd: (4)==(26), (26)==(4)
├── scan pg_type@secondary [as=t]
│ ├── columns: oid:26!null typtype:32 typbasetype:50
│ ├── stats: [rows=1000, distinct(26)=100, null(26)=0]
│ ├── cost: 1394.02
│ ├── cost: 1454.02
│ └── ordering: +26
├── sort
│ ├── columns: attname:3!null atttypid:4
│ ├── stats: [rows=20, distinct(3)=2, null(3)=0, distinct(4)=18.2927193, null(4)=0.2]
│ ├── cost: 1316.17877
│ ├── cost: 1376.17877
│ ├── ordering: +4
│ └── select
│ ├── columns: attname:3!null atttypid:4
│ ├── stats: [rows=20, distinct(3)=2, null(3)=0, distinct(4)=18.2927193, null(4)=0.2]
│ ├── cost: 1314.04
│ ├── cost: 1374.04
│ ├── scan pg_attribute [as=a]
│ │ ├── columns: attname:3 atttypid:4
│ │ ├── stats: [rows=1000, distinct(3)=100, null(3)=10, distinct(4)=100, null(4)=10]
│ │ └── cost: 1304.02
│ │ └── cost: 1364.02
│ └── filters
│ └── attname:3 IN ('descriptor_id', 'descriptor_name') [outer=(3), constraints=(/3: [/'descriptor_id' - /'descriptor_id'] [/'descriptor_name' - /'descriptor_name']; tight)]
└── filters (true)
Expand All @@ -905,23 +905,23 @@ WHERE
project
├── columns: attname:3!null atttypid:4!null typbasetype:50 typtype:32
├── stats: [rows=99]
├── cost: 2148.69
├── cost: 2808.69
├── fd: ()-->(3)
└── inner-join (lookup pg_type@secondary [as=t])
├── columns: attname:3!null atttypid:4!null oid:26!null typtype:32 typbasetype:50
├── key columns: [4] = [26]
├── stats: [rows=99, distinct(4)=8.5617925, null(4)=0, distinct(26)=8.5617925, null(26)=0]
├── cost: 2147.69
├── cost: 2807.69
├── fd: ()-->(3), (4)==(26), (26)==(4)
├── select
│ ├── columns: attname:3!null atttypid:4
│ ├── stats: [rows=10, distinct(3)=1, null(3)=0, distinct(4)=9.5617925, null(4)=0.1]
│ ├── cost: 1314.04
│ ├── cost: 1374.04
│ ├── fd: ()-->(3)
│ ├── scan pg_attribute [as=a]
│ │ ├── columns: attname:3 atttypid:4
│ │ ├── stats: [rows=1000, distinct(3)=100, null(3)=10, distinct(4)=100, null(4)=10]
│ │ └── cost: 1304.02
│ │ └── cost: 1364.02
│ └── filters
│ └── attname:3 = 'descriptor_id' [outer=(3), constraints=(/3: [/'descriptor_id' - /'descriptor_id']; tight), fd=()-->(3)]
└── filters (true)
4 changes: 2 additions & 2 deletions pkg/sql/opt/xform/testdata/coster/virtual-scan
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ SELECT * FROM information_schema.schemata WHERE SCHEMA_NAME='public'
select
├── columns: catalog_name:2!null schema_name:3!null default_character_set_name:4 sql_path:5 crdb_is_user_defined:6
├── stats: [rows=10, distinct(3)=1, null(3)=0]
├── cost: 1164.04
├── cost: 1224.04
├── fd: ()-->(3)
├── scan schemata
│ ├── columns: catalog_name:2!null schema_name:3!null default_character_set_name:4 sql_path:5 crdb_is_user_defined:6
│ ├── stats: [rows=1000, distinct(2)=100, null(2)=0, distinct(3)=100, null(3)=0]
│ └── cost: 1154.02
│ └── cost: 1214.02
└── filters
└── schema_name:3 = 'public' [outer=(3), constraints=(/3: [/'public' - /'public']; tight), fd=()-->(3)]
Loading

0 comments on commit 0830131

Please sign in to comment.