Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

roachtest,workload: switch non-bundle IMPORT to IMPORT INTO #72097

Merged
merged 2 commits into from
Oct 28, 2021
Merged

roachtest,workload: switch non-bundle IMPORT to IMPORT INTO #72097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ecc973
roachtest: switch import roachtest to use IMPORT INTO
adityamaru Oct 25, 2021
1b4f988
workload: switch workload SQL files to use IMPORT INTO
adityamaru Oct 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions pkg/cmd/roachtest/tests/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ package tests

import (
"context"
gosql "database/sql"
"fmt"
"path/filepath"
"strings"
Expand All @@ -26,6 +27,15 @@ import (
"github.com/cockroachdb/errors"
)

func readCreateTableFromFixture(fixtureURI string, gatewayDB *gosql.DB) (string, error) {
row := make([]byte, 0)
err := gatewayDB.QueryRow(fmt.Sprintf(`SELECT crdb_internal.read_file('%s')`, fixtureURI)).Scan(&row)
if err != nil {
return "", err
}
return string(row), err
}

func registerImportNodeShutdown(r registry.Registry) {
getImportRunner := func(ctx context.Context, gatewayNode int) jobStarter {
startImport := func(c cluster.Cluster) (jobID string, err error) {
Expand All @@ -36,8 +46,7 @@ func registerImportNodeShutdown(r registry.Registry) {
tableName = "part"
}
importStmt := fmt.Sprintf(`
IMPORT TABLE %[1]s
CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/%[1]s.sql?AUTH=implicit'
IMPORT INTO %[1]s
CSV DATA (
'gs://cockroach-fixtures/tpch-csv/sf-100/%[1]s.tbl.1?AUTH=implicit',
'gs://cockroach-fixtures/tpch-csv/sf-100/%[1]s.tbl.2?AUTH=implicit',
Expand All @@ -52,6 +61,17 @@ func registerImportNodeShutdown(r registry.Registry) {
gatewayDB := c.Conn(ctx, gatewayNode)
defer gatewayDB.Close()

createStmt, err := readCreateTableFromFixture(
fmt.Sprintf("gs://cockroach-fixtures/tpch-csv/schema/%s.sql?AUTH=implicit", tableName), gatewayDB)
if err != nil {
return "", err
}

// Create the table to be imported into.
if _, err = gatewayDB.ExecContext(ctx, createStmt); err != nil {
return jobID, err
}

err = gatewayDB.QueryRowContext(ctx, importStmt).Scan(&jobID)
return
}
Expand Down Expand Up @@ -239,13 +259,23 @@ func registerImportTPCH(r registry.Registry) {
t.WorkerStatus(`running import`)
defer t.WorkerStatus()

createStmt, err := readCreateTableFromFixture(
"gs://cockroach-fixtures/tpch-csv/schema/lineitem.sql?AUTH=implicit", conn)
if err != nil {
return err
}

// Create table to import into.
if _, err := conn.ExecContext(ctx, createStmt); err != nil {
return err
}

// Tick once before starting the import, and once after to capture the
// total elapsed time. This is used by roachperf to compute and display
// the average MB/sec per node.
tick()
_, err := conn.Exec(`
IMPORT TABLE csv.lineitem
CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/lineitem.sql?AUTH=implicit'
_, err = conn.Exec(`
IMPORT INTO csv.lineitem
CSV DATA (
'gs://cockroach-fixtures/tpch-csv/sf-100/lineitem.tbl.1?AUTH=implicit',
'gs://cockroach-fixtures/tpch-csv/sf-100/lineitem.tbl.2?AUTH=implicit',
Expand Down
117 changes: 109 additions & 8 deletions pkg/workload/tpch/import-sf1.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,43 @@
-- BACKUP DATABASE tpch TO 'gs://cockroach-fixtures/workload/tpch/scalefactor=1/backup';
--

IMPORT TABLE region CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/region.sql' CSV DATA(
CREATE TABLE region (
r_regionkey INTEGER NOT NULL PRIMARY KEY,
r_name CHAR(25) NOT NULL,
r_comment VARCHAR(152)
);

IMPORT INTO region CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/region.tbl'
) WITH delimiter='|';

IMPORT TABLE nation CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/nation.sql' CSV DATA(
CREATE TABLE nation (
n_nationkey INTEGER NOT NULL PRIMARY KEY,
n_name CHAR(25) NOT NULL,
n_regionkey INTEGER NOT NULL,
n_comment VARCHAR(152),
INDEX n_rk (n_regionkey ASC)
);

IMPORT INTO nation CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/nation.tbl'
) WITH delimiter='|';

ALTER TABLE nation ADD CONSTRAINT nation_fkey_region FOREIGN KEY (n_regionkey) references region (r_regionkey);

IMPORT TABLE part CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/part.sql' CSV DATA(
CREATE TABLE part (
p_partkey INTEGER NOT NULL PRIMARY KEY,
p_name VARCHAR(55) NOT NULL,
p_mfgr CHAR(25) NOT NULL,
p_brand CHAR(10) NOT NULL,
p_type VARCHAR(25) NOT NULL,
p_size INTEGER NOT NULL,
p_container CHAR(10) NOT NULL,
p_retailprice FLOAT NOT NULL,
p_comment VARCHAR(23) NOT NULL
);

IMPORT INTO part CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/part.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/part.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/part.tbl.3',
Expand All @@ -36,7 +62,18 @@ IMPORT TABLE part CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/part.sql
'gs://cockroach-fixtures/tpch-csv/sf-1/part.tbl.8'
) WITH delimiter='|';

IMPORT TABLE supplier CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/supplier.sql' CSV DATA(
CREATE TABLE supplier (
s_suppkey INTEGER NOT NULL PRIMARY KEY,
s_name CHAR(25) NOT NULL,
s_address VARCHAR(40) NOT NULL,
s_nationkey INTEGER NOT NULL,
s_phone CHAR(15) NOT NULL,
s_acctbal FLOAT NOT NULL,
s_comment VARCHAR(101) NOT NULL,
INDEX s_nk (s_nationkey ASC)
);

IMPORT INTO supplier CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/supplier.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/supplier.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/supplier.tbl.3',
Expand All @@ -49,7 +86,17 @@ IMPORT TABLE supplier CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/supp

ALTER TABLE supplier ADD CONSTRAINT supplier_fkey_nation FOREIGN KEY (s_nationkey) references nation (n_nationkey);

IMPORT TABLE partsupp CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/partsupp.sql' CSV DATA(
CREATE TABLE partsupp (
ps_partkey INTEGER NOT NULL,
ps_suppkey INTEGER NOT NULL,
ps_availqty INTEGER NOT NULL,
ps_supplycost FLOAT NOT NULL,
ps_comment VARCHAR(199) NOT NULL,
PRIMARY KEY (ps_partkey, ps_suppkey),
INDEX ps_sk (ps_suppkey ASC)
);

IMPORT INTO partsupp CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/partsupp.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/partsupp.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/partsupp.tbl.3',
Expand All @@ -63,7 +110,19 @@ IMPORT TABLE partsupp CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/part
ALTER TABLE partsupp ADD CONSTRAINT partsupp_fkey_part FOREIGN KEY (ps_partkey) references part (p_partkey);
ALTER TABLE partsupp ADD CONSTRAINT partsupp_fkey_supplier FOREIGN KEY (ps_suppkey) references supplier (s_suppkey);

IMPORT TABLE customer CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/customer.sql' CSV DATA(
CREATE TABLE customer (
c_custkey INTEGER NOT NULL PRIMARY KEY,
c_name VARCHAR(25) NOT NULL,
c_address VARCHAR(40) NOT NULL,
c_nationkey INTEGER NOT NULL,
c_phone CHAR(15) NOT NULL,
c_acctbal FLOAT NOT NULL,
c_mktsegment CHAR(10) NOT NULL,
c_comment VARCHAR(117) NOT NULL,
INDEX c_nk (c_nationkey ASC)
);

IMPORT INTO customer CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/customer.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/customer.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/customer.tbl.3',
Expand All @@ -76,7 +135,21 @@ IMPORT TABLE customer CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/cust

ALTER TABLE customer ADD CONSTRAINT customer_fkey_nation FOREIGN KEY (c_nationkey) references nation (n_nationkey);

IMPORT TABLE orders CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/orders.sql' CSV DATA(
CREATE TABLE orders (
o_orderkey INTEGER NOT NULL PRIMARY KEY,
o_custkey INTEGER NOT NULL,
o_orderstatus CHAR(1) NOT NULL,
o_totalprice FLOAT NOT NULL,
o_orderdate DATE NOT NULL,
o_orderpriority CHAR(15) NOT NULL,
o_clerk CHAR(15) NOT NULL,
o_shippriority INTEGER NOT NULL,
o_comment VARCHAR(79) NOT NULL,
INDEX o_ck (o_custkey ASC),
INDEX o_od (o_orderdate ASC)
);

IMPORT INTO orders CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/orders.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/orders.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/orders.tbl.3',
Expand All @@ -89,7 +162,35 @@ IMPORT TABLE orders CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/orders

ALTER TABLE orders ADD CONSTRAINT orders_fkey_customer FOREIGN KEY (o_custkey) references customer (c_custkey);

IMPORT TABLE lineitem CREATE USING 'gs://cockroach-fixtures/tpch-csv/schema/lineitem.sql' CSV DATA(
CREATE TABLE lineitem (
l_orderkey INTEGER NOT NULL,
l_partkey INTEGER NOT NULL,
l_suppkey INTEGER NOT NULL,
l_linenumber INTEGER NOT NULL,
l_quantity FLOAT NOT NULL,
l_extendedprice FLOAT NOT NULL,
l_discount FLOAT NOT NULL,
l_tax FLOAT NOT NULL,
l_returnflag CHAR(1) NOT NULL,
l_linestatus CHAR(1) NOT NULL,
l_shipdate DATE NOT NULL,
l_commitdate DATE NOT NULL,
l_receiptdate DATE NOT NULL,
l_shipinstruct CHAR(25) NOT NULL,
l_shipmode CHAR(10) NOT NULL,
l_comment VARCHAR(44) NOT NULL,
PRIMARY KEY (l_orderkey, l_linenumber),
INDEX l_ok (l_orderkey ASC),
INDEX l_pk (l_partkey ASC),
INDEX l_sk (l_suppkey ASC),
INDEX l_sd (l_shipdate ASC),
INDEX l_cd (l_commitdate ASC),
INDEX l_rd (l_receiptdate ASC),
INDEX l_pk_sk (l_partkey ASC, l_suppkey ASC),
INDEX l_sk_pk (l_suppkey ASC, l_partkey ASC)
);

IMPORT INTO lineitem CSV DATA(
'gs://cockroach-fixtures/tpch-csv/sf-1/lineitem.tbl.1',
'gs://cockroach-fixtures/tpch-csv/sf-1/lineitem.tbl.2',
'gs://cockroach-fixtures/tpch-csv/sf-1/lineitem.tbl.3',
Expand Down
Loading