From 560842472be95c6952d6c9bf4907b28b50bcef55 Mon Sep 17 00:00:00 2001 From: Aditya Maru Date: Thu, 2 Sep 2021 13:10:14 -0400 Subject: [PATCH] importccl: add IMPORT INTO UDT support for default+computed columns This change builds on the work in #69674 and uses the type descriptors stored on the import job during planning to construct a custom type resolver. This resolver is used when hydrating the types used by the table being imported into, and when processing default + computed columns. Informs: #61133 Release note (sql change): IMPORT INTO now supports UDT for default and computed columns. Release justification: fixes for high-priority or high-severity bugs in existing functionality Co-authored-by: Anne Zhu --- pkg/ccl/importccl/BUILD.bazel | 1 + pkg/ccl/importccl/import_processor.go | 14 +- pkg/ccl/importccl/import_processor_test.go | 4 +- pkg/ccl/importccl/import_stmt.go | 14 +- pkg/ccl/importccl/import_stmt_test.go | 75 ++++- pkg/ccl/importccl/import_type_resolver.go | 74 +++++ pkg/ccl/importccl/read_import_avro.go | 2 + pkg/ccl/importccl/read_import_avro_test.go | 7 +- pkg/ccl/importccl/read_import_base.go | 28 +- pkg/ccl/importccl/read_import_base_test.go | 4 + pkg/ccl/importccl/read_import_csv.go | 2 + pkg/ccl/importccl/read_import_mysql.go | 6 +- pkg/ccl/importccl/read_import_mysql_test.go | 4 +- pkg/ccl/importccl/read_import_mysqlout.go | 2 + pkg/ccl/importccl/read_import_pgcopy.go | 2 + pkg/ccl/importccl/read_import_pgdump.go | 3 +- pkg/ccl/importccl/read_import_workload.go | 5 +- pkg/sql/distsql_plan_csv.go | 8 +- pkg/sql/execinfrapb/processors_bulk_io.pb.go | 306 +++++++++++-------- pkg/sql/execinfrapb/processors_bulk_io.proto | 5 +- pkg/sql/row/row_converter.go | 10 +- pkg/sql/rowexec/bulk_row_writer.go | 5 +- 22 files changed, 403 insertions(+), 178 deletions(-) create mode 100644 pkg/ccl/importccl/import_type_resolver.go diff --git a/pkg/ccl/importccl/BUILD.bazel b/pkg/ccl/importccl/BUILD.bazel index ef9f2c343aef..1bb0b6c739a5 100644 --- a/pkg/ccl/importccl/BUILD.bazel +++ b/pkg/ccl/importccl/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "import_processor.go", "import_stmt.go", "import_table_creation.go", + "import_type_resolver.go", "read_import_avro.go", "read_import_base.go", "read_import_csv.go", diff --git a/pkg/ccl/importccl/import_processor.go b/pkg/ccl/importccl/import_processor.go index 3a35b9da6e87..0925491792e5 100644 --- a/pkg/ccl/importccl/import_processor.go +++ b/pkg/ccl/importccl/import_processor.go @@ -216,6 +216,7 @@ func injectTimeIntoEvalCtx(ctx *tree.EvalContext, walltime int64) { func makeInputConverter( ctx context.Context, + semaCtx *tree.SemaContext, spec *execinfrapb.ReadImportDataSpec, evalCtx *tree.EvalContext, kvCh chan row.KVBatch, @@ -275,23 +276,24 @@ func makeInputConverter( return newWorkloadReader(kvCh, singleTable, evalCtx), nil } return newCSVInputReader( - kvCh, spec.Format.Csv, spec.WalltimeNanos, int(spec.ReaderParallelism), + semaCtx, kvCh, spec.Format.Csv, spec.WalltimeNanos, int(spec.ReaderParallelism), singleTable, singleTableTargetCols, evalCtx, seqChunkProvider), nil case roachpb.IOFileFormat_MysqlOutfile: return newMysqloutfileReader( - spec.Format.MysqlOut, kvCh, spec.WalltimeNanos, + semaCtx, spec.Format.MysqlOut, kvCh, spec.WalltimeNanos, int(spec.ReaderParallelism), singleTable, singleTableTargetCols, evalCtx) case roachpb.IOFileFormat_Mysqldump: - return newMysqldumpReader(ctx, kvCh, spec.WalltimeNanos, spec.Tables, evalCtx, spec.Format.MysqlDump) + return newMysqldumpReader(ctx, semaCtx, kvCh, spec.WalltimeNanos, spec.Tables, evalCtx, + spec.Format.MysqlDump) case roachpb.IOFileFormat_PgCopy: - return newPgCopyReader(spec.Format.PgCopy, kvCh, spec.WalltimeNanos, + return newPgCopyReader(semaCtx, spec.Format.PgCopy, kvCh, spec.WalltimeNanos, int(spec.ReaderParallelism), singleTable, singleTableTargetCols, evalCtx) case roachpb.IOFileFormat_PgDump: - return newPgDumpReader(ctx, int64(spec.Progress.JobID), kvCh, spec.Format.PgDump, + return newPgDumpReader(ctx, semaCtx, int64(spec.Progress.JobID), kvCh, spec.Format.PgDump, spec.WalltimeNanos, spec.Tables, evalCtx) case roachpb.IOFileFormat_Avro: return newAvroInputReader( - kvCh, singleTable, spec.Format.Avro, spec.WalltimeNanos, + semaCtx, kvCh, singleTable, spec.Format.Avro, spec.WalltimeNanos, int(spec.ReaderParallelism), evalCtx) default: return nil, errors.Errorf( diff --git a/pkg/ccl/importccl/import_processor_test.go b/pkg/ccl/importccl/import_processor_test.go index 7be883ed2099..3bb5667d8af3 100644 --- a/pkg/ccl/importccl/import_processor_test.go +++ b/pkg/ccl/importccl/import_processor_test.go @@ -114,7 +114,9 @@ func TestConverterFlushesBatches(t *testing.T) { } kvCh := make(chan row.KVBatch, batchSize) - conv, err := makeInputConverter(ctx, converterSpec, &evalCtx, kvCh, nil /* seqChunkProvider */) + semaCtx := tree.MakeSemaContext() + conv, err := makeInputConverter(ctx, &semaCtx, converterSpec, &evalCtx, kvCh, + nil /* seqChunkProvider */) if err != nil { t.Fatalf("makeInputConverter() error = %v", err) } diff --git a/pkg/ccl/importccl/import_stmt.go b/pkg/ccl/importccl/import_stmt.go index 0f98a9ac5e66..a156600e4089 100644 --- a/pkg/ccl/importccl/import_stmt.go +++ b/pkg/ccl/importccl/import_stmt.go @@ -2067,6 +2067,12 @@ func (r *importResumer) Resume(ctx context.Context, execCtx interface{}) error { } } } + + typeDescs := make([]*descpb.TypeDescriptor, len(details.Types)) + for i, t := range details.Types { + typeDescs[i] = t.Desc + } + // If details.Walltime is still 0, then it was not set during // `prepareTableDescsForIngestion`. This indicates that we are in an IMPORT INTO, // and that the walltime was not set in a previous run of IMPORT. @@ -2098,7 +2104,7 @@ func (r *importResumer) Resume(ctx context.Context, execCtx interface{}) error { } } - res, err := ingestWithRetry(ctx, p, r.job, tables, files, format, details.Walltime, + res, err := ingestWithRetry(ctx, p, r.job, tables, typeDescs, files, format, details.Walltime, r.testingKnobs.alwaysFlushJobProgress) if err != nil { return err @@ -2123,7 +2129,7 @@ func (r *importResumer) Resume(ctx context.Context, execCtx interface{}) error { } // If the table being imported into referenced UDTs, ensure that a concurrent - // schema change on any of the types has not modified the type descriptor. If + // schema change on any of the typeDescs has not modified the type descriptor. If // it has, it is unsafe to import the data and we fail the import job. if err := r.checkForUDTModification(ctx, p.ExecCfg()); err != nil { return err @@ -2178,6 +2184,7 @@ func ingestWithRetry( execCtx sql.JobExecContext, job *jobs.Job, tables map[string]*execinfrapb.ReadImportDataSpec_ImportTable, + typeDescs []*descpb.TypeDescriptor, from []string, format roachpb.IOFileFormat, walltime int64, @@ -2206,7 +2213,8 @@ func ingestWithRetry( AttemptNumber: retryCount, RetryError: tracing.RedactAndTruncateError(err), }) - res, err = sql.DistIngest(ctx, execCtx, job, tables, from, format, walltime, alwaysFlushProgress) + res, err = sql.DistIngest(ctx, execCtx, job, tables, typeDescs, from, format, walltime, + alwaysFlushProgress) if err == nil { break } diff --git a/pkg/ccl/importccl/import_stmt_test.go b/pkg/ccl/importccl/import_stmt_test.go index f25086d5e7c6..cba806322a6a 100644 --- a/pkg/ccl/importccl/import_stmt_test.go +++ b/pkg/ccl/importccl/import_stmt_test.go @@ -1200,7 +1200,7 @@ CREATE TABLE t (a duration); }) } -func TestImportUserDefinedTypes(t *testing.T) { +func TestImportIntoUserDefinedTypes(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) ctx := context.Background() @@ -1275,6 +1275,22 @@ func TestImportUserDefinedTypes(t *testing.T) { verifyQuery: "SELECT * FROM t ORDER BY a", expected: [][]string{{"hello", "hello"}, {"hi", "hi"}}, }, + // Test CSV default and computed column imports. + { + create: ` +a greeting, b greeting default 'hi', c greeting +AS ( +CASE a +WHEN 'hello' THEN 'hi' +WHEN 'hi' THEN 'hello' +END +) STORED`, + intoCols: "a", + typ: "CSV", + contents: "hello\nhi\n", + verifyQuery: "SELECT * FROM t ORDER BY a", + expected: [][]string{{"hello", "hi", "hi"}, {"hi", "hi", "hello"}}, + }, // Test AVRO imports. { create: "a greeting, b greeting", @@ -1284,6 +1300,22 @@ func TestImportUserDefinedTypes(t *testing.T) { verifyQuery: "SELECT * FROM t ORDER BY a", expected: [][]string{{"hello", "hello"}, {"hi", "hi"}}, }, + // Test AVRO default and computed column imports. + { + create: ` +a greeting, b greeting, c greeting +AS ( +CASE a +WHEN 'hello' THEN 'hi' +WHEN 'hi' THEN 'hello' +END +) STORED`, + intoCols: "a, b", + typ: "AVRO", + contents: avroData, + verifyQuery: "SELECT * FROM t ORDER BY a", + expected: [][]string{{"hello", "hello", "hi"}, {"hi", "hi", "hello"}}, + }, // Test DELIMITED imports. { create: "a greeting, b greeting", @@ -1293,6 +1325,22 @@ func TestImportUserDefinedTypes(t *testing.T) { verifyQuery: "SELECT * FROM t ORDER BY a", expected: [][]string{{"hello", "hello"}, {"hi", "hi"}}, }, + // Test DELIMITED default and computed column imports. + { + create: ` +a greeting, b greeting default 'hi', c greeting +AS ( +CASE a +WHEN 'hello' THEN 'hi' +WHEN 'hi' THEN 'hello' +END +) STORED`, + intoCols: "a", + typ: "DELIMITED", + contents: "hello\nhi\n", + verifyQuery: "SELECT * FROM t ORDER BY a", + expected: [][]string{{"hello", "hi", "hi"}, {"hi", "hi", "hello"}}, + }, // Test PGCOPY imports. { create: "a greeting, b greeting", @@ -1302,13 +1350,21 @@ func TestImportUserDefinedTypes(t *testing.T) { verifyQuery: "SELECT * FROM t ORDER BY a", expected: [][]string{{"hello", "hello"}, {"hi", "hi"}}, }, - // Test table with default value. + // Test PGCOPY default and computed column imports. { - create: "a greeting, b greeting default 'hi'", - intoCols: "a, b", - typ: "PGCOPY", - contents: "hello\nhi\thi\n", - errString: "type OID 100052 does not exist", + create: ` +a greeting, b greeting default 'hi', c greeting +AS ( +CASE a +WHEN 'hello' THEN 'hi' +WHEN 'hi' THEN 'hello' +END +) STORED`, + intoCols: "a", + typ: "PGCOPY", + contents: "hello\nhi\n", + verifyQuery: "SELECT * FROM t ORDER BY a", + expected: [][]string{{"hello", "hi", "hi"}, {"hi", "hi", "hello"}}, }, // Test table with an invalid enum value. { @@ -3685,6 +3741,7 @@ func BenchmarkCSVConvertRecord(b *testing.B) { }() importCtx := ¶llelImportContext{ + semaCtx: &semaCtx, evalCtx: &evalCtx, tableDesc: tableDesc.ImmutableCopy().(catalog.TableDescriptor), kvCh: kvCh, @@ -4681,7 +4738,7 @@ func BenchmarkDelimitedConvertRecord(b *testing.B) { for i, col := range tableDesc.Columns { cols[i] = tree.Name(col.Name) } - r, err := newMysqloutfileReader(roachpb.MySQLOutfileOptions{ + r, err := newMysqloutfileReader(&semaCtx, roachpb.MySQLOutfileOptions{ RowSeparator: '\n', FieldSeparator: '\t', }, kvCh, 0, 0, @@ -4783,7 +4840,7 @@ func BenchmarkPgCopyConvertRecord(b *testing.B) { for i, col := range tableDesc.Columns { cols[i] = tree.Name(col.Name) } - r, err := newPgCopyReader(roachpb.PgCopyOptions{ + r, err := newPgCopyReader(&semaCtx, roachpb.PgCopyOptions{ Delimiter: '\t', Null: `\N`, MaxRowSize: 4096, diff --git a/pkg/ccl/importccl/import_type_resolver.go b/pkg/ccl/importccl/import_type_resolver.go new file mode 100644 index 000000000000..de1aedfff62f --- /dev/null +++ b/pkg/ccl/importccl/import_type_resolver.go @@ -0,0 +1,74 @@ +// Copyright 2017 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package importccl + +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/sql/catalog" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/errors" + "github.com/lib/pq/oid" +) + +type importTypeResolver struct { + typeIDToDesc map[descpb.ID]*descpb.TypeDescriptor + typeNameToDesc map[string]*descpb.TypeDescriptor +} + +func newImportTypeResolver(typeDescs []*descpb.TypeDescriptor) importTypeResolver { + itr := importTypeResolver{ + typeIDToDesc: make(map[descpb.ID]*descpb.TypeDescriptor), + typeNameToDesc: make(map[string]*descpb.TypeDescriptor), + } + for _, typeDesc := range typeDescs { + itr.typeIDToDesc[typeDesc.GetID()] = typeDesc + itr.typeNameToDesc[typeDesc.GetName()] = typeDesc + } + return itr +} + +var _ tree.TypeReferenceResolver = &importTypeResolver{} + +func (i importTypeResolver) ResolveType( + _ context.Context, _ *tree.UnresolvedObjectName, +) (*types.T, error) { + return nil, errors.New("importTypeResolver does not implement ResolveType") +} + +func (i importTypeResolver) ResolveTypeByOID(ctx context.Context, oid oid.Oid) (*types.T, error) { + id, err := typedesc.UserDefinedTypeOIDToID(oid) + if err != nil { + return nil, err + } + name, desc, err := i.GetTypeDescriptor(ctx, id) + if err != nil { + return nil, err + } + return desc.MakeTypesT(ctx, &name, i) +} + +var _ catalog.TypeDescriptorResolver = &importTypeResolver{} + +// GetTypeDescriptor implements the sqlbase.TypeDescriptorResolver interface. +func (i importTypeResolver) GetTypeDescriptor( + _ context.Context, id descpb.ID, +) (tree.TypeName, catalog.TypeDescriptor, error) { + var desc *descpb.TypeDescriptor + var ok bool + if desc, ok = i.typeIDToDesc[id]; !ok { + return tree.TypeName{}, nil, errors.Newf("type descriptor could not be resolved for type id %d", id) + } + typeDesc := typedesc.NewBuilder(desc).BuildImmutableType() + name := tree.MakeUnqualifiedTypeName(desc.GetName()) + return name, typeDesc, nil +} diff --git a/pkg/ccl/importccl/read_import_avro.go b/pkg/ccl/importccl/read_import_avro.go index 0804b8c6bbd6..211d4dbaaf7b 100644 --- a/pkg/ccl/importccl/read_import_avro.go +++ b/pkg/ccl/importccl/read_import_avro.go @@ -453,6 +453,7 @@ type avroInputReader struct { var _ inputConverter = &avroInputReader{} func newAvroInputReader( + semaCtx *tree.SemaContext, kvCh chan row.KVBatch, tableDesc catalog.TableDescriptor, avroOpts roachpb.AvroOptions, @@ -463,6 +464,7 @@ func newAvroInputReader( return &avroInputReader{ importContext: ¶llelImportContext{ + semaCtx: semaCtx, walltime: walltime, numWorkers: parallelism, evalCtx: evalCtx, diff --git a/pkg/ccl/importccl/read_import_avro_test.go b/pkg/ccl/importccl/read_import_avro_test.go index e80b8d526d25..3d47b7ecd2c9 100644 --- a/pkg/ccl/importccl/read_import_avro_test.go +++ b/pkg/ccl/importccl/read_import_avro_test.go @@ -246,14 +246,15 @@ func (th *testHelper) newRecordStream( opts.SchemaJSON = th.schemaJSON th.genRecordsData(t, format, numRecords, opts.RecordSeparator, records) } + semaCtx := tree.MakeSemaContext() - avro, err := newAvroInputReader(nil, th.schemaTable, opts, 0, 1, &th.evalCtx) + avro, err := newAvroInputReader(&semaCtx, nil, th.schemaTable, opts, 0, 1, &th.evalCtx) require.NoError(t, err) producer, consumer, err := newImportAvroPipeline(avro, &fileReader{Reader: records}) require.NoError(t, err) conv, err := row.NewDatumRowConverter( - context.Background(), th.schemaTable, nil, th.evalCtx.Copy(), nil, + context.Background(), &semaCtx, th.schemaTable, nil, th.evalCtx.Copy(), nil, nil /* seqChunkProvider */, nil, /* metrics */ ) require.NoError(t, err) @@ -595,7 +596,7 @@ func benchmarkAvroImport(b *testing.B, avroOpts roachpb.AvroOptions, testData st input, err := os.Open(testData) require.NoError(b, err) - avro, err := newAvroInputReader(kvCh, + avro, err := newAvroInputReader(&semaCtx, kvCh, tableDesc.ImmutableCopy().(catalog.TableDescriptor), avroOpts, 0, 0, &evalCtx) require.NoError(b, err) diff --git a/pkg/ccl/importccl/read_import_base.go b/pkg/ccl/importccl/read_import_base.go index 4f586b0ed096..112d33fec605 100644 --- a/pkg/ccl/importccl/read_import_base.go +++ b/pkg/ccl/importccl/read_import_base.go @@ -24,7 +24,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cloud" - "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security" "github.com/cockroachdb/cockroach/pkg/sql/catalog" @@ -51,29 +50,21 @@ func runImport( // Used to send ingested import rows to the KV layer. kvCh := make(chan row.KVBatch, 10) - // Install type metadata in all of the import tables. The DB is nil in some - // tests, so check first here. - if flowCtx.Cfg.DB != nil { - if err := flowCtx.Cfg.DB.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { - resolver := flowCtx.TypeResolverFactory.NewTypeResolver(txn) - for _, table := range spec.Tables { - if err := typedesc.HydrateTypesInTableDescriptor(ctx, table.Desc, resolver); err != nil { - return err - } - } - return nil - }); err != nil { + // Install type metadata in all of the import tables. + importResolver := newImportTypeResolver(spec.Types) + for _, table := range spec.Tables { + if err := typedesc.HydrateTypesInTableDescriptor(ctx, table.Desc, importResolver); err != nil { return nil, err } - // Release leases on any accessed types now that type metadata is installed. - flowCtx.TypeResolverFactory.Descriptors.ReleaseAll(ctx) } evalCtx := flowCtx.NewEvalCtx() // TODO(adityamaru): Should we just plumb the flowCtx instead of this // assignment. evalCtx.DB = flowCtx.Cfg.DB - conv, err := makeInputConverter(ctx, spec, evalCtx, kvCh, seqChunkProvider) + semaCtx := tree.MakeSemaContext() + semaCtx.TypeResolver = importResolver + conv, err := makeInputConverter(ctx, &semaCtx, spec, evalCtx, kvCh, seqChunkProvider) if err != nil { return nil, err } @@ -414,6 +405,7 @@ type parallelImportContext struct { walltime int64 // Import time stamp. numWorkers int // Parallelism. batchSize int // Number of records to batch. + semaCtx *tree.SemaContext // Semantic analysis context. evalCtx *tree.EvalContext // Evaluation context. tableDesc catalog.TableDescriptor // Table descriptor we're importing into. targetCols tree.NameList // List of columns to import. nil if importing all columns. @@ -446,8 +438,8 @@ func makeDatumConverter( ctx context.Context, importCtx *parallelImportContext, fileCtx *importFileContext, ) (*row.DatumRowConverter, error) { conv, err := row.NewDatumRowConverter( - ctx, importCtx.tableDesc, importCtx.targetCols, importCtx.evalCtx, importCtx.kvCh, - importCtx.seqChunkProvider, nil /* metrics */) + ctx, importCtx.semaCtx, importCtx.tableDesc, importCtx.targetCols, importCtx.evalCtx, + importCtx.kvCh, importCtx.seqChunkProvider, nil /* metrics */) if err == nil { conv.KvBatch.Source = fileCtx.source } diff --git a/pkg/ccl/importccl/read_import_base_test.go b/pkg/ccl/importccl/read_import_base_test.go index 0bfffb92ff3f..938a76680b07 100644 --- a/pkg/ccl/importccl/read_import_base_test.go +++ b/pkg/ccl/importccl/read_import_base_test.go @@ -131,7 +131,9 @@ func TestParallelImportProducerHandlesConsumerErrors(t *testing.T) { }() // Prepare import context, which flushes to kvCh frequently. + semaCtx := tree.MakeSemaContext() importCtx := ¶llelImportContext{ + semaCtx: &semaCtx, numWorkers: 1, batchSize: 2, evalCtx: testEvalCtx, @@ -170,9 +172,11 @@ func TestParallelImportProducerHandlesCancellation(t *testing.T) { }() // Prepare import context, which flushes to kvCh frequently. + semaCtx := tree.MakeSemaContext() importCtx := ¶llelImportContext{ numWorkers: 1, batchSize: 2, + semaCtx: &semaCtx, evalCtx: testEvalCtx, tableDesc: tabledesc.NewBuilder(&descr).BuildImmutableTable(), kvCh: kvCh, diff --git a/pkg/ccl/importccl/read_import_csv.go b/pkg/ccl/importccl/read_import_csv.go index fb7d9a6bbd1c..8274c72551e8 100644 --- a/pkg/ccl/importccl/read_import_csv.go +++ b/pkg/ccl/importccl/read_import_csv.go @@ -35,6 +35,7 @@ type csvInputReader struct { var _ inputConverter = &csvInputReader{} func newCSVInputReader( + semaCtx *tree.SemaContext, kvCh chan row.KVBatch, opts roachpb.CSVOptions, walltime int64, @@ -51,6 +52,7 @@ func newCSVInputReader( return &csvInputReader{ importCtx: ¶llelImportContext{ + semaCtx: semaCtx, walltime: walltime, numWorkers: parallelism, evalCtx: evalCtx, diff --git a/pkg/ccl/importccl/read_import_mysql.go b/pkg/ccl/importccl/read_import_mysql.go index 900580e23a13..8fdcbcb60e89 100644 --- a/pkg/ccl/importccl/read_import_mysql.go +++ b/pkg/ccl/importccl/read_import_mysql.go @@ -61,6 +61,7 @@ var _ inputConverter = &mysqldumpReader{} func newMysqldumpReader( ctx context.Context, + semaCtx *tree.SemaContext, kvCh chan row.KVBatch, walltime int64, tables map[string]*execinfrapb.ReadImportDataSpec_ImportTable, @@ -75,8 +76,9 @@ func newMysqldumpReader( converters[name] = nil continue } - conv, err := row.NewDatumRowConverter(ctx, tabledesc.NewBuilder(table.Desc).BuildImmutableTable(), - nil /* targetColNames */, evalCtx, kvCh, nil /* seqChunkProvider */, nil /* metrics */) + conv, err := row.NewDatumRowConverter(ctx, semaCtx, tabledesc.NewBuilder(table.Desc). + BuildImmutableTable(), nil /* targetColNames */, evalCtx, kvCh, + nil /* seqChunkProvider */, nil /* metrics */) if err != nil { return nil, err } diff --git a/pkg/ccl/importccl/read_import_mysql_test.go b/pkg/ccl/importccl/read_import_mysql_test.go index 29bc132195bc..14b9b923703a 100644 --- a/pkg/ccl/importccl/read_import_mysql_test.go +++ b/pkg/ccl/importccl/read_import_mysql_test.go @@ -51,10 +51,12 @@ func TestMysqldumpDataReader(t *testing.T) { opts := roachpb.MysqldumpOptions{} kvCh := make(chan row.KVBatch, 50) + semaCtx := tree.MakeSemaContext() // When creating a new dump reader, we need to pass in the walltime that will be used as // a parameter used for generating unique rowid, random, and gen_random_uuid as default // expressions. Here, the parameter doesn't matter so we pass in 0. - converter, err := newMysqldumpReader(ctx, kvCh, 0 /*walltime*/, tables, testEvalCtx, opts) + converter, err := newMysqldumpReader(ctx, &semaCtx, kvCh, 0 /*walltime*/, tables, + testEvalCtx, opts) if err != nil { t.Fatal(err) diff --git a/pkg/ccl/importccl/read_import_mysqlout.go b/pkg/ccl/importccl/read_import_mysqlout.go index f0cefd641a12..177ef290a7d0 100644 --- a/pkg/ccl/importccl/read_import_mysqlout.go +++ b/pkg/ccl/importccl/read_import_mysqlout.go @@ -34,6 +34,7 @@ type mysqloutfileReader struct { var _ inputConverter = &mysqloutfileReader{} func newMysqloutfileReader( + semaCtx *tree.SemaContext, opts roachpb.MySQLOutfileOptions, kvCh chan row.KVBatch, walltime int64, @@ -44,6 +45,7 @@ func newMysqloutfileReader( ) (*mysqloutfileReader, error) { return &mysqloutfileReader{ importCtx: ¶llelImportContext{ + semaCtx: semaCtx, walltime: walltime, numWorkers: parallelism, evalCtx: evalCtx, diff --git a/pkg/ccl/importccl/read_import_pgcopy.go b/pkg/ccl/importccl/read_import_pgcopy.go index 6ed0c32c63eb..fd514e0ef682 100644 --- a/pkg/ccl/importccl/read_import_pgcopy.go +++ b/pkg/ccl/importccl/read_import_pgcopy.go @@ -40,6 +40,7 @@ type pgCopyReader struct { var _ inputConverter = &pgCopyReader{} func newPgCopyReader( + semaCtx *tree.SemaContext, opts roachpb.PgCopyOptions, kvCh chan row.KVBatch, walltime int64, @@ -50,6 +51,7 @@ func newPgCopyReader( ) (*pgCopyReader, error) { return &pgCopyReader{ importCtx: ¶llelImportContext{ + semaCtx: semaCtx, walltime: walltime, numWorkers: parallelism, evalCtx: evalCtx, diff --git a/pkg/ccl/importccl/read_import_pgdump.go b/pkg/ccl/importccl/read_import_pgdump.go index 417a657881b9..9170d3c30947 100644 --- a/pkg/ccl/importccl/read_import_pgdump.go +++ b/pkg/ccl/importccl/read_import_pgdump.go @@ -937,6 +937,7 @@ var _ inputConverter = &pgDumpReader{} // newPgDumpReader creates a new inputConverter for pg_dump files. func newPgDumpReader( ctx context.Context, + semaCtx *tree.SemaContext, jobID int64, kvCh chan row.KVBatch, opts roachpb.PgDumpOptions, @@ -958,7 +959,7 @@ func newPgDumpReader( for i, col := range tableDesc.VisibleColumns() { colSubMap[col.GetName()] = i } - conv, err := row.NewDatumRowConverter(ctx, tableDesc, targetCols, evalCtx, kvCh, + conv, err := row.NewDatumRowConverter(ctx, semaCtx, tableDesc, targetCols, evalCtx, kvCh, nil /* seqChunkProvider */, nil /* metrics */) if err != nil { return nil, err diff --git a/pkg/ccl/importccl/read_import_workload.go b/pkg/ccl/importccl/read_import_workload.go index ed626038c2e9..69d7f5f4b9dd 100644 --- a/pkg/ccl/importccl/read_import_workload.go +++ b/pkg/ccl/importccl/read_import_workload.go @@ -217,8 +217,9 @@ func NewWorkloadKVConverter( // // This worker needs its own EvalContext and DatumAlloc. func (w *WorkloadKVConverter) Worker(ctx context.Context, evalCtx *tree.EvalContext) error { - conv, err := row.NewDatumRowConverter(ctx, w.tableDesc, nil /* targetColNames */, evalCtx, - w.kvCh, nil /* seqChunkProvider */, nil /* metrics */) + semaCtx := tree.MakeSemaContext() + conv, err := row.NewDatumRowConverter(ctx, &semaCtx, w.tableDesc, nil, /* targetColNames */ + evalCtx, w.kvCh, nil /* seqChunkProvider */, nil /* metrics */) if err != nil { return err } diff --git a/pkg/sql/distsql_plan_csv.go b/pkg/sql/distsql_plan_csv.go index 63d7fc657ba3..892353cdf384 100644 --- a/pkg/sql/distsql_plan_csv.go +++ b/pkg/sql/distsql_plan_csv.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/physicalplan" @@ -113,13 +114,13 @@ func getLastImportSummary(job *jobs.Job) roachpb.BulkOpSummary { func makeImportReaderSpecs( job *jobs.Job, tables map[string]*execinfrapb.ReadImportDataSpec_ImportTable, + typeDescs []*descpb.TypeDescriptor, from []string, format roachpb.IOFileFormat, nodes []roachpb.NodeID, walltime int64, user security.SQLUsername, ) []*execinfrapb.ReadImportDataSpec { - // For each input file, assign it to a node. inputSpecs := make([]*execinfrapb.ReadImportDataSpec, 0, len(nodes)) progress := job.Progress() @@ -130,6 +131,7 @@ func makeImportReaderSpecs( if i < len(nodes) { spec := &execinfrapb.ReadImportDataSpec{ Tables: tables, + Types: typeDescs, Format: format, Progress: execinfrapb.JobProgress{ JobID: job.ID(), @@ -195,6 +197,7 @@ func DistIngest( execCtx JobExecContext, job *jobs.Job, tables map[string]*execinfrapb.ReadImportDataSpec_ImportTable, + typeDescs []*descpb.TypeDescriptor, from []string, format roachpb.IOFileFormat, walltime int64, @@ -221,7 +224,8 @@ func DistIngest( accumulatedBulkSummary.BulkOpSummary = getLastImportSummary(job) accumulatedBulkSummary.Unlock() - inputSpecs := makeImportReaderSpecs(job, tables, from, format, nodes, walltime, execCtx.User()) + inputSpecs := makeImportReaderSpecs(job, tables, typeDescs, from, format, nodes, walltime, + execCtx.User()) p := planCtx.NewPhysicalPlan() diff --git a/pkg/sql/execinfrapb/processors_bulk_io.pb.go b/pkg/sql/execinfrapb/processors_bulk_io.pb.go index 2c5738e0d046..41887d84fa70 100644 --- a/pkg/sql/execinfrapb/processors_bulk_io.pb.go +++ b/pkg/sql/execinfrapb/processors_bulk_io.pb.go @@ -256,6 +256,7 @@ type ReadImportDataSpec struct { // User who initiated the import. This is used to check access privileges // when using FileTable ExternalStorage. UserProto github_com_cockroachdb_cockroach_pkg_security.SQLUsernameProto `protobuf:"bytes,15,opt,name=user_proto,json=userProto,casttype=github.com/cockroachdb/cockroach/pkg/security.SQLUsernameProto" json:"user_proto"` + Types []*descpb.TypeDescriptor `protobuf:"bytes,16,rep,name=types" json:"types,omitempty"` } func (m *ReadImportDataSpec) Reset() { *m = ReadImportDataSpec{} } @@ -786,131 +787,132 @@ func init() { } var fileDescriptor_6d46d06b67eadaca = []byte{ - // 1973 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xdd, 0x6e, 0x1b, 0xc7, - 0x15, 0xd6, 0xf2, 0x4f, 0xe4, 0xa1, 0x28, 0x53, 0x63, 0x27, 0xd9, 0xa8, 0xa8, 0x24, 0x30, 0xb6, - 0xcb, 0xba, 0x30, 0x89, 0xd8, 0x6d, 0x60, 0xa4, 0x4d, 0x5c, 0x93, 0xb2, 0x1c, 0xca, 0xb1, 0xad, - 0x2e, 0x2d, 0x1b, 0x08, 0xda, 0x2e, 0x96, 0xbb, 0x23, 0x6a, 0xcc, 0xe5, 0xce, 0x7a, 0x66, 0xd6, - 0x32, 0x7d, 0xdb, 0x17, 0xe8, 0x6d, 0x5f, 0xa2, 0x40, 0x81, 0xbe, 0x41, 0x6f, 0x7c, 0x99, 0xcb, - 0x00, 0x05, 0x84, 0x46, 0xbe, 0xe8, 0x33, 0xd4, 0x57, 0xc5, 0xfc, 0x2c, 0xb9, 0x92, 0x25, 0x4b, - 0x4a, 0x90, 0x1b, 0x6a, 0x35, 0x67, 0xbe, 0x6f, 0xce, 0x39, 0x73, 0xfe, 0x76, 0xa1, 0xc9, 0x9f, - 0x87, 0x6d, 0xfc, 0x12, 0xfb, 0x24, 0xda, 0x61, 0x5e, 0x3c, 0x68, 0xc7, 0x8c, 0xfa, 0x98, 0x73, - 0xca, 0xb8, 0x3b, 0x48, 0xc2, 0x91, 0x4b, 0x68, 0x2b, 0x66, 0x54, 0x50, 0x64, 0xfb, 0xd4, 0x1f, - 0x31, 0xea, 0xf9, 0xbb, 0x2d, 0xfe, 0x3c, 0x6c, 0x05, 0x84, 0x0b, 0xfe, 0x3c, 0x64, 0x49, 0xb4, - 0xfc, 0xe1, 0x33, 0x3a, 0xe0, 0x6d, 0xf9, 0x13, 0x0f, 0xd4, 0x1f, 0x8d, 0x58, 0xb6, 0xd5, 0xee, - 0x78, 0xd0, 0x26, 0xf4, 0xfa, 0x0e, 0x65, 0x63, 0x4f, 0xa4, 0x92, 0x4f, 0xe4, 0xa9, 0xbe, 0x27, - 0xbc, 0x90, 0x0e, 0xdb, 0x01, 0xe6, 0x7e, 0x3c, 0x68, 0x73, 0xc1, 0x12, 0x5f, 0x24, 0x0c, 0x07, - 0x66, 0xd3, 0x95, 0xf7, 0xa9, 0xe6, 0x71, 0x9c, 0x9e, 0x92, 0x08, 0x12, 0xb6, 0x77, 0x43, 0xbf, - 0x2d, 0xc8, 0x18, 0x73, 0xe1, 0x8d, 0x63, 0x23, 0xb9, 0x34, 0xa4, 0x43, 0xaa, 0x1e, 0xdb, 0xf2, - 0xc9, 0xac, 0xa2, 0x54, 0xab, 0xc0, 0x13, 0x9e, 0x59, 0x5b, 0x4a, 0xd7, 0xbc, 0x98, 0xe8, 0xa5, + // 1994 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0xd7, 0xf2, 0x4b, 0xe4, 0xa3, 0x28, 0x53, 0xe3, 0x7c, 0x6c, 0x54, 0x54, 0x12, 0x18, 0xdb, + 0x65, 0x5d, 0x98, 0x44, 0xec, 0x36, 0x30, 0x92, 0x26, 0xae, 0x49, 0x59, 0x0e, 0xe5, 0xd8, 0x56, + 0x97, 0x96, 0x0d, 0x04, 0x6d, 0x17, 0xcb, 0xdd, 0x11, 0x35, 0xe6, 0x72, 0x67, 0x3d, 0x33, 0x6b, + 0x99, 0xbe, 0x16, 0xbd, 0xf7, 0xda, 0x7f, 0xa2, 0x40, 0x81, 0xfe, 0x07, 0xbd, 0xf8, 0x98, 0x63, + 0x80, 0x02, 0x42, 0x23, 0x1f, 0xfa, 0x37, 0xd4, 0xa7, 0x62, 0x3e, 0x96, 0x5c, 0xc9, 0x92, 0x25, + 0x25, 0xc8, 0x85, 0x5a, 0xcd, 0x9b, 0xdf, 0x6f, 0xde, 0x9b, 0x79, 0x5f, 0x33, 0xd0, 0xe4, 0xcf, + 0xc2, 0x36, 0x7e, 0x81, 0x7d, 0x12, 0xed, 0x30, 0x2f, 0x1e, 0xb4, 0x63, 0x46, 0x7d, 0xcc, 0x39, + 0x65, 0xdc, 0x1d, 0x24, 0xe1, 0xc8, 0x25, 0xb4, 0x15, 0x33, 0x2a, 0x28, 0xb2, 0x7d, 0xea, 0x8f, + 0x18, 0xf5, 0xfc, 0xdd, 0x16, 0x7f, 0x16, 0xb6, 0x02, 0xc2, 0x05, 0x7f, 0x16, 0xb2, 0x24, 0x5a, + 0xfe, 0xe0, 0x29, 0x1d, 0xf0, 0xb6, 0xfc, 0x89, 0x07, 0xea, 0x8f, 0x46, 0x2c, 0xdb, 0x6a, 0x76, + 0x3c, 0x68, 0x13, 0x7a, 0x6d, 0x87, 0xb2, 0xb1, 0x27, 0x52, 0xc9, 0xc7, 0x72, 0x55, 0xdf, 0x13, + 0x5e, 0x48, 0x87, 0xed, 0x00, 0x73, 0x3f, 0x1e, 0xb4, 0xb9, 0x60, 0x89, 0x2f, 0x12, 0x86, 0x03, + 0x33, 0xe9, 0xf2, 0xbb, 0x54, 0xf3, 0x38, 0x4e, 0x57, 0x49, 0x04, 0x09, 0xdb, 0xbb, 0xa1, 0xdf, + 0x16, 0x64, 0x8c, 0xb9, 0xf0, 0xc6, 0xb1, 0x91, 0xbc, 0x37, 0xa4, 0x43, 0xaa, 0x3e, 0xdb, 0xf2, + 0xcb, 0x8c, 0xa2, 0x54, 0xab, 0xc0, 0x13, 0x9e, 0x19, 0x5b, 0x4a, 0xc7, 0xbc, 0x98, 0xe8, 0xa1, 0xc6, 0x7f, 0x0b, 0xb0, 0xd8, 0xf1, 0xfc, 0xd1, 0x0e, 0x09, 0x43, 0xcc, 0xfa, 0x31, 0xf6, 0xd1, - 0x3d, 0x28, 0x88, 0x49, 0x8c, 0x6d, 0x6b, 0xcd, 0x6a, 0x2e, 0xde, 0xb8, 0xde, 0x3a, 0xc9, 0x21, - 0xad, 0xc3, 0xb8, 0xd6, 0xe3, 0x49, 0x8c, 0x3b, 0x85, 0xd7, 0xfb, 0xab, 0x73, 0x8e, 0x22, 0x40, - 0x1d, 0x28, 0x0a, 0x6f, 0x10, 0x62, 0x3b, 0xb7, 0x66, 0x35, 0xab, 0x37, 0xae, 0x1e, 0x61, 0xe2, - 0xcf, 0x43, 0x65, 0xdf, 0x63, 0xb9, 0x67, 0x1d, 0x73, 0x9f, 0x91, 0x58, 0x50, 0x66, 0x28, 0x34, - 0x14, 0xdd, 0x85, 0x22, 0x8f, 0xbd, 0x88, 0xdb, 0xf9, 0xb5, 0x7c, 0xb3, 0x7a, 0xe3, 0x97, 0x27, - 0x6b, 0xa3, 0x68, 0x1c, 0xec, 0x05, 0x52, 0x1d, 0x2f, 0x4a, 0x69, 0x14, 0x1a, 0x7d, 0x0a, 0xe5, - 0x20, 0x61, 0x9e, 0x20, 0x34, 0xb2, 0x0b, 0x6b, 0x56, 0x33, 0xdf, 0xf9, 0x40, 0x8a, 0xdf, 0xee, - 0xaf, 0xd6, 0xa4, 0x3b, 0x5b, 0xeb, 0x46, 0xe8, 0x4c, 0xb7, 0xa1, 0x4f, 0x00, 0xfc, 0xdd, 0x24, - 0x1a, 0xb9, 0x9c, 0xbc, 0xc2, 0x76, 0x51, 0x81, 0x34, 0x67, 0x45, 0xad, 0xf7, 0xc9, 0x2b, 0x8c, - 0xee, 0x40, 0x65, 0x8f, 0x11, 0x81, 0xef, 0xf0, 0x47, 0x3b, 0xf6, 0xbc, 0x32, 0xf3, 0xe7, 0x19, - 0x15, 0xe5, 0x9d, 0xb5, 0x76, 0x43, 0xbf, 0xf5, 0x38, 0xbd, 0xb3, 0x94, 0x62, 0x8a, 0x42, 0xb7, - 0xa1, 0xcc, 0xb0, 0x17, 0x28, 0x86, 0xca, 0xd9, 0x19, 0xa6, 0x20, 0xc4, 0xe1, 0x22, 0x89, 0x02, - 0xfc, 0x12, 0x73, 0x57, 0x50, 0x77, 0x60, 0x2e, 0xc5, 0x2e, 0xaf, 0xe5, 0x9b, 0xb5, 0x4e, 0xf7, - 0xed, 0xfe, 0xea, 0xed, 0x21, 0x11, 0xbb, 0xc9, 0xa0, 0xe5, 0xd3, 0x71, 0x7b, 0xca, 0x1c, 0x0c, - 0x66, 0xcf, 0xed, 0x78, 0x34, 0x6c, 0xbf, 0x1b, 0xab, 0xad, 0x9e, 0xa4, 0xed, 0xad, 0x3b, 0x4b, - 0x86, 0xff, 0x31, 0x4d, 0xaf, 0xbc, 0x71, 0x0d, 0x0a, 0xf2, 0xbe, 0x51, 0x15, 0xe6, 0x7b, 0xd1, - 0x0b, 0x2f, 0x24, 0x41, 0x7d, 0x0e, 0x01, 0x94, 0xba, 0x34, 0x4c, 0xc6, 0x51, 0xdd, 0x42, 0x15, - 0x28, 0x2a, 0x78, 0x3d, 0xb7, 0x59, 0x28, 0x97, 0xea, 0xf3, 0x8d, 0x7f, 0x58, 0x50, 0xdd, 0xa4, - 0x83, 0x2d, 0x46, 0x87, 0x0c, 0x73, 0x8e, 0xfe, 0x0c, 0xa5, 0x67, 0x74, 0xe0, 0x92, 0x40, 0x05, - 0x5a, 0xbe, 0x73, 0x4f, 0x9a, 0x75, 0xb0, 0xbf, 0x5a, 0xdc, 0xa4, 0x83, 0xde, 0xfa, 0xdb, 0xfd, - 0xd5, 0xcf, 0xce, 0xa4, 0x76, 0x26, 0x29, 0x5b, 0x0a, 0xe9, 0x14, 0x9f, 0xd1, 0x41, 0x2f, 0x40, - 0x4d, 0x58, 0xf0, 0x69, 0x24, 0x18, 0x19, 0x24, 0xea, 0xda, 0x65, 0x10, 0xe6, 0x8c, 0xf3, 0x0e, - 0x49, 0x90, 0x0d, 0x05, 0x1e, 0x52, 0x61, 0xe7, 0xd7, 0xac, 0x66, 0x31, 0x8d, 0x60, 0xb9, 0xd2, - 0x78, 0x5d, 0x06, 0x24, 0x43, 0xaa, 0x37, 0x8e, 0x29, 0x13, 0xeb, 0x9e, 0xf0, 0x54, 0x86, 0x7c, - 0x01, 0x25, 0x9d, 0xe8, 0x76, 0x59, 0x5d, 0xd8, 0x6a, 0xe6, 0xc2, 0x4c, 0x8a, 0xb5, 0x7a, 0x8f, - 0x36, 0x48, 0x88, 0x37, 0xd4, 0x36, 0xc3, 0x69, 0x40, 0xe8, 0x0a, 0x54, 0xb9, 0x37, 0x8e, 0x43, - 0xac, 0x43, 0x2b, 0x97, 0x39, 0x16, 0xb4, 0x40, 0xc5, 0xd6, 0x13, 0x28, 0xa9, 0x1c, 0xe0, 0x76, - 0x45, 0xc5, 0xfe, 0xad, 0x93, 0x63, 0xff, 0x5d, 0x1d, 0x75, 0x3a, 0xf0, 0xbb, 0x91, 0x60, 0x13, - 0xc5, 0x6d, 0x39, 0x86, 0x0d, 0xdd, 0x83, 0x7c, 0xc2, 0x88, 0x3d, 0xaf, 0x48, 0x7f, 0x73, 0x2e, - 0xd2, 0x6d, 0x46, 0x14, 0xa3, 0x23, 0x19, 0xd0, 0x37, 0x00, 0x0c, 0xf3, 0x64, 0x8c, 0xdd, 0x98, - 0x72, 0x7b, 0x51, 0xf1, 0xfd, 0xf6, 0x5c, 0x7c, 0x8e, 0x82, 0x6f, 0x51, 0xad, 0xa7, 0x53, 0x61, - 0xe9, 0xff, 0xe8, 0x1e, 0x94, 0x63, 0x13, 0x29, 0x76, 0x49, 0x39, 0xf9, 0xca, 0xc9, 0xcc, 0x99, - 0xb0, 0x4a, 0xb3, 0x23, 0x05, 0xa3, 0xdb, 0xf0, 0x31, 0x1f, 0x91, 0xd8, 0x1d, 0x13, 0xce, 0x49, - 0x34, 0x74, 0x77, 0x28, 0xc3, 0x64, 0x18, 0xb9, 0x23, 0x3c, 0xe1, 0x36, 0xac, 0x59, 0xcd, 0xb2, - 0x81, 0x7c, 0x28, 0xb7, 0x3d, 0xd0, 0xbb, 0x36, 0xf4, 0xa6, 0xfb, 0x78, 0xc2, 0xd1, 0x35, 0xa8, - 0xed, 0x79, 0x61, 0x28, 0xcb, 0xc4, 0x43, 0x2f, 0xa2, 0xdc, 0xae, 0x66, 0x4a, 0xc1, 0x61, 0x11, - 0xba, 0x01, 0x4b, 0x4c, 0x55, 0xa0, 0x2d, 0x8f, 0x79, 0x61, 0x88, 0x43, 0xc2, 0xc7, 0x76, 0x2d, - 0x73, 0xbf, 0xef, 0x8a, 0x11, 0x06, 0x48, 0x38, 0x66, 0xae, 0xaa, 0xc7, 0xf6, 0x85, 0x35, 0xab, - 0x59, 0xe9, 0x6c, 0x98, 0xe2, 0xf4, 0xe5, 0xd9, 0x32, 0x17, 0xfb, 0x09, 0x23, 0x62, 0xd2, 0xea, - 0xff, 0xe1, 0xeb, 0x6d, 0x8e, 0x59, 0xe4, 0x8d, 0xf1, 0x96, 0x64, 0x73, 0x2a, 0x92, 0x59, 0x3d, - 0x2e, 0x27, 0x50, 0xd5, 0xce, 0x57, 0x81, 0x81, 0x7e, 0x0f, 0x05, 0x99, 0xe4, 0x2a, 0xf7, 0xce, - 0x57, 0x9a, 0x2d, 0x47, 0x21, 0xd1, 0x65, 0x00, 0xe1, 0xb1, 0x21, 0x16, 0x5d, 0x1a, 0x72, 0x3b, - 0xb7, 0x96, 0x6f, 0x56, 0x8c, 0x3c, 0xb3, 0xbe, 0xcc, 0xa1, 0x9a, 0x89, 0x44, 0x54, 0x87, 0xfc, - 0x08, 0x4f, 0xd4, 0xa9, 0x15, 0x47, 0x3e, 0xa2, 0x87, 0x50, 0x7c, 0xe1, 0x85, 0x49, 0xda, 0x24, - 0xce, 0x17, 0xe4, 0x19, 0x8b, 0x1c, 0x4d, 0xf3, 0x79, 0xee, 0x96, 0xb5, 0xfc, 0x19, 0x94, 0xd3, - 0x48, 0xcd, 0x9e, 0x58, 0xd4, 0x27, 0x5e, 0xca, 0x9e, 0x58, 0xc9, 0xe2, 0x7e, 0x07, 0x8b, 0x87, - 0x23, 0xf2, 0x34, 0x74, 0x3e, 0x83, 0xde, 0x2c, 0x94, 0x2d, 0x55, 0xec, 0xf2, 0xf5, 0xc2, 0x66, - 0xa1, 0x5c, 0xa8, 0x17, 0x37, 0x0b, 0xe5, 0x62, 0xbd, 0xb4, 0x59, 0x28, 0x2f, 0xd4, 0x6b, 0x8d, - 0xef, 0x2d, 0xf8, 0xa8, 0x2f, 0x18, 0xf6, 0xc6, 0xbd, 0x68, 0x88, 0xb9, 0x2c, 0x3c, 0xd3, 0x7a, - 0xd2, 0x86, 0x8b, 0xb1, 0xc7, 0x04, 0x91, 0x8b, 0xae, 0x17, 0x04, 0x32, 0x70, 0x31, 0xb7, 0x2d, - 0xe9, 0x53, 0x07, 0x4d, 0x45, 0x77, 0x52, 0x09, 0xea, 0x00, 0x70, 0xe1, 0x31, 0xe1, 0xca, 0xd0, - 0x33, 0x9e, 0x3b, 0x5b, 0xdf, 0x51, 0x30, 0xb9, 0x8a, 0x7e, 0x05, 0x8b, 0x5c, 0xe9, 0x93, 0x9e, - 0xa8, 0xea, 0x5f, 0x25, 0x0d, 0x6c, 0x2d, 0x33, 0x47, 0xa2, 0xcb, 0xd3, 0x62, 0xad, 0xbb, 0x67, - 0xed, 0x50, 0xb1, 0x36, 0x25, 0xb7, 0xf1, 0x6f, 0x0b, 0x7e, 0x76, 0xc4, 0xc6, 0x0d, 0x46, 0x23, - 0x41, 0xcc, 0x64, 0xe1, 0xc0, 0xc5, 0x5d, 0x32, 0xdc, 0x75, 0xf7, 0x3c, 0x81, 0x99, 0xeb, 0x09, - 0x57, 0x69, 0x63, 0x62, 0xf0, 0x4c, 0xfa, 0xd7, 0x25, 0xfe, 0xa9, 0x84, 0xdf, 0x11, 0x7d, 0x09, - 0x46, 0x1d, 0xa8, 0x09, 0xe6, 0xf9, 0x23, 0x1c, 0xb8, 0x7a, 0x50, 0xc8, 0xa9, 0x3a, 0xf4, 0xd1, - 0x31, 0x25, 0x39, 0x33, 0x16, 0x2c, 0x18, 0x4c, 0x5f, 0x4d, 0x07, 0x33, 0xeb, 0xf2, 0xef, 0xb1, - 0xee, 0x5f, 0xf3, 0x7a, 0x54, 0x4a, 0xe2, 0xe9, 0xc5, 0xdd, 0x4c, 0xa7, 0x13, 0xeb, 0x2c, 0x87, - 0x9a, 0x59, 0xe4, 0x2b, 0xa8, 0x93, 0x48, 0x30, 0x1a, 0x24, 0xfe, 0xf9, 0x94, 0xbe, 0x30, 0x83, - 0x69, 0xbd, 0x6f, 0x42, 0x35, 0xc0, 0x3b, 0x5e, 0x12, 0x0a, 0x57, 0x56, 0x74, 0x7d, 0x7f, 0xc8, - 0x28, 0x0f, 0xeb, 0x5a, 0xb4, 0xed, 0xf4, 0x1c, 0x30, 0xdb, 0xb6, 0x19, 0x41, 0x7f, 0xb1, 0xe0, - 0x62, 0xc2, 0x08, 0x77, 0x07, 0x13, 0x37, 0xa4, 0xbe, 0x17, 0x12, 0x31, 0x71, 0x47, 0x2f, 0xec, - 0x82, 0x52, 0xe1, 0xcb, 0xf7, 0x8f, 0x7b, 0x33, 0xdb, 0x65, 0x2f, 0xe0, 0x9d, 0xc9, 0xd7, 0x86, - 0xe1, 0xfe, 0x0b, 0xdd, 0x6a, 0x2e, 0x1d, 0xec, 0xaf, 0xd6, 0xb7, 0x9d, 0x5e, 0x56, 0xf4, 0xc4, - 0xa9, 0x27, 0x47, 0x36, 0x23, 0x07, 0xaa, 0xe3, 0x17, 0xbe, 0xef, 0xee, 0x90, 0x50, 0x60, 0xa6, - 0xc6, 0xab, 0xc5, 0x43, 0x21, 0x90, 0xda, 0xff, 0xe0, 0x49, 0xb7, 0xbb, 0xa1, 0x36, 0xcd, 0x2c, - 0x9b, 0xad, 0x39, 0x20, 0x59, 0xf4, 0x33, 0xfa, 0x0a, 0x00, 0x47, 0x3e, 0x9b, 0xc4, 0xaa, 0xdf, - 0xeb, 0xae, 0xd1, 0x3c, 0x86, 0x52, 0x36, 0xe6, 0xbb, 0xd3, 0x8d, 0x8f, 0xd4, 0x2f, 0x77, 0x32, - 0x58, 0xf4, 0x08, 0x96, 0x06, 0xca, 0x5a, 0x37, 0x93, 0x66, 0xe7, 0x18, 0xef, 0x2e, 0x68, 0x74, - 0x7f, 0x9a, 0x6c, 0xf7, 0xc1, 0x2c, 0xb9, 0x38, 0x0a, 0x34, 0x5d, 0xf9, 0xec, 0x74, 0x35, 0x8d, - 0xbd, 0x1b, 0x05, 0x8a, 0x6c, 0x1b, 0x4a, 0xf1, 0xc8, 0x25, 0x41, 0x3a, 0x18, 0xdc, 0x3c, 0xf3, - 0x9d, 0x6d, 0x8d, 0x7a, 0x81, 0x99, 0x09, 0x2a, 0x32, 0xbe, 0xb7, 0xee, 0xf7, 0xd6, 0xb9, 0x53, - 0x8c, 0xe5, 0xf2, 0x91, 0x46, 0x04, 0x3f, 0x55, 0x23, 0xea, 0xc2, 0x07, 0xc7, 0x86, 0xce, 0x31, - 0xbd, 0xe1, 0xe4, 0x4a, 0x7d, 0x0b, 0x60, 0x66, 0x4b, 0x16, 0x59, 0x38, 0x06, 0x59, 0xce, 0x20, - 0x1b, 0x1c, 0x2e, 0x38, 0x98, 0x0b, 0xca, 0xb0, 0x0c, 0x03, 0x95, 0xc5, 0x9f, 0x43, 0x3e, 0x20, - 0xcc, 0x94, 0xa1, 0xc6, 0x31, 0x01, 0x73, 0xf7, 0xa5, 0x90, 0xc6, 0x84, 0x7d, 0x41, 0x99, 0x37, - 0x4c, 0x5f, 0x72, 0x24, 0x48, 0xce, 0x8e, 0xb1, 0x27, 0x76, 0xb5, 0x86, 0xe9, 0xec, 0x28, 0x57, - 0xb2, 0x8d, 0xa0, 0xf1, 0x00, 0xc0, 0xbc, 0x9e, 0x48, 0xe5, 0x2e, 0x43, 0x89, 0x86, 0x41, 0x3a, - 0xf9, 0xd6, 0x66, 0xe5, 0xe6, 0x51, 0x18, 0xc8, 0x72, 0x43, 0xc3, 0xa0, 0x17, 0xa0, 0x8f, 0xa1, - 0x1c, 0xe1, 0x3d, 0x57, 0x75, 0x69, 0xc9, 0xbe, 0xe0, 0xcc, 0x47, 0x78, 0x4f, 0x36, 0xe3, 0xc6, - 0x3f, 0x2d, 0xa8, 0x1b, 0x23, 0x64, 0x21, 0xd0, 0x4e, 0xf8, 0x14, 0x0a, 0xb2, 0x96, 0x18, 0x33, - 0x4e, 0x29, 0x25, 0x6a, 0xab, 0x7c, 0xb9, 0xda, 0x21, 0x72, 0xc0, 0xcc, 0x9d, 0xf6, 0x72, 0x75, - 0xc4, 0x65, 0x69, 0x41, 0x53, 0x68, 0x74, 0x15, 0xaa, 0xe9, 0xb8, 0xd5, 0x0b, 0x5e, 0x9a, 0x1a, - 0xaa, 0x77, 0x64, 0x05, 0x8d, 0xff, 0xe5, 0xa6, 0xbe, 0x9f, 0x56, 0xd0, 0x0d, 0x58, 0x60, 0x7a, - 0x49, 0x67, 0xc5, 0x39, 0x7a, 0x41, 0xd5, 0x00, 0x55, 0x4e, 0x1c, 0xce, 0xfd, 0xdc, 0x8f, 0xc8, - 0xfd, 0x0e, 0x94, 0x18, 0x56, 0xd3, 0xa1, 0x7e, 0xe5, 0xbc, 0x7c, 0xea, 0x2b, 0xe7, 0x08, 0x4f, - 0xd2, 0x09, 0x5f, 0x23, 0xe5, 0xe8, 0x6e, 0x32, 0x54, 0x57, 0xd5, 0x5f, 0x9f, 0xea, 0xd9, 0x33, - 0xa5, 0xe8, 0x8f, 0x08, 0xfb, 0xbf, 0xe5, 0xe0, 0x62, 0x3f, 0x0e, 0x89, 0xb8, 0x13, 0x05, 0x7d, - 0xdf, 0x13, 0xc2, 0xb4, 0xe4, 0x3f, 0x41, 0x49, 0xbd, 0xcd, 0xa6, 0x2d, 0xec, 0xf6, 0xc9, 0x9a, - 0x1e, 0x03, 0x4f, 0xb5, 0x57, 0xfa, 0x74, 0x25, 0x4f, 0xea, 0x08, 0x4d, 0x9a, 0x71, 0x66, 0xee, - 0x87, 0x3a, 0x73, 0xd9, 0x85, 0xa5, 0x77, 0x8e, 0x41, 0x9b, 0x30, 0x8f, 0xe5, 0x2b, 0x1c, 0x4e, - 0x15, 0xbf, 0x76, 0xaa, 0x8b, 0xa7, 0xa9, 0x62, 0xf8, 0x53, 0x82, 0xc6, 0xdf, 0xf3, 0x50, 0xeb, - 0xf6, 0x9f, 0x3c, 0x95, 0xaf, 0xe4, 0xda, 0x2b, 0x57, 0x65, 0x63, 0xe5, 0x82, 0x44, 0xfa, 0x8b, - 0x81, 0x95, 0x49, 0xee, 0xac, 0x00, 0xfd, 0x02, 0x16, 0x64, 0x8d, 0x73, 0x63, 0xe5, 0x91, 0xe8, - 0x50, 0x15, 0xa8, 0xaa, 0xea, 0xa7, 0x05, 0xe8, 0x0b, 0x98, 0xa7, 0x3a, 0xd6, 0x54, 0x7a, 0x54, - 0x8f, 0x6d, 0x75, 0xdd, 0xfe, 0x13, 0x13, 0x90, 0xa9, 0x86, 0x06, 0x33, 0xfb, 0x16, 0xc1, 0xe8, - 0x1e, 0x37, 0x23, 0x58, 0xf6, 0x5b, 0x84, 0x43, 0xf7, 0xf8, 0x91, 0x0f, 0x16, 0xf3, 0xc7, 0x7f, - 0xb0, 0xf8, 0x23, 0x2c, 0xf9, 0x74, 0x1c, 0xcb, 0x94, 0x94, 0xc3, 0xa6, 0x4f, 0x03, 0xec, 0x9b, - 0xee, 0xfb, 0x9e, 0xf4, 0x97, 0x59, 0xd3, 0x9d, 0xc1, 0xd2, 0x61, 0x2c, 0xc3, 0xd4, 0x95, 0x44, - 0x47, 0x5a, 0x48, 0xe9, 0x27, 0x6a, 0x21, 0x8d, 0xa7, 0xb0, 0xd4, 0x49, 0x42, 0x69, 0x75, 0xe6, - 0xce, 0xa6, 0x5f, 0x9b, 0xac, 0x1f, 0xfc, 0xb5, 0xe9, 0xda, 0x15, 0xb8, 0x70, 0xc4, 0x54, 0x54, - 0x86, 0xc2, 0x43, 0x1a, 0xe1, 0xfa, 0x9c, 0x7c, 0xba, 0xf7, 0x8a, 0xc4, 0x75, 0xab, 0x73, 0xfd, - 0xf5, 0xf7, 0x2b, 0x73, 0xaf, 0x0f, 0x56, 0xac, 0x6f, 0x0f, 0x56, 0xac, 0xef, 0x0e, 0x56, 0xac, - 0xff, 0x1c, 0xac, 0x58, 0x7f, 0x7d, 0xb3, 0x32, 0xf7, 0xed, 0x9b, 0x95, 0xb9, 0xef, 0xde, 0xac, - 0xcc, 0x7d, 0x53, 0xcd, 0x7c, 0xd0, 0xfb, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07, 0x00, 0x6a, - 0x48, 0x7d, 0x14, 0x00, 0x00, + 0x5d, 0x28, 0x88, 0x49, 0x8c, 0x6d, 0x6b, 0xcd, 0x6a, 0x2e, 0x5e, 0xbf, 0xd6, 0x3a, 0x69, 0x43, + 0x5a, 0x87, 0x71, 0xad, 0x47, 0x93, 0x18, 0x77, 0x0a, 0xaf, 0xf6, 0x57, 0xe7, 0x1c, 0x45, 0x80, + 0x3a, 0x50, 0x14, 0xde, 0x20, 0xc4, 0x76, 0x6e, 0xcd, 0x6a, 0x56, 0xaf, 0x5f, 0x39, 0xc2, 0xc4, + 0x9f, 0x85, 0xca, 0xbe, 0x47, 0x72, 0xce, 0x3a, 0xe6, 0x3e, 0x23, 0xb1, 0xa0, 0xcc, 0x50, 0x68, + 0x28, 0xba, 0x03, 0x45, 0x1e, 0x7b, 0x11, 0xb7, 0xf3, 0x6b, 0xf9, 0x66, 0xf5, 0xfa, 0x2f, 0x4f, + 0xd6, 0x46, 0xd1, 0x38, 0xd8, 0x0b, 0xa4, 0x3a, 0x5e, 0x94, 0xd2, 0x28, 0x34, 0xfa, 0x04, 0xca, + 0x41, 0xc2, 0x3c, 0x41, 0x68, 0x64, 0x17, 0xd6, 0xac, 0x66, 0xbe, 0xf3, 0xbe, 0x14, 0xbf, 0xd9, + 0x5f, 0xad, 0xc9, 0xed, 0x6c, 0xad, 0x1b, 0xa1, 0x33, 0x9d, 0x86, 0x3e, 0x06, 0xf0, 0x77, 0x93, + 0x68, 0xe4, 0x72, 0xf2, 0x12, 0xdb, 0x45, 0x05, 0xd2, 0x9c, 0x15, 0x35, 0xde, 0x27, 0x2f, 0x31, + 0xba, 0x0d, 0x95, 0x3d, 0x46, 0x04, 0xbe, 0xcd, 0x1f, 0xee, 0xd8, 0xf3, 0xca, 0xcc, 0x9f, 0x67, + 0x54, 0x94, 0x67, 0xd6, 0xda, 0x0d, 0xfd, 0xd6, 0xa3, 0xf4, 0xcc, 0x52, 0x8a, 0x29, 0x0a, 0xdd, + 0x82, 0x32, 0xc3, 0x5e, 0xa0, 0x18, 0x2a, 0x67, 0x67, 0x98, 0x82, 0x10, 0x87, 0x8b, 0x24, 0x0a, + 0xf0, 0x0b, 0xcc, 0x5d, 0x41, 0xdd, 0x81, 0x39, 0x14, 0xbb, 0xbc, 0x96, 0x6f, 0xd6, 0x3a, 0xdd, + 0x37, 0xfb, 0xab, 0xb7, 0x86, 0x44, 0xec, 0x26, 0x83, 0x96, 0x4f, 0xc7, 0xed, 0x29, 0x73, 0x30, + 0x98, 0x7d, 0xb7, 0xe3, 0xd1, 0xb0, 0xfd, 0xb6, 0xaf, 0xb6, 0x7a, 0x92, 0xb6, 0xb7, 0xee, 0x2c, + 0x19, 0xfe, 0x47, 0x34, 0x3d, 0xf2, 0xc6, 0x55, 0x28, 0xc8, 0xf3, 0x46, 0x55, 0x98, 0xef, 0x45, + 0xcf, 0xbd, 0x90, 0x04, 0xf5, 0x39, 0x04, 0x50, 0xea, 0xd2, 0x30, 0x19, 0x47, 0x75, 0x0b, 0x55, + 0xa0, 0xa8, 0xe0, 0xf5, 0xdc, 0x66, 0xa1, 0x5c, 0xaa, 0xcf, 0x37, 0xfe, 0x61, 0x41, 0x75, 0x93, + 0x0e, 0xb6, 0x18, 0x1d, 0x32, 0xcc, 0x39, 0xfa, 0x13, 0x94, 0x9e, 0xd2, 0x81, 0x4b, 0x02, 0xe5, + 0x68, 0xf9, 0xce, 0x5d, 0x69, 0xd6, 0xc1, 0xfe, 0x6a, 0x71, 0x93, 0x0e, 0x7a, 0xeb, 0x6f, 0xf6, + 0x57, 0x3f, 0x3d, 0x93, 0xda, 0x99, 0xa0, 0x6c, 0x29, 0xa4, 0x53, 0x7c, 0x4a, 0x07, 0xbd, 0x00, + 0x35, 0x61, 0xc1, 0xa7, 0x91, 0x60, 0x64, 0x90, 0xa8, 0x63, 0x97, 0x4e, 0x98, 0x33, 0x9b, 0x77, + 0x48, 0x82, 0x6c, 0x28, 0xf0, 0x90, 0x0a, 0x3b, 0xbf, 0x66, 0x35, 0x8b, 0xa9, 0x07, 0xcb, 0x91, + 0xc6, 0x5f, 0x2a, 0x80, 0xa4, 0x4b, 0xf5, 0xc6, 0x31, 0x65, 0x62, 0xdd, 0x13, 0x9e, 0x8a, 0x90, + 0x2f, 0xa0, 0xa4, 0x03, 0xdd, 0x2e, 0xab, 0x03, 0x5b, 0xcd, 0x1c, 0x98, 0x09, 0xb1, 0x56, 0xef, + 0xe1, 0x06, 0x09, 0xf1, 0x86, 0x9a, 0x66, 0x38, 0x0d, 0x08, 0x5d, 0x86, 0x2a, 0xf7, 0xc6, 0x71, + 0x88, 0xb5, 0x6b, 0xe5, 0x32, 0xcb, 0x82, 0x16, 0x28, 0xdf, 0x7a, 0x0c, 0x25, 0x15, 0x03, 0xdc, + 0xae, 0x28, 0xdf, 0xbf, 0x79, 0xb2, 0xef, 0xbf, 0xad, 0xa3, 0x0e, 0x07, 0x7e, 0x27, 0x12, 0x6c, + 0xa2, 0xb8, 0x2d, 0xc7, 0xb0, 0xa1, 0xbb, 0x90, 0x4f, 0x18, 0xb1, 0xe7, 0x15, 0xe9, 0x6f, 0xce, + 0x45, 0xba, 0xcd, 0x88, 0x62, 0x74, 0x24, 0x03, 0xfa, 0x06, 0x80, 0x61, 0x9e, 0x8c, 0xb1, 0x1b, + 0x53, 0x6e, 0x2f, 0x2a, 0xbe, 0xcf, 0xcf, 0xc5, 0xe7, 0x28, 0xf8, 0x16, 0xd5, 0x7a, 0x3a, 0x15, + 0x96, 0xfe, 0x8f, 0xee, 0x42, 0x39, 0x36, 0x9e, 0x62, 0x97, 0xd4, 0x26, 0x5f, 0x3e, 0x99, 0x39, + 0xe3, 0x56, 0x69, 0x74, 0xa4, 0x60, 0x74, 0x0b, 0x3e, 0xe2, 0x23, 0x12, 0xbb, 0x63, 0xc2, 0x39, + 0x89, 0x86, 0xee, 0x0e, 0x65, 0x98, 0x0c, 0x23, 0x77, 0x84, 0x27, 0xdc, 0x86, 0x35, 0xab, 0x59, + 0x36, 0x90, 0x0f, 0xe4, 0xb4, 0xfb, 0x7a, 0xd6, 0x86, 0x9e, 0x74, 0x0f, 0x4f, 0x38, 0xba, 0x0a, + 0xb5, 0x3d, 0x2f, 0x0c, 0x65, 0x9a, 0x78, 0xe0, 0x45, 0x94, 0xdb, 0xd5, 0x4c, 0x2a, 0x38, 0x2c, + 0x42, 0xd7, 0x61, 0x89, 0xa9, 0x0c, 0xb4, 0xe5, 0x31, 0x2f, 0x0c, 0x71, 0x48, 0xf8, 0xd8, 0xae, + 0x65, 0xce, 0xf7, 0x6d, 0x31, 0xc2, 0x00, 0x09, 0xc7, 0xcc, 0x55, 0xf9, 0xd8, 0xbe, 0xb0, 0x66, + 0x35, 0x2b, 0x9d, 0x0d, 0x93, 0x9c, 0xbe, 0x3c, 0x5b, 0xe4, 0x62, 0x3f, 0x61, 0x44, 0x4c, 0x5a, + 0xfd, 0xdf, 0x7f, 0xbd, 0xcd, 0x31, 0x8b, 0xbc, 0x31, 0xde, 0x92, 0x6c, 0x4e, 0x45, 0x32, 0xab, + 0x4f, 0xf4, 0x39, 0x14, 0x65, 0x52, 0xe6, 0x76, 0x5d, 0x9d, 0xd3, 0xe5, 0x93, 0x92, 0xf1, 0x24, + 0xce, 0xe4, 0x62, 0x47, 0x63, 0x96, 0x13, 0xa8, 0xea, 0x93, 0x53, 0x5e, 0x85, 0x7e, 0x07, 0x05, + 0x99, 0x21, 0x54, 0xe0, 0x9e, 0x2f, 0xaf, 0x5b, 0x8e, 0x42, 0xa2, 0x4b, 0x00, 0xc2, 0x63, 0x43, + 0x2c, 0xba, 0x34, 0xe4, 0x76, 0x6e, 0x2d, 0xdf, 0xac, 0x18, 0x79, 0x66, 0x7c, 0x99, 0x43, 0x35, + 0xe3, 0xc6, 0xa8, 0x0e, 0xf9, 0x11, 0x9e, 0xa8, 0x55, 0x2b, 0x8e, 0xfc, 0x44, 0x0f, 0xa0, 0xf8, + 0xdc, 0x0b, 0x93, 0xb4, 0xc2, 0x9c, 0x2f, 0x42, 0x32, 0x16, 0x39, 0x9a, 0xe6, 0xb3, 0xdc, 0x4d, + 0x6b, 0xf9, 0x53, 0x28, 0xa7, 0x6e, 0x9e, 0x5d, 0xb1, 0xa8, 0x57, 0x7c, 0x2f, 0xbb, 0x62, 0x25, + 0x8b, 0xfb, 0x2d, 0x2c, 0x1e, 0x76, 0xe7, 0xd3, 0xd0, 0xf9, 0x0c, 0x7a, 0xb3, 0x50, 0xb6, 0x54, + 0xa6, 0xcc, 0xd7, 0x0b, 0x9b, 0x85, 0x72, 0xa1, 0x5e, 0xdc, 0x2c, 0x94, 0x8b, 0xf5, 0xd2, 0x66, + 0xa1, 0xbc, 0x50, 0xaf, 0x35, 0xbe, 0xb7, 0xe0, 0xc3, 0xbe, 0x60, 0xd8, 0x1b, 0xf7, 0xa2, 0x21, + 0xe6, 0x32, 0x6b, 0x4d, 0x93, 0x51, 0x1b, 0x2e, 0xc6, 0x1e, 0x13, 0x44, 0x0e, 0xba, 0x5e, 0x10, + 0x48, 0xaf, 0xc7, 0xdc, 0xb6, 0xe4, 0x9e, 0x3a, 0x68, 0x2a, 0xba, 0x9d, 0x4a, 0x50, 0x07, 0x80, + 0x0b, 0x8f, 0x09, 0x57, 0xfa, 0xad, 0xd9, 0xb9, 0xb3, 0x15, 0x2d, 0x05, 0x93, 0xa3, 0xe8, 0x57, + 0xb0, 0xc8, 0x95, 0x3e, 0xe9, 0x8a, 0x2a, 0x79, 0x56, 0xd2, 0xa8, 0xd0, 0x32, 0xb3, 0x24, 0xba, + 0x34, 0xcd, 0xf4, 0xba, 0xf4, 0xd6, 0x0e, 0x65, 0x7a, 0x93, 0xaf, 0x1b, 0xff, 0xb6, 0xe0, 0x67, + 0x47, 0x6c, 0xdc, 0x60, 0x34, 0x12, 0xc4, 0xb4, 0x25, 0x0e, 0x5c, 0xdc, 0x25, 0xc3, 0x5d, 0x77, + 0xcf, 0x13, 0x98, 0xb9, 0x9e, 0x70, 0x95, 0x36, 0xc6, 0x07, 0xcf, 0xa4, 0x7f, 0x5d, 0xe2, 0x9f, + 0x48, 0xf8, 0x6d, 0xd1, 0x97, 0x60, 0xd4, 0x81, 0x9a, 0x60, 0x9e, 0x3f, 0xc2, 0x81, 0xab, 0xbb, + 0x8c, 0x9c, 0x0a, 0x8e, 0x0f, 0x8f, 0xc9, 0xe7, 0x99, 0x9e, 0x62, 0xc1, 0x60, 0xfa, 0xaa, 0xb5, + 0x98, 0x59, 0x97, 0x7f, 0x87, 0x75, 0xff, 0x9a, 0xd7, 0x7d, 0x56, 0x12, 0x4f, 0x0f, 0xee, 0x46, + 0xda, 0xda, 0x58, 0x67, 0x59, 0xd4, 0x34, 0x32, 0x5f, 0x41, 0x9d, 0x44, 0x82, 0xd1, 0x20, 0xf1, + 0xcf, 0xa7, 0xf4, 0x85, 0x19, 0x4c, 0xeb, 0x7d, 0x03, 0xaa, 0x01, 0xde, 0xf1, 0x92, 0x50, 0xb8, + 0xb2, 0x1c, 0xe8, 0xf3, 0x43, 0x46, 0x79, 0x58, 0xd7, 0xa2, 0x6d, 0xa7, 0xe7, 0x80, 0x99, 0xb6, + 0xcd, 0x08, 0xfa, 0xb3, 0x05, 0x17, 0x13, 0x46, 0xb8, 0x3b, 0x98, 0xb8, 0x21, 0xf5, 0xbd, 0x90, + 0x88, 0x89, 0x3b, 0x7a, 0x6e, 0x17, 0x94, 0x0a, 0x5f, 0xbe, 0xbb, 0x57, 0x9c, 0xd9, 0x2e, 0x0b, + 0x09, 0xef, 0x4c, 0xbe, 0x36, 0x0c, 0xf7, 0x9e, 0xeb, 0x3a, 0xf5, 0xde, 0xc1, 0xfe, 0x6a, 0x7d, + 0xdb, 0xe9, 0x65, 0x45, 0x8f, 0x9d, 0x7a, 0x72, 0x64, 0x32, 0x72, 0xa0, 0x3a, 0x7e, 0xee, 0xfb, + 0xee, 0x0e, 0x09, 0x05, 0x66, 0xaa, 0x37, 0x5b, 0x3c, 0xe4, 0x02, 0xa9, 0xfd, 0xf7, 0x1f, 0x77, + 0xbb, 0x1b, 0x6a, 0xd2, 0xcc, 0xb2, 0xd9, 0x98, 0x03, 0x92, 0x45, 0x7f, 0xa3, 0xaf, 0x00, 0x70, + 0xe4, 0xb3, 0x49, 0xac, 0x9a, 0x05, 0x5d, 0x72, 0x9a, 0xc7, 0x50, 0xca, 0xaa, 0x7e, 0x67, 0x3a, + 0xf1, 0xa1, 0xfa, 0xe5, 0x4e, 0x06, 0x8b, 0x1e, 0xc2, 0xd2, 0x40, 0x59, 0xeb, 0x66, 0xc2, 0xec, + 0x1c, 0xbd, 0xe1, 0x05, 0x8d, 0xee, 0x4f, 0x83, 0xed, 0x1e, 0x98, 0x21, 0x17, 0x47, 0x81, 0xa6, + 0x2b, 0x9f, 0x9d, 0xae, 0xa6, 0xb1, 0x77, 0xa2, 0x40, 0x91, 0x6d, 0x43, 0x29, 0x1e, 0xb9, 0x24, + 0x48, 0xbb, 0x8a, 0x1b, 0x67, 0x3e, 0xb3, 0xad, 0x51, 0x2f, 0x30, 0x0d, 0x45, 0x45, 0xfa, 0xf7, + 0xd6, 0xbd, 0xde, 0x3a, 0x77, 0x8a, 0xb1, 0x1c, 0x3e, 0x52, 0xc5, 0xe0, 0x27, 0xaa, 0x62, 0xcb, + 0x5d, 0x78, 0xff, 0x58, 0xd7, 0x39, 0xa6, 0x36, 0x9c, 0x9c, 0xa9, 0x6f, 0x02, 0xcc, 0x6c, 0xc9, + 0x22, 0x0b, 0xc7, 0x20, 0xcb, 0x19, 0x64, 0x83, 0xc3, 0x05, 0x07, 0x73, 0x41, 0x19, 0x96, 0x6e, + 0xa0, 0xa2, 0xf8, 0x33, 0xc8, 0x07, 0x84, 0x99, 0x34, 0xd4, 0x38, 0xc6, 0x61, 0xee, 0xbc, 0x10, + 0xd2, 0x98, 0xb0, 0x2f, 0x28, 0xf3, 0x86, 0xe9, 0x0d, 0x49, 0x82, 0x64, 0xe3, 0x19, 0x7b, 0x62, + 0x57, 0x6b, 0x98, 0x36, 0x9e, 0x72, 0x24, 0x5b, 0x08, 0x1a, 0xf7, 0x01, 0xcc, 0xdd, 0x46, 0x2a, + 0x77, 0x09, 0x4a, 0x34, 0x0c, 0xd2, 0xb6, 0xb9, 0x36, 0x4b, 0x37, 0x0f, 0xc3, 0x40, 0xa6, 0x1b, + 0x1a, 0x06, 0xbd, 0x00, 0x7d, 0x04, 0xe5, 0x08, 0xef, 0xb9, 0xaa, 0x4a, 0x4b, 0xf6, 0x05, 0x67, + 0x3e, 0xc2, 0x7b, 0xb2, 0x18, 0x37, 0xfe, 0x69, 0x41, 0xdd, 0x18, 0x21, 0x13, 0x81, 0xde, 0x84, + 0x4f, 0xa0, 0x20, 0x73, 0x89, 0x31, 0xe3, 0x94, 0x54, 0xa2, 0xa6, 0xca, 0x9b, 0xd9, 0x0e, 0x91, + 0xdd, 0x69, 0xee, 0xb4, 0x9b, 0xd9, 0x91, 0x2d, 0x4b, 0x13, 0x9a, 0x42, 0xa3, 0x2b, 0x50, 0x4d, + 0x7b, 0xb5, 0x5e, 0xf0, 0xc2, 0xe4, 0x50, 0x3d, 0x23, 0x2b, 0x68, 0xfc, 0x2f, 0x37, 0xdd, 0xfb, + 0x69, 0x06, 0xdd, 0x80, 0x05, 0xa6, 0x87, 0x74, 0x54, 0x9c, 0xa3, 0x16, 0x54, 0x0d, 0x50, 0xc5, + 0xc4, 0xe1, 0xd8, 0xcf, 0xfd, 0x88, 0xd8, 0xef, 0x40, 0x89, 0x61, 0xd5, 0x5a, 0xea, 0xfb, 0xea, + 0xa5, 0x53, 0xef, 0xab, 0x23, 0x3c, 0x49, 0xaf, 0x07, 0x1a, 0x29, 0xfb, 0x7e, 0x13, 0xa1, 0x3a, + 0xab, 0xfe, 0xfa, 0xd4, 0x9d, 0x3d, 0x53, 0x88, 0xfe, 0x08, 0xb7, 0xff, 0x5b, 0x0e, 0x2e, 0xf6, + 0xe3, 0x90, 0x88, 0xdb, 0x51, 0xd0, 0xf7, 0x3d, 0x21, 0x4c, 0x49, 0xfe, 0x23, 0x94, 0xd4, 0x55, + 0x38, 0x2d, 0x61, 0xb7, 0x4e, 0xd6, 0xf4, 0x18, 0x78, 0xaa, 0xbd, 0xd2, 0xa7, 0x2b, 0x79, 0xd2, + 0x8d, 0xd0, 0xa4, 0x99, 0xcd, 0xcc, 0xfd, 0xd0, 0xcd, 0x5c, 0x76, 0x61, 0xe9, 0xad, 0x65, 0xd0, + 0x26, 0xcc, 0x63, 0x79, 0xff, 0xc3, 0xa9, 0xe2, 0x57, 0x4f, 0xdd, 0xe2, 0x69, 0xa8, 0x18, 0xfe, + 0x94, 0xa0, 0xf1, 0xf7, 0x3c, 0xd4, 0xba, 0xfd, 0xc7, 0x4f, 0xe4, 0x7d, 0x5e, 0xef, 0xca, 0x15, + 0x59, 0x58, 0xb9, 0x20, 0x91, 0x7e, 0x6e, 0xb0, 0x32, 0xc1, 0x9d, 0x15, 0xa0, 0x5f, 0xc0, 0x82, + 0xcc, 0x71, 0x6e, 0xac, 0x76, 0x24, 0x3a, 0x94, 0x05, 0xaa, 0x2a, 0xfb, 0x69, 0x01, 0xfa, 0x02, + 0xe6, 0xa9, 0xf6, 0x35, 0x15, 0x1e, 0xd5, 0x63, 0x4b, 0x5d, 0xb7, 0xff, 0xd8, 0x38, 0x64, 0xaa, + 0xa1, 0xc1, 0xcc, 0x1e, 0x32, 0x18, 0xdd, 0xe3, 0xa6, 0x05, 0xcb, 0x3e, 0x64, 0x38, 0x74, 0x8f, + 0x1f, 0x79, 0xed, 0x98, 0x3f, 0xfe, 0xb5, 0xe3, 0x0f, 0xb0, 0xe4, 0xd3, 0x71, 0x2c, 0x43, 0x52, + 0x36, 0x9b, 0x3e, 0x0d, 0xb0, 0x6f, 0xaa, 0xef, 0x3b, 0xc2, 0x5f, 0x46, 0x4d, 0x77, 0x06, 0x4b, + 0x9b, 0xb1, 0x0c, 0x53, 0x57, 0x12, 0x1d, 0x29, 0x21, 0xa5, 0x9f, 0xa8, 0x84, 0x34, 0x9e, 0xc0, + 0x52, 0x27, 0x09, 0xa5, 0xd5, 0x99, 0x33, 0x9b, 0x3e, 0x55, 0x59, 0x3f, 0xf8, 0xa9, 0xea, 0xea, + 0x65, 0xb8, 0x70, 0xc4, 0x54, 0x54, 0x86, 0xc2, 0x03, 0x1a, 0xe1, 0xfa, 0x9c, 0xfc, 0xba, 0xfb, + 0x92, 0xc4, 0x75, 0xab, 0x73, 0xed, 0xd5, 0xf7, 0x2b, 0x73, 0xaf, 0x0e, 0x56, 0xac, 0x6f, 0x0f, + 0x56, 0xac, 0xef, 0x0e, 0x56, 0xac, 0xff, 0x1c, 0xac, 0x58, 0x7f, 0x7d, 0xbd, 0x32, 0xf7, 0xed, + 0xeb, 0x95, 0xb9, 0xef, 0x5e, 0xaf, 0xcc, 0x7d, 0x53, 0xcd, 0xbc, 0x06, 0xfe, 0x3f, 0x00, 0x00, + 0xff, 0xff, 0x27, 0x57, 0x83, 0x0b, 0xba, 0x14, 0x00, 0x00, } func (m *BackfillerSpec) Marshal() (dAtA []byte, err error) { @@ -1049,6 +1051,22 @@ func (m *ReadImportDataSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Types) > 0 { + for iNdEx := len(m.Types) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Types[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintProcessorsBulkIo(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x82 + } + } i -= len(m.UserProto) copy(dAtA[i:], m.UserProto) i = encodeVarintProcessorsBulkIo(dAtA, i, uint64(len(m.UserProto))) @@ -1936,6 +1954,12 @@ func (m *ReadImportDataSpec) Size() (n int) { } l = len(m.UserProto) n += 1 + l + sovProcessorsBulkIo(uint64(l)) + if len(m.Types) > 0 { + for _, e := range m.Types { + l = e.Size() + n += 2 + l + sovProcessorsBulkIo(uint64(l)) + } + } return n } @@ -3153,6 +3177,40 @@ func (m *ReadImportDataSpec) Unmarshal(dAtA []byte) error { } m.UserProto = github_com_cockroachdb_cockroach_pkg_security.SQLUsernameProto(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 16: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Types", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProcessorsBulkIo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthProcessorsBulkIo + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthProcessorsBulkIo + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Types = append(m.Types, &descpb.TypeDescriptor{}) + if err := m.Types[len(m.Types)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipProcessorsBulkIo(dAtA[iNdEx:]) diff --git a/pkg/sql/execinfrapb/processors_bulk_io.proto b/pkg/sql/execinfrapb/processors_bulk_io.proto index 025f05deffa7..2cb0a5c524cc 100644 --- a/pkg/sql/execinfrapb/processors_bulk_io.proto +++ b/pkg/sql/execinfrapb/processors_bulk_io.proto @@ -140,7 +140,10 @@ message ReadImportDataSpec { // User who initiated the import. This is used to check access privileges // when using FileTable ExternalStorage. optional string user_proto = 15 [(gogoproto.nullable) = false, (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/security.SQLUsernameProto"]; - // NEXTID: 16 + + repeated sqlbase.TypeDescriptor types = 16; + + // NEXTID: 17 } message StreamIngestionDataSpec { diff --git a/pkg/sql/row/row_converter.go b/pkg/sql/row/row_converter.go index e665f48759fa..3fc9fd8c670e 100644 --- a/pkg/sql/row/row_converter.go +++ b/pkg/sql/row/row_converter.go @@ -297,6 +297,7 @@ func (c *DatumRowConverter) getSequenceAnnotation( // NewDatumRowConverter returns an instance of a DatumRowConverter. func NewDatumRowConverter( ctx context.Context, + baseSemaCtx *tree.SemaContext, tableDesc catalog.TableDescriptor, targetColNames tree.NameList, evalCtx *tree.EvalContext, @@ -331,12 +332,15 @@ func NewDatumRowConverter( } var txCtx transform.ExprTransformContext - semaCtx := tree.MakeSemaContext() relevantColumns := func(col catalog.Column) bool { return col.HasDefault() || col.IsComputed() } + + // We take a copy of the baseSemaCtx since this method is called by the parallel + // import workers. + semaCtxCopy := *baseSemaCtx cols := schemaexpr.ProcessColumnSet(targetCols, tableDesc, relevantColumns) - defaultExprs, err := schemaexpr.MakeDefaultExprs(ctx, cols, &txCtx, c.EvalCtx, &semaCtx) + defaultExprs, err := schemaexpr.MakeDefaultExprs(ctx, cols, &txCtx, c.EvalCtx, &semaCtxCopy) if err != nil { return nil, errors.Wrap(err, "process default and computed columns") } @@ -445,7 +449,7 @@ func NewDatumRowConverter( c.tableDesc, tree.NewUnqualifiedTableName(tree.Name(c.tableDesc.GetName())), c.EvalCtx, - &semaCtx) + &semaCtxCopy) if err != nil { return nil, errors.Wrapf(err, "error evaluating computed expression for IMPORT INTO") } diff --git a/pkg/sql/rowexec/bulk_row_writer.go b/pkg/sql/rowexec/bulk_row_writer.go index 5f500748b758..9c9ef9435a80 100644 --- a/pkg/sql/rowexec/bulk_row_writer.go +++ b/pkg/sql/rowexec/bulk_row_writer.go @@ -101,9 +101,10 @@ func (sp *bulkRowWriter) work(ctx context.Context) error { kvCh := make(chan row.KVBatch, 10) var g ctxgroup.Group + semaCtx := tree.MakeSemaContext() conv, err := row.NewDatumRowConverter( - ctx, sp.tableDesc, nil /* targetColNames */, sp.EvalCtx, kvCh, nil, /* seqChunkProvider */ - sp.flowCtx.GetRowMetrics(), + ctx, &semaCtx, sp.tableDesc, nil /* targetColNames */, sp.EvalCtx, kvCh, nil, + /* seqChunkProvider */ sp.flowCtx.GetRowMetrics(), ) if err != nil { return err