-
Notifications
You must be signed in to change notification settings - Fork 3.8k
/
execplan.go
2591 lines (2486 loc) · 99.4 KB
/
execplan.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2019 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
package colbuilder
import (
"context"
"reflect"
"strings"
"github.com/cockroachdb/cockroach/pkg/col/coldata"
"github.com/cockroachdb/cockroach/pkg/col/coldataext"
"github.com/cockroachdb/cockroach/pkg/col/typeconv"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
"github.com/cockroachdb/cockroach/pkg/sql/colconv"
"github.com/cockroachdb/cockroach/pkg/sql/colexec"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecagg"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecargs"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecdisk"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecjoin"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecproj"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecprojconst"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils"
"github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecwindow"
"github.com/cockroachdb/cockroach/pkg/sql/colexecerror"
"github.com/cockroachdb/cockroach/pkg/sql/colexecop"
"github.com/cockroachdb/cockroach/pkg/sql/colfetcher"
"github.com/cockroachdb/cockroach/pkg/sql/colmem"
"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
"github.com/cockroachdb/cockroach/pkg/sql/execinfra/execreleasable"
"github.com/cockroachdb/cockroach/pkg/sql/execinfrapb"
"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree/treecmp"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb"
"github.com/cockroachdb/cockroach/pkg/sql/types"
"github.com/cockroachdb/cockroach/pkg/util"
"github.com/cockroachdb/cockroach/pkg/util/buildutil"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/mon"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/redact"
)
func checkNumIn(inputs []colexecargs.OpWithMetaInfo, numIn int) error {
if len(inputs) != numIn {
return errors.Errorf("expected %d input(s), got %d", numIn, len(inputs))
}
return nil
}
// wrapRowSources, given input Operators, integrates toWrap into a columnar
// execution flow and returns toWrap's output as an Operator.
// - materializerSafeToRelease indicates whether the materializers created in
// order to row-sourcify the inputs are safe to be released on the flow cleanup.
func wrapRowSources(
ctx context.Context,
flowCtx *execinfra.FlowCtx,
inputs []colexecargs.OpWithMetaInfo,
inputTypes [][]*types.T,
streamingMemAccount *mon.BoundAccount,
processorID int32,
newToWrap func([]execinfra.RowSource) (execinfra.RowSource, error),
materializerSafeToRelease bool,
factory coldata.ColumnFactory,
) (*colexec.Columnarizer, []execreleasable.Releasable, error) {
var toWrapInputs []execinfra.RowSource
var releasables []execreleasable.Releasable
for i := range inputs {
// Optimization: if the input is a Columnarizer, its input is
// necessarily a execinfra.RowSource, so remove the unnecessary
// conversion.
if c, ok := inputs[i].Root.(*colexec.Columnarizer); ok {
// Since this Columnarizer has been previously added to Closers and
// MetadataSources, this call ensures that all future calls are noops.
// Modifying the slices at this stage is difficult.
c.MarkAsRemovedFromFlow()
toWrapInputs = append(toWrapInputs, c.Input())
} else {
toWrapInput := colexec.NewMaterializer(
flowCtx,
processorID,
inputs[i],
inputTypes[i],
)
// We passed the ownership over the meta components to the
// materializer.
// TODO(yuzefovich): possibly set the length to 0 in order to be
// able to pool the underlying slices.
inputs[i].StatsCollectors = nil
inputs[i].MetadataSources = nil
inputs[i].ToClose = nil
toWrapInputs = append(toWrapInputs, toWrapInput)
if materializerSafeToRelease {
releasables = append(releasables, toWrapInput)
}
}
}
toWrap, err := newToWrap(toWrapInputs)
if err != nil {
return nil, releasables, err
}
proc, isProcessor := toWrap.(execinfra.Processor)
if !isProcessor {
return nil, nil, errors.AssertionFailedf("unexpectedly %T is not an execinfra.Processor", toWrap)
}
var c *colexec.Columnarizer
if proc.MustBeStreaming() {
c = colexec.NewStreamingColumnarizer(
colmem.NewAllocator(ctx, streamingMemAccount, factory), flowCtx, processorID, toWrap,
)
} else {
c = colexec.NewBufferingColumnarizer(
colmem.NewAllocator(ctx, streamingMemAccount, factory), flowCtx, processorID, toWrap,
)
}
return c, releasables, nil
}
type opResult struct {
*colexecargs.NewColOperatorResult
}
func needHashAggregator(aggSpec *execinfrapb.AggregatorSpec) (bool, error) {
var groupCols, orderedCols util.FastIntSet
for _, col := range aggSpec.OrderedGroupCols {
orderedCols.Add(int(col))
}
for _, col := range aggSpec.GroupCols {
if !orderedCols.Contains(int(col)) {
return true, nil
}
groupCols.Add(int(col))
}
if !orderedCols.SubsetOf(groupCols) {
return false, errors.AssertionFailedf("ordered cols must be a subset of grouping cols")
}
return false, nil
}
// IsSupported returns an error if the given spec is not supported by the
// vectorized engine (neither natively nor by wrapping the corresponding row
// execution processor).
func IsSupported(mode sessiondatapb.VectorizeExecMode, spec *execinfrapb.ProcessorSpec) error {
err := supportedNatively(spec)
if err != nil {
if wrapErr := canWrap(mode, spec); wrapErr == nil {
// We don't support this spec natively, but we can wrap the row
// execution processor.
return nil
}
}
return err
}
// supportedNatively checks whether we have a columnar operator equivalent to a
// processor described by spec. Note that it doesn't perform any other checks
// (like validity of the number of inputs).
func supportedNatively(spec *execinfrapb.ProcessorSpec) error {
switch {
case spec.Core.Noop != nil:
return nil
case spec.Core.Values != nil:
return nil
case spec.Core.TableReader != nil:
return nil
case spec.Core.JoinReader != nil:
if !spec.Core.JoinReader.IsIndexJoin() {
return errLookupJoinUnsupported
}
return nil
case spec.Core.Filterer != nil:
return nil
case spec.Core.Aggregator != nil:
for _, agg := range spec.Core.Aggregator.Aggregations {
if agg.FilterColIdx != nil {
return errors.Newf("filtering aggregation not supported")
}
}
return nil
case spec.Core.Distinct != nil:
return nil
case spec.Core.Ordinality != nil:
return nil
case spec.Core.HashJoiner != nil:
if !spec.Core.HashJoiner.OnExpr.Empty() && spec.Core.HashJoiner.Type != descpb.InnerJoin {
return errors.Newf("can't plan vectorized non-inner hash joins with ON expressions")
}
return nil
case spec.Core.MergeJoiner != nil:
if !spec.Core.MergeJoiner.OnExpr.Empty() && spec.Core.MergeJoiner.Type != descpb.InnerJoin {
return errors.Errorf("can't plan non-inner merge join with ON expressions")
}
return nil
case spec.Core.Sorter != nil:
return nil
case spec.Core.Windower != nil:
for _, wf := range spec.Core.Windower.WindowFns {
if wf.FilterColIdx != tree.NoColumnIdx {
return errors.Newf("window functions with FILTER clause are not supported")
}
if wf.Func.AggregateFunc != nil {
if !colexecagg.IsAggOptimized(*wf.Func.AggregateFunc) {
return errors.Newf("default aggregate window functions not supported")
}
}
}
return nil
case spec.Core.LocalPlanNode != nil:
// LocalPlanNode core is special (we don't have any plans on vectorizing
// it at the moment), so we want to return a custom error for it to
// distinguish from other unsupported cores.
return errLocalPlanNodeWrap
default:
return errCoreUnsupportedNatively
}
}
var (
errCoreUnsupportedNatively = errors.New("unsupported processor core")
errLocalPlanNodeWrap = errors.New("LocalPlanNode core needs to be wrapped")
errChangeAggregatorWrap = errors.New("core.ChangeAggregator is not supported")
errChangeFrontierWrap = errors.New("core.ChangeFrontier is not supported")
errReadImportWrap = errors.New("core.ReadImport is not supported")
errBackupDataWrap = errors.New("core.BackupData is not supported")
errBackfillerWrap = errors.New("core.Backfiller is not supported (not an execinfra.RowSource)")
errExporterWrap = errors.New("core.Exporter is not supported (not an execinfra.RowSource)")
errSamplerWrap = errors.New("core.Sampler is not supported (not an execinfra.RowSource)")
errSampleAggregatorWrap = errors.New("core.SampleAggregator is not supported (not an execinfra.RowSource)")
errExperimentalWrappingProhibited = errors.New("wrapping for non-JoinReader and non-LocalPlanNode cores is prohibited in vectorize=experimental_always")
errWrappedCast = errors.New("mismatched types in NewColOperator and unsupported casts")
errLookupJoinUnsupported = errors.New("lookup join reader is unsupported in vectorized")
)
func canWrap(mode sessiondatapb.VectorizeExecMode, spec *execinfrapb.ProcessorSpec) error {
if mode == sessiondatapb.VectorizeExperimentalAlways && spec.Core.JoinReader == nil && spec.Core.LocalPlanNode == nil {
return errExperimentalWrappingProhibited
}
switch {
case spec.Core.Noop != nil:
case spec.Core.TableReader != nil:
case spec.Core.JoinReader != nil:
case spec.Core.Sorter != nil:
case spec.Core.Aggregator != nil:
case spec.Core.Distinct != nil:
case spec.Core.MergeJoiner != nil:
case spec.Core.HashJoiner != nil:
case spec.Core.Values != nil:
case spec.Core.Backfiller != nil:
return errBackfillerWrap
case spec.Core.ReadImport != nil:
return errReadImportWrap
case spec.Core.Exporter != nil:
return errExporterWrap
case spec.Core.Sampler != nil:
return errSamplerWrap
case spec.Core.SampleAggregator != nil:
return errSampleAggregatorWrap
case spec.Core.ZigzagJoiner != nil:
case spec.Core.ProjectSet != nil:
case spec.Core.Windower != nil:
case spec.Core.LocalPlanNode != nil:
case spec.Core.ChangeAggregator != nil:
// Currently, there is an issue with cleaning up the changefeed flows
// (#55408), so we fallback to the row-by-row engine.
return errChangeAggregatorWrap
case spec.Core.ChangeFrontier != nil:
// Currently, there is an issue with cleaning up the changefeed flows
// (#55408), so we fallback to the row-by-row engine.
return errChangeFrontierWrap
case spec.Core.Ordinality != nil:
case spec.Core.BulkRowWriter != nil:
case spec.Core.InvertedFilterer != nil:
case spec.Core.InvertedJoiner != nil:
case spec.Core.BackupData != nil:
return errBackupDataWrap
case spec.Core.SplitAndScatter != nil:
case spec.Core.RestoreData != nil:
case spec.Core.Filterer != nil:
case spec.Core.StreamIngestionData != nil:
case spec.Core.StreamIngestionFrontier != nil:
default:
return errors.AssertionFailedf("unexpected processor core %q", spec.Core)
}
return nil
}
// createDiskBackedSort creates a new disk-backed operator that sorts the input
// according to ordering.
// - matchLen specifies the length of the prefix of ordering columns the input
// is already ordered on.
// - maxNumberPartitions (when non-zero) overrides the semi-dynamically
// computed maximum number of partitions that the external sorter will have
// at once.
// - processorID is the ProcessorID of the processor core that requested
// creation of this operator. It is used only to distinguish memory monitors.
// - post describes the post-processing spec of the processor. It will be used
// to determine whether top K sort can be planned. If you want the general sort
// operator, then pass in empty struct.
func (r opResult) createDiskBackedSort(
ctx context.Context,
flowCtx *execinfra.FlowCtx,
args *colexecargs.NewColOperatorArgs,
input colexecop.Operator,
inputTypes []*types.T,
ordering execinfrapb.Ordering,
limit int64,
matchLen uint32,
maxNumberPartitions int,
processorID int32,
opNamePrefix redact.RedactableString,
factory coldata.ColumnFactory,
) colexecop.Operator {
var (
sorterMemMonitorName redact.RedactableString
inMemorySorter colexecop.Operator
)
if len(ordering.Columns) == int(matchLen) {
// The input is already fully ordered, so there is nothing to sort.
return input
}
totalMemLimit := execinfra.GetWorkMemLimit(flowCtx)
spoolMemLimit := totalMemLimit * 4 / 5
maxOutputBatchMemSize := totalMemLimit - spoolMemLimit
if totalMemLimit == 1 {
// If total memory limit is 1, we're likely in a "force disk spill"
// scenario, so we'll set all internal limits to 1 too (if we don't,
// they will end up as 0 which is treated as "no limit").
spoolMemLimit = 1
maxOutputBatchMemSize = 1
}
if limit != 0 {
// There is a limit specified, so we know exactly how many rows the
// sorter should output. Use a top K sorter, which uses a heap to avoid
// storing more rows than necessary.
var topKSorterMemAccount *mon.BoundAccount
topKSorterMemAccount, sorterMemMonitorName = args.MonitorRegistry.CreateMemAccountForSpillStrategyWithLimit(
ctx, flowCtx, spoolMemLimit, opNamePrefix+"topk-sort", processorID,
)
inMemorySorter = colexec.NewTopKSorter(
colmem.NewAllocator(ctx, topKSorterMemAccount, factory), input,
inputTypes, ordering.Columns, int(matchLen), uint64(limit), maxOutputBatchMemSize,
)
} else if matchLen > 0 {
// The input is already partially ordered. Use a chunks sorter to avoid
// loading all the rows into memory.
opName := opNamePrefix + "sort-chunks"
deselectorUnlimitedAllocator := colmem.NewAllocator(
ctx, args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, opName, processorID,
), factory,
)
var sortChunksMemAccount *mon.BoundAccount
sortChunksMemAccount, sorterMemMonitorName = args.MonitorRegistry.CreateMemAccountForSpillStrategyWithLimit(
ctx, flowCtx, spoolMemLimit, opName, processorID,
)
inMemorySorter = colexec.NewSortChunks(
deselectorUnlimitedAllocator, colmem.NewAllocator(ctx, sortChunksMemAccount, factory),
input, inputTypes, ordering.Columns, int(matchLen), maxOutputBatchMemSize,
)
} else {
// No optimizations possible. Default to the standard sort operator.
var sorterMemAccount *mon.BoundAccount
sorterMemAccount, sorterMemMonitorName = args.MonitorRegistry.CreateMemAccountForSpillStrategyWithLimit(
ctx, flowCtx, spoolMemLimit, opNamePrefix+"sort-all", processorID,
)
inMemorySorter = colexec.NewSorter(
colmem.NewAllocator(ctx, sorterMemAccount, factory), input,
inputTypes, ordering.Columns, maxOutputBatchMemSize,
)
}
if args.TestingKnobs.DiskSpillingDisabled {
// In some testing scenarios we actually don't want to create a
// disk-backed sort.
return inMemorySorter
}
// NOTE: when spilling to disk, we're using the same general external
// sorter regardless of which sorter variant we have instantiated (i.e.
// we don't take advantage of the limits and of partial ordering). We
// could improve this.
return colexecdisk.NewOneInputDiskSpiller(
input, inMemorySorter.(colexecop.BufferingInMemoryOperator),
sorterMemMonitorName,
func(input colexecop.Operator) colexecop.Operator {
opName := opNamePrefix + "external-sorter"
// We are using unlimited memory monitors here because external
// sort itself is responsible for making sure that we stay within
// the memory limit.
sortUnlimitedAllocator := colmem.NewAllocator(
ctx, args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, opName+"-sort", processorID,
), factory)
mergeUnlimitedAllocator := colmem.NewAllocator(
ctx, args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, opName+"-merge", processorID,
), factory)
outputUnlimitedAllocator := colmem.NewAllocator(
ctx, args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, opName+"-output", processorID,
), factory)
diskAccount := args.MonitorRegistry.CreateDiskAccount(ctx, flowCtx, opName, processorID)
es := colexecdisk.NewExternalSorter(
sortUnlimitedAllocator,
mergeUnlimitedAllocator,
outputUnlimitedAllocator,
input, inputTypes, ordering, uint64(limit),
int(matchLen),
execinfra.GetWorkMemLimit(flowCtx),
maxNumberPartitions,
args.TestingKnobs.NumForcedRepartitions,
args.TestingKnobs.DelegateFDAcquisitions,
args.DiskQueueCfg,
args.FDSemaphore,
diskAccount,
)
r.ToClose = append(r.ToClose, es.(colexecop.Closer))
return es
},
args.TestingKnobs.SpillingCallbackFn,
)
}
// makeDiskBackedSorterConstructor creates a colexec.DiskBackedSorterConstructor
// that can be used by the hash-based partitioner.
// NOTE: unless DelegateFDAcquisitions testing knob is set to true, it is up to
// the caller to acquire the necessary file descriptors up front.
func (r opResult) makeDiskBackedSorterConstructor(
ctx context.Context,
flowCtx *execinfra.FlowCtx,
args *colexecargs.NewColOperatorArgs,
opNamePrefix redact.RedactableString,
factory coldata.ColumnFactory,
) colexecdisk.DiskBackedSorterConstructor {
return func(input colexecop.Operator, inputTypes []*types.T, orderingCols []execinfrapb.Ordering_Column, maxNumberPartitions int) colexecop.Operator {
if maxNumberPartitions < colexecop.ExternalSorterMinPartitions {
colexecerror.InternalError(errors.AssertionFailedf(
"external sorter is attempted to be created with %d partitions, minimum %d required",
maxNumberPartitions, colexecop.ExternalSorterMinPartitions,
))
}
sortArgs := *args
if !args.TestingKnobs.DelegateFDAcquisitions {
// Set the FDSemaphore to nil. This indicates that no FDs should be
// acquired. The hash-based partitioner will do this up front.
sortArgs.FDSemaphore = nil
}
return r.createDiskBackedSort(
ctx, flowCtx, &sortArgs, input, inputTypes,
execinfrapb.Ordering{Columns: orderingCols}, 0, /* limit */
0 /* matchLen */, maxNumberPartitions, args.Spec.ProcessorID,
opNamePrefix+"-", factory,
)
}
}
// TODO(yuzefovich): introduce some way to unit test that the meta info tracking
// works correctly. See #64256 for more details.
// takeOverMetaInfo iterates over all meta components from the input trees and
// passes the responsibility of handling those to target. The input objects are
// updated in-place accordingly.
func takeOverMetaInfo(target *colexecargs.OpWithMetaInfo, inputs []colexecargs.OpWithMetaInfo) {
for i := range inputs {
target.StatsCollectors = append(target.StatsCollectors, inputs[i].StatsCollectors...)
target.MetadataSources = append(target.MetadataSources, inputs[i].MetadataSources...)
target.ToClose = append(target.ToClose, inputs[i].ToClose...)
inputs[i].MetadataSources = nil
inputs[i].StatsCollectors = nil
inputs[i].ToClose = nil
}
}
// createAndWrapRowSource takes a processor spec, creating the row source and
// wrapping it using wrapRowSources. Note that the post process spec is included
// in the processor creation, so make sure to clear it if it will be inspected
// again. opResult is updated with the new ColumnTypes and the resulting
// Columnarizer if there is no error.
// - causeToWrap is an error that prompted us to wrap a processor core into the
// vectorized plan (for example, it could be an unsupported processor core, an
// unsupported function, etc).
func (r opResult) createAndWrapRowSource(
ctx context.Context,
flowCtx *execinfra.FlowCtx,
args *colexecargs.NewColOperatorArgs,
inputs []colexecargs.OpWithMetaInfo,
inputTypes [][]*types.T,
spec *execinfrapb.ProcessorSpec,
factory coldata.ColumnFactory,
causeToWrap error,
) error {
if args.ProcessorConstructor == nil {
return errors.New("processorConstructor is nil")
}
log.VEventf(ctx, 1, "planning a row-execution processor in the vectorized flow: %v", causeToWrap)
if err := canWrap(flowCtx.EvalCtx.SessionData().VectorizeMode, spec); err != nil {
log.VEventf(ctx, 1, "planning a wrapped processor failed: %v", err)
// Return the original error for why we don't support this spec
// natively since it is more interesting.
return causeToWrap
}
// Note that the materializers aren't safe to release in all cases since in
// some cases they could be released before being closed. Namely, this would
// occur if we have a subquery with LocalPlanNode core and a materializer is
// added in order to wrap that core - what will happen is that all
// releasables are put back into their pools upon the subquery's flow
// cleanup, yet the subquery planNode tree isn't closed yet since its
// closure is done when the main planNode tree is being closed.
// TODO(yuzefovich): currently there are some other cases as well, figure
// those out. I believe all those cases can occur **only** if we have
// LocalPlanNode cores which is the case when we have non-empty
// LocalProcessors.
materializerSafeToRelease := len(args.LocalProcessors) == 0
c, releasables, err := wrapRowSources(
ctx,
flowCtx,
inputs,
inputTypes,
args.StreamingMemAccount,
spec.ProcessorID,
func(inputs []execinfra.RowSource) (execinfra.RowSource, error) {
// We provide a slice with a single nil as 'outputs' parameter
// because all processors expect a single output. Passing nil is ok
// here because when wrapping the processor, the materializer will
// be its output, and it will be set up in wrapRowSources.
proc, err := args.ProcessorConstructor(
ctx, flowCtx, spec.ProcessorID, &spec.Core, &spec.Post, inputs,
[]execinfra.RowReceiver{nil} /* outputs */, args.LocalProcessors,
)
if err != nil {
return nil, err
}
var (
rs execinfra.RowSource
ok bool
)
if rs, ok = proc.(execinfra.RowSource); !ok {
return nil, errors.AssertionFailedf(
"processor %s is not an execinfra.RowSource", spec.Core.String(),
)
}
r.ColumnTypes = rs.OutputTypes()
return rs, nil
},
materializerSafeToRelease,
factory,
)
if err != nil {
return err
}
r.Root = c
r.Columnarizer = c
if buildutil.CrdbTestBuild {
r.Root = colexec.NewInvariantsChecker(r.Root)
}
takeOverMetaInfo(&r.OpWithMetaInfo, inputs)
r.MetadataSources = append(r.MetadataSources, r.Root.(colexecop.MetadataSource))
r.ToClose = append(r.ToClose, r.Root.(colexecop.Closer))
r.Releasables = append(r.Releasables, releasables...)
return nil
}
// MaybeRemoveRootColumnarizer examines whether r represents such a tree of
// operators that has a columnarizer as its root with no responsibility over
// other meta components. If that's the case, the input to the columnarizer is
// returned and the columnarizer is marked as removed from the flow; otherwise,
// nil is returned.
func MaybeRemoveRootColumnarizer(r colexecargs.OpWithMetaInfo) execinfra.RowSource {
root := r.Root
if buildutil.CrdbTestBuild {
// We might have an invariants checker as the root right now, we gotta
// peek inside of it if so.
root = colexec.MaybeUnwrapInvariantsChecker(root)
}
c, isColumnarizer := root.(*colexec.Columnarizer)
if !isColumnarizer {
return nil
}
// We have the columnarizer as the root, and it must be included into the
// MetadataSources and ToClose slices, so if we don't see any other objects,
// then the responsibility over other meta components has been claimed by
// the children of the columnarizer.
if len(r.StatsCollectors) != 0 || len(r.MetadataSources) != 1 || len(r.ToClose) != 1 {
return nil
}
c.MarkAsRemovedFromFlow()
return c.Input()
}
func getStreamingAllocator(
ctx context.Context, args *colexecargs.NewColOperatorArgs,
) *colmem.Allocator {
return colmem.NewAllocator(ctx, args.StreamingMemAccount, args.Factory)
}
// NOTE: throughout this file we do not append an output type of a projecting
// operator to the passed-in type schema - we, instead, always allocate a new
// type slice and copy over the old schema and set the output column of a
// projecting operator in the next slot. We attempt to enforce this by a linter
// rule, and such behavior prevents the type schema corruption scenario as
// described below.
//
// Without explicit new allocations, it is possible that planSelectionOperators
// (and other planning functions) reuse the same array for filterColumnTypes as
// result.ColumnTypes is using because there was enough capacity to do so.
// As an example, consider the following scenario in the context of
// planFilterExpr method:
// 1. r.ColumnTypes={types.Bool} with len=1 and cap=4
// 2. planSelectionOperators adds another types.Int column, so
// filterColumnTypes={types.Bool, types.Int} with len=2 and cap=4
// Crucially, it uses exact same underlying array as r.ColumnTypes
// uses.
// 3. we project out second column, so r.ColumnTypes={types.Bool}
// 4. later, we add another types.Float column, so
// r.ColumnTypes={types.Bool, types.Float}, but there is enough
// capacity in the array, so we simply overwrite the second slot
// with the new type which corrupts filterColumnTypes to become
// {types.Bool, types.Float}, and we can get into a runtime type
// mismatch situation.
// NewColOperator creates a new columnar operator according to the given spec.
func NewColOperator(
ctx context.Context, flowCtx *execinfra.FlowCtx, args *colexecargs.NewColOperatorArgs,
) (_ *colexecargs.NewColOperatorResult, err error) {
result := opResult{NewColOperatorResult: colexecargs.GetNewColOperatorResult()}
r := result.NewColOperatorResult
spec := args.Spec
inputs := args.Inputs
if args.Factory == nil {
// This code path is only used in tests.
args.Factory = coldataext.NewExtendedColumnFactory(flowCtx.EvalCtx)
}
factory := args.Factory
if args.ExprHelper == nil {
args.ExprHelper = colexecargs.NewExprHelper()
args.ExprHelper.SemaCtx = flowCtx.NewSemaContext(flowCtx.Txn)
}
if args.MonitorRegistry == nil {
args.MonitorRegistry = &colexecargs.MonitorRegistry{}
}
core := &spec.Core
post := &spec.Post
if err = supportedNatively(spec); err != nil {
inputTypes := make([][]*types.T, len(spec.Input))
for inputIdx, input := range spec.Input {
inputTypes[inputIdx] = make([]*types.T, len(input.ColumnTypes))
copy(inputTypes[inputIdx], input.ColumnTypes)
}
err = result.createAndWrapRowSource(ctx, flowCtx, args, inputs, inputTypes, spec, factory, err)
// The wrapped processors need to be passed the post-process specs,
// since they inspect them to figure out information about needed
// columns. This means that we'll let those processors do any renders
// or filters, which isn't ideal. We could improve this.
post = &execinfrapb.PostProcessSpec{}
} else {
switch {
case core.Noop != nil:
if err := checkNumIn(inputs, 1); err != nil {
return r, err
}
result.Root = colexecop.NewNoop(inputs[0].Root)
result.ColumnTypes = make([]*types.T, len(spec.Input[0].ColumnTypes))
copy(result.ColumnTypes, spec.Input[0].ColumnTypes)
case core.Values != nil:
if err := checkNumIn(inputs, 0); err != nil {
return r, err
}
if core.Values.NumRows == 0 || len(core.Values.Columns) == 0 {
// To simplify valuesOp we handle some special cases with
// fixedNumTuplesNoInputOp.
result.Root = colexecutils.NewFixedNumTuplesNoInputOp(
getStreamingAllocator(ctx, args), int(core.Values.NumRows), nil, /* opToInitialize */
)
} else {
result.Root = colexec.NewValuesOp(getStreamingAllocator(ctx, args), core.Values)
}
result.ColumnTypes = make([]*types.T, len(core.Values.Columns))
for i, col := range core.Values.Columns {
result.ColumnTypes[i] = col.Type
}
case core.TableReader != nil:
if err := checkNumIn(inputs, 0); err != nil {
return r, err
}
// We have to create a separate account in order for the cFetcher to
// be able to precisely track the size of its output batch. This
// memory account is "streaming" in its nature, so we create an
// unlimited one.
cFetcherMemAcc := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, "cfetcher" /* opName */, spec.ProcessorID,
)
kvFetcherMemAcc := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, "kvfetcher" /* opName */, spec.ProcessorID,
)
estimatedRowCount := spec.EstimatedRowCount
scanOp, err := colfetcher.NewColBatchScan(
ctx, colmem.NewAllocator(ctx, cFetcherMemAcc, factory), kvFetcherMemAcc,
flowCtx, core.TableReader, post, estimatedRowCount,
)
if err != nil {
return r, err
}
result.finishScanPlanning(scanOp, scanOp.ResultTypes)
case core.JoinReader != nil:
if err := checkNumIn(inputs, 1); err != nil {
return r, err
}
if !core.JoinReader.IsIndexJoin() {
return r, errors.AssertionFailedf("lookup join reader is unsupported in vectorized")
}
// We have to create a separate account in order for the cFetcher to
// be able to precisely track the size of its output batch. This
// memory account is "streaming" in its nature, so we create an
// unlimited one.
cFetcherMemAcc := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, "cfetcher" /* opName */, spec.ProcessorID,
)
kvFetcherMemAcc := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, "kvfetcher" /* opName */, spec.ProcessorID,
)
// We might use the Streamer API which requires a separate memory
// account that is bound to an unlimited memory monitor.
streamerBudgetAcc := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, "streamer" /* opName */, spec.ProcessorID,
)
streamerDiskMonitor := args.MonitorRegistry.CreateDiskMonitor(
ctx, flowCtx, "streamer" /* opName */, spec.ProcessorID,
)
inputTypes := make([]*types.T, len(spec.Input[0].ColumnTypes))
copy(inputTypes, spec.Input[0].ColumnTypes)
indexJoinOp, err := colfetcher.NewColIndexJoin(
ctx, getStreamingAllocator(ctx, args),
colmem.NewAllocator(ctx, cFetcherMemAcc, factory),
kvFetcherMemAcc, streamerBudgetAcc, flowCtx,
inputs[0].Root, core.JoinReader, post, inputTypes, streamerDiskMonitor,
)
if err != nil {
return r, err
}
result.finishScanPlanning(indexJoinOp, indexJoinOp.ResultTypes)
case core.Filterer != nil:
if err := checkNumIn(inputs, 1); err != nil {
return r, err
}
result.ColumnTypes = make([]*types.T, len(spec.Input[0].ColumnTypes))
copy(result.ColumnTypes, spec.Input[0].ColumnTypes)
result.Root = inputs[0].Root
if err := result.planAndMaybeWrapFilter(
ctx, flowCtx, args, spec.ProcessorID, core.Filterer.Filter, factory,
); err != nil {
return r, err
}
case core.Aggregator != nil:
if err := checkNumIn(inputs, 1); err != nil {
return r, err
}
aggSpec := core.Aggregator
if len(aggSpec.Aggregations) == 0 {
// We can get an aggregator when no aggregate functions are
// present if HAVING clause is present, for example, with a
// query as follows: SELECT 1 FROM t HAVING true. In this case,
// we plan a special operator that outputs a batch of length 1
// without actual columns once and then zero-length batches. The
// actual "data" will be added by projections below.
// TODO(solon): The distsql plan for this case includes a
// TableReader, so we end up creating an orphaned colBatchScan.
// We should avoid that. Ideally the optimizer would not plan a
// scan in this unusual case.
result.Root, err = colexecutils.NewFixedNumTuplesNoInputOp(
getStreamingAllocator(ctx, args), 1 /* numTuples */, inputs[0].Root,
), nil
// We make ColumnTypes non-nil so that sanity check doesn't
// panic.
result.ColumnTypes = []*types.T{}
break
}
if aggSpec.IsRowCount() {
result.Root, err = colexec.NewCountOp(getStreamingAllocator(ctx, args), inputs[0].Root), nil
result.ColumnTypes = []*types.T{types.Int}
break
}
var needHash bool
needHash, err = needHashAggregator(aggSpec)
if err != nil {
return r, err
}
inputTypes := make([]*types.T, len(spec.Input[0].ColumnTypes))
copy(inputTypes, spec.Input[0].ColumnTypes)
// Make a copy of the evalCtx since we're modifying it below.
evalCtx := flowCtx.NewEvalCtx()
newAggArgs := &colexecagg.NewAggregatorArgs{
Input: inputs[0].Root,
InputTypes: inputTypes,
Spec: aggSpec,
EvalCtx: evalCtx,
}
newAggArgs.Constructors, newAggArgs.ConstArguments, newAggArgs.OutputTypes, err = colexecagg.ProcessAggregations(
evalCtx, args.ExprHelper.SemaCtx, aggSpec.Aggregations, inputTypes,
)
if err != nil {
return r, err
}
result.ColumnTypes = newAggArgs.OutputTypes
if needHash {
opName := redact.RedactableString("hash-aggregator")
outputUnlimitedAllocator := colmem.NewAllocator(
ctx,
args.MonitorRegistry.CreateUnlimitedMemAccount(ctx, flowCtx, opName+"-output", spec.ProcessorID),
factory,
)
// We have separate unit tests that instantiate the in-memory
// hash aggregators, so we don't need to look at
// args.TestingKnobs.DiskSpillingDisabled and always instantiate
// a disk-backed one here.
diskSpillingDisabled := !colexec.HashAggregationDiskSpillingEnabled.Get(&flowCtx.Cfg.Settings.SV)
if diskSpillingDisabled {
// The disk spilling is disabled by the cluster setting, so
// we give an unlimited memory account to the in-memory
// hash aggregator and don't set up the disk spiller.
hashAggregatorUnlimitedMemAccount := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, opName, spec.ProcessorID,
)
newAggArgs.Allocator = colmem.NewAllocator(
ctx, hashAggregatorUnlimitedMemAccount, factory,
)
newAggArgs.MemAccount = hashAggregatorUnlimitedMemAccount
evalCtx.SingleDatumAggMemAccount = hashAggregatorUnlimitedMemAccount
maxOutputBatchMemSize := execinfra.GetWorkMemLimit(flowCtx)
// The second argument is nil because we disable the
// tracking of the input tuples.
result.Root = colexec.NewHashAggregator(
newAggArgs, nil, /* newSpillingQueueArgs */
outputUnlimitedAllocator, maxOutputBatchMemSize,
)
} else {
// We will divide the available memory equally between the
// two usages - the hash aggregation itself and the input
// tuples tracking.
totalMemLimit := execinfra.GetWorkMemLimit(flowCtx)
// We will give 20% of the hash aggregation budget to the
// output batch.
maxOutputBatchMemSize := totalMemLimit / 10
hashAggregationMemLimit := totalMemLimit/2 - maxOutputBatchMemSize
inputTuplesTrackingMemLimit := totalMemLimit / 2
if totalMemLimit == 1 {
// If total memory limit is 1, we're likely in a "force
// disk spill" scenario, so we'll set all internal
// limits to 1 too (if we don't, they will end up as 0
// which is treated as "no limit").
maxOutputBatchMemSize = 1
hashAggregationMemLimit = 1
inputTuplesTrackingMemLimit = 1
}
hashAggregatorMemAccount, hashAggregatorMemMonitorName := args.MonitorRegistry.CreateMemAccountForSpillStrategyWithLimit(
ctx, flowCtx, hashAggregationMemLimit, opName, spec.ProcessorID,
)
spillingQueueMemMonitorName := hashAggregatorMemMonitorName + "-spilling-queue"
// We need to create a separate memory account for the
// spilling queue because it looks at how much memory it has
// already used in order to decide when to spill to disk.
spillingQueueMemAccount := args.MonitorRegistry.CreateUnlimitedMemAccount(
ctx, flowCtx, spillingQueueMemMonitorName, spec.ProcessorID,
)
newAggArgs.Allocator = colmem.NewAllocator(ctx, hashAggregatorMemAccount, factory)
newAggArgs.MemAccount = hashAggregatorMemAccount
inMemoryHashAggregator := colexec.NewHashAggregator(
newAggArgs,
&colexecutils.NewSpillingQueueArgs{
UnlimitedAllocator: colmem.NewAllocator(ctx, spillingQueueMemAccount, factory),
Types: inputTypes,
MemoryLimit: inputTuplesTrackingMemLimit,
DiskQueueCfg: args.DiskQueueCfg,
FDSemaphore: args.FDSemaphore,
DiskAcc: args.MonitorRegistry.CreateDiskAccount(ctx, flowCtx, spillingQueueMemMonitorName, spec.ProcessorID),
},
outputUnlimitedAllocator,
maxOutputBatchMemSize,
)
ehaOpName := redact.RedactableString("external-hash-aggregator")
ehaMemAccount := args.MonitorRegistry.CreateUnlimitedMemAccount(ctx, flowCtx, ehaOpName, spec.ProcessorID)
// Note that we will use an unlimited memory account here
// even for the in-memory hash aggregator since it is easier
// to do so than to try to replace the memory account if the
// spilling to disk occurs (if we don't replace it in such
// case, the wrapped aggregate functions might hit a memory
// error even when used by the external hash aggregator).
evalCtx.SingleDatumAggMemAccount = ehaMemAccount
result.Root = colexecdisk.NewOneInputDiskSpiller(
inputs[0].Root, inMemoryHashAggregator.(colexecop.BufferingInMemoryOperator),
hashAggregatorMemMonitorName,
func(input colexecop.Operator) colexecop.Operator {
newAggArgs := *newAggArgs
// Note that the hash-based partitioner will make
// sure that partitions to process using the
// in-memory hash aggregator fit under the limit, so
// we use an unlimited allocator.
newAggArgs.Allocator = colmem.NewAllocator(ctx, ehaMemAccount, factory)
newAggArgs.MemAccount = ehaMemAccount
newAggArgs.Input = input
eha, toClose := colexecdisk.NewExternalHashAggregator(
flowCtx,
args,
&newAggArgs,
result.makeDiskBackedSorterConstructor(ctx, flowCtx, args, ehaOpName, factory),
args.MonitorRegistry.CreateDiskAccount(ctx, flowCtx, ehaOpName, spec.ProcessorID),
// Note that here we can use the same allocator
// object as we passed to the in-memory hash
// aggregator because only one (either in-memory
// or external) operator will reach the output
// state.
outputUnlimitedAllocator,
maxOutputBatchMemSize,
)
result.ToClose = append(result.ToClose, toClose)
return eha
},
args.TestingKnobs.SpillingCallbackFn,
)
}
} else {
evalCtx.SingleDatumAggMemAccount = args.StreamingMemAccount
newAggArgs.Allocator = getStreamingAllocator(ctx, args)
newAggArgs.MemAccount = args.StreamingMemAccount
result.Root = colexec.NewOrderedAggregator(newAggArgs)
}
result.ToClose = append(result.ToClose, result.Root.(colexecop.Closer))
case core.Distinct != nil:
if err := checkNumIn(inputs, 1); err != nil {
return r, err
}
result.ColumnTypes = make([]*types.T, len(spec.Input[0].ColumnTypes))
copy(result.ColumnTypes, spec.Input[0].ColumnTypes)
if len(core.Distinct.OrderedColumns) == len(core.Distinct.DistinctColumns) {
result.Root = colexecbase.NewOrderedDistinct(
inputs[0].Root, core.Distinct.OrderedColumns, result.ColumnTypes,
core.Distinct.NullsAreDistinct, core.Distinct.ErrorOnDup,
)
} else {
// We have separate unit tests that instantiate in-memory
// distinct operators, so we don't need to look at
// args.TestingKnobs.DiskSpillingDisabled and always instantiate
// a disk-backed one here.
distinctMemAccount, distinctMemMonitorName := args.MonitorRegistry.CreateMemAccountForSpillStrategy(
ctx, flowCtx, "distinct" /* opName */, spec.ProcessorID,
)
// TODO(yuzefovich): we have an implementation of partially
// ordered distinct, and we should plan it when we have
// non-empty ordered columns and we think that the probability
// of distinct tuples in the input is about 0.01 or less.
allocator := colmem.NewAllocator(ctx, distinctMemAccount, factory)
inMemoryUnorderedDistinct := colexec.NewUnorderedDistinct(
allocator, inputs[0].Root, core.Distinct.DistinctColumns, result.ColumnTypes,
core.Distinct.NullsAreDistinct, core.Distinct.ErrorOnDup,
)
edOpName := redact.RedactableString("external-distinct")
diskAccount := args.MonitorRegistry.CreateDiskAccount(ctx, flowCtx, edOpName, spec.ProcessorID)
result.Root = colexecdisk.NewOneInputDiskSpiller(
inputs[0].Root, inMemoryUnorderedDistinct.(colexecop.BufferingInMemoryOperator),
distinctMemMonitorName,
func(input colexecop.Operator) colexecop.Operator {
unlimitedAllocator := colmem.NewAllocator(
ctx, args.MonitorRegistry.CreateUnlimitedMemAccount(ctx, flowCtx, edOpName, spec.ProcessorID), factory,
)
ed, toClose := colexecdisk.NewExternalDistinct(
unlimitedAllocator,