Skip to content

Commit

Permalink
colexec: clean up external distinct test a bit
Browse files Browse the repository at this point in the history
Release note: None
  • Loading branch information
yuzefovich committed Apr 9, 2021
1 parent b8fa9b1 commit fa8f5db
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 22 deletions.
32 changes: 13 additions & 19 deletions pkg/sql/colexec/external_distinct_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,38 +67,35 @@ func TestExternalDistinct(t *testing.T) {
for tcIdx, tc := range distinctTestCases {
log.Infof(context.Background(), "spillForced=%t/%d", spillForced, tcIdx)
var semsToCheck []semaphore.Semaphore
var outputOrdering execinfrapb.Ordering
verifier := colexectestutils.UnorderedVerifier
// Check that the external distinct and the disk-backed sort
// were added as Closers.
numExpectedClosers := 2
if tc.isOrderedOnDistinctCols {
outputOrdering = convertDistinctColsToOrdering(tc.distinctCols)
verifier = colexectestutils.OrderedVerifier
// The final disk-backed sort must also be added as a
// Closer.
numExpectedClosers++
}
colexectestutils.RunTestsWithTyps(
t,
testAllocator,
[]colexectestutils.Tuples{tc.tuples},
[][]*types.T{tc.typs},
tc.expected,
// We're using an unordered verifier because the in-memory
// unordered distinct is free to change the order of the tuples
// when exporting them into an external distinct.
colexectestutils.UnorderedVerifier,
verifier,
func(input []colexecop.Operator) (colexecop.Operator, error) {
// A sorter should never exceed ExternalSorterMinPartitions, even
// during repartitioning. A panic will happen if a sorter requests
// more than this number of file descriptors.
sem := colexecop.NewTestingSemaphore(colexecop.ExternalSorterMinPartitions)
semsToCheck = append(semsToCheck, sem)
var outputOrdering execinfrapb.Ordering
if tc.isOrderedOnDistinctCols {
outputOrdering = convertDistinctColsToOrdering(tc.distinctCols)
}
distinct, newAccounts, newMonitors, closers, err := createExternalDistinct(
ctx, flowCtx, input, tc.typs, tc.distinctCols, outputOrdering,
queueCfg, sem, nil /* spillingCallbackFn */, numForcedRepartitions,
)
// Check that the external distinct and the disk-backed sort
// were added as Closers.
numExpectedClosers := 2
if len(outputOrdering.Columns) > 0 {
// The final disk-backed sort must also be added as a
// Closer.
numExpectedClosers++
}
require.Equal(t, numExpectedClosers, len(closers))
accounts = append(accounts, newAccounts...)
monitors = append(monitors, newMonitors...)
Expand Down Expand Up @@ -385,9 +382,6 @@ func createExternalDistinct(
}
args.TestingKnobs.SpillingCallbackFn = spillingCallbackFn
args.TestingKnobs.NumForcedRepartitions = numForcedRepartitions
// External sorter relies on different memory accounts to
// understand when to start a new partition, so we will not use
// the streaming memory account.
result, err := colexecargs.TestNewColOperator(ctx, flowCtx, args)
return result.Op, result.OpAccounts, result.OpMonitors, result.ToClose, err
}
3 changes: 0 additions & 3 deletions pkg/sql/colexec/external_sort_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -487,9 +487,6 @@ func createDiskBackedSorter(
DiskQueueCfg: diskQueueCfg,
FDSemaphore: testingSemaphore,
}
// External sorter relies on different memory accounts to
// understand when to start a new partition, so we will not use
// the streaming memory account.
args.TestingKnobs.SpillingCallbackFn = spillingCallbackFn
args.TestingKnobs.NumForcedRepartitions = numForcedRepartitions
args.TestingKnobs.DelegateFDAcquisitions = delegateFDAcquisitions
Expand Down

0 comments on commit fa8f5db

Please sign in to comment.