From 801423bdbc27495159fbb407273e850a7a902c21 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Fri, 15 Feb 2019 18:18:42 -0800 Subject: [PATCH] Remove support for multiple table layouts --- .../benchmark/AbstractOperatorBenchmark.java | 4 +- .../hive/TestHiveIntegrationSmokeTest.java | 6 +- .../java/io/prestosql/metadata/Metadata.java | 2 +- .../prestosql/metadata/MetadataManager.java | 16 +- .../prestosql/metadata/TableLayoutResult.java | 18 - .../iterative/rule/ExtractSpatialJoins.java | 6 +- .../iterative/rule/PickTableLayout.java | 80 +- .../planner/optimizations/AddExchanges.java | 98 +-- .../optimizations/BeginTableWrite.java | 12 +- .../optimizations/MetadataQueryOptimizer.java | 7 +- .../metadata/AbstractMockMetadata.java | 2 +- .../optimizations/TestAddExchanges.java | 795 ------------------ 12 files changed, 59 insertions(+), 987 deletions(-) delete mode 100644 presto-main/src/test/java/io/prestosql/sql/planner/optimizations/TestAddExchanges.java diff --git a/presto-benchmark/src/main/java/io/prestosql/benchmark/AbstractOperatorBenchmark.java b/presto-benchmark/src/main/java/io/prestosql/benchmark/AbstractOperatorBenchmark.java index 7c17c605aea1e..7c720e1158fb1 100644 --- a/presto-benchmark/src/main/java/io/prestosql/benchmark/AbstractOperatorBenchmark.java +++ b/presto-benchmark/src/main/java/io/prestosql/benchmark/AbstractOperatorBenchmark.java @@ -170,8 +170,8 @@ protected final OperatorFactory createTableScanOperator(int operatorId, PlanNode List columnHandles = columnHandlesBuilder.build(); // get the split for this table - List layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); - Split split = getLocalQuerySplit(session, layouts.get(0).getLayout().getHandle()); + Optional layout = metadata.getLayout(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); + Split split = getLocalQuerySplit(session, layout.get().getLayout().getHandle()); return new OperatorFactory() { diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java index 6f5e69a4cd772..030a713eec4d1 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java @@ -25,7 +25,6 @@ import io.prestosql.metadata.QualifiedObjectName; import io.prestosql.metadata.TableHandle; import io.prestosql.metadata.TableLayout; -import io.prestosql.metadata.TableLayoutResult; import io.prestosql.metadata.TableMetadata; import io.prestosql.plugin.hive.HiveSessionProperties.InsertExistingPartitionsBehavior; import io.prestosql.spi.connector.CatalogSchemaTableName; @@ -1673,8 +1672,9 @@ private Object getHiveTableProperty(String tableName, Function tableHandle = metadata.getTableHandle(transactionSession, new QualifiedObjectName(catalog, TPCH_SCHEMA, tableName)); assertTrue(tableHandle.isPresent()); - List layouts = metadata.getLayouts(transactionSession, tableHandle.get(), Constraint.alwaysTrue(), Optional.empty()); - TableLayout layout = getOnlyElement(layouts).getLayout(); + TableLayout layout = metadata.getLayout(transactionSession, tableHandle.get(), Constraint.alwaysTrue(), Optional.empty()) + .get() + .getLayout(); return propertyGetter.apply((HiveTableLayoutHandle) layout.getHandle().getConnectorHandle()); }); } diff --git a/presto-main/src/main/java/io/prestosql/metadata/Metadata.java b/presto-main/src/main/java/io/prestosql/metadata/Metadata.java index 93aad49cd027e..264fb3e4fc0f3 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/Metadata.java +++ b/presto-main/src/main/java/io/prestosql/metadata/Metadata.java @@ -73,7 +73,7 @@ public interface Metadata Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName tableName, Map analyzeProperties); - List getLayouts(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns); + Optional getLayout(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns); TableLayout getLayout(Session session, TableLayoutHandle handle); diff --git a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java index 69a2cee9b4732..49a76f3e8ffbd 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java +++ b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java @@ -88,7 +88,6 @@ import java.util.concurrent.ConcurrentMap; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.prestosql.metadata.QualifiedObjectName.convertFromSchemaTableName; import static io.prestosql.metadata.TableLayout.fromConnectorLayout; @@ -373,10 +372,10 @@ public Optional getSystemTable(Session session, QualifiedObjectName } @Override - public List getLayouts(Session session, TableHandle table, Constraint constraint, Optional> desiredColumns) + public Optional getLayout(Session session, TableHandle table, Constraint constraint, Optional> desiredColumns) { if (constraint.getSummary().isNone()) { - return ImmutableList.of(); + return Optional.empty(); } ConnectorId connectorId = table.getConnectorId(); @@ -387,10 +386,15 @@ public List getLayouts(Session session, TableHandle table, Co ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); List layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns); + if (layouts.isEmpty()) { + return Optional.empty(); + } + + if (layouts.size() > 1) { + throw new PrestoException(NOT_SUPPORTED, format("Connector returned multiple layouts for table %s", table)); + } - return layouts.stream() - .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) - .collect(toImmutableList()); + return Optional.of(new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layouts.get(0).getTableLayout()), layouts.get(0).getUnenforcedConstraint())); } @Override diff --git a/presto-main/src/main/java/io/prestosql/metadata/TableLayoutResult.java b/presto-main/src/main/java/io/prestosql/metadata/TableLayoutResult.java index 77c3a2905a706..c545333fa995b 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/TableLayoutResult.java +++ b/presto-main/src/main/java/io/prestosql/metadata/TableLayoutResult.java @@ -14,18 +14,13 @@ package io.prestosql.metadata; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.predicate.TupleDomain; -import io.prestosql.sql.planner.plan.TableScanNode; -import java.util.List; import java.util.Map; -import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; public class TableLayoutResult @@ -49,19 +44,6 @@ public TupleDomain getUnenforcedConstraint() return unenforcedConstraint; } - public boolean hasAllOutputs(TableScanNode node) - { - if (!layout.getColumns().isPresent()) { - return true; - } - Set columns = ImmutableSet.copyOf(layout.getColumns().get()); - List nodeColumnHandles = node.getOutputSymbols().stream() - .map(node.getAssignments()::get) - .collect(toImmutableList()); - - return columns.containsAll(nodeColumnHandles); - } - public static TupleDomain computeEnforced(TupleDomain predicate, TupleDomain unenforced) { if (predicate.isNone()) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/ExtractSpatialJoins.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/ExtractSpatialJoins.java index d02859f5e810c..9a5accbfaf8ca 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/ExtractSpatialJoins.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/ExtractSpatialJoins.java @@ -469,11 +469,11 @@ private static KdbTree loadKdbTree(String tableName, Session session, Metadata m ColumnHandle kdbTreeColumn = Iterables.getOnlyElement(visibleColumnHandles); - List layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.of(ImmutableSet.of(kdbTreeColumn))); - checkSpatialPartitioningTable(!layouts.isEmpty(), "Table is empty: %s", name); + Optional layout = metadata.getLayout(session, tableHandle, Constraint.alwaysTrue(), Optional.of(ImmutableSet.of(kdbTreeColumn))); + checkSpatialPartitioningTable(layout.isPresent(), "Table is empty: %s", name); Optional kdbTree = Optional.empty(); - try (SplitSource splitSource = splitManager.getSplits(session, layouts.get(0).getLayout().getHandle(), UNGROUPED_SCHEDULING)) { + try (SplitSource splitSource = splitManager.getSplits(session, layout.get().getLayout().getHandle(), UNGROUPED_SCHEDULING)) { while (!Thread.currentThread().isInterrupted()) { SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)); List splits = splitBatch.getSplits(); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PickTableLayout.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PickTableLayout.java index 2ea5da69cb74d..9fc236b99158b 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PickTableLayout.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PickTableLayout.java @@ -47,14 +47,11 @@ import io.prestosql.sql.tree.NodeRef; import io.prestosql.sql.tree.NullLiteral; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.intersection; import static io.prestosql.SystemSessionProperties.isNewOptimizerEnabled; @@ -71,7 +68,6 @@ import static io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; /** * These rules should not be run after AddExchanges so as not to overwrite the TableLayout @@ -237,11 +233,10 @@ private static PlanNode planTableScan( idAllocator, metadata, parser, - domainTranslator) - .get(0); + domainTranslator); } - public static List pushFilterIntoTableScan( + public static PlanNode pushFilterIntoTableScan( TableScanNode node, Expression predicate, boolean pruneWithPredicateExpression, @@ -288,8 +283,7 @@ public static List pushFilterIntoTableScan( constraint = new Constraint<>(newDomain); } - // Layouts will be returned in order of the connector's preference - List layouts = metadata.getLayouts( + Optional layout = metadata.getLayout( session, node.getTable(), constraint, @@ -297,51 +291,37 @@ public static List pushFilterIntoTableScan( .map(node.getAssignments()::get) .collect(toImmutableSet()))); - if (layouts.isEmpty()) { - return ImmutableList.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of())); + if (!layout.isPresent() || layout.get().getLayout().getPredicate().isNone()) { + return new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()); } - // Filter out layouts that cannot supply all the required columns - layouts = layouts.stream() - .filter(layout -> layout.hasAllOutputs(node)) - .collect(toList()); - checkState(!layouts.isEmpty(), "No usable layouts for %s", node); - - if (layouts.stream().anyMatch(layout -> layout.getLayout().getPredicate().isNone())) { - return ImmutableList.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of())); + TableScanNode tableScan = new TableScanNode( + node.getId(), + node.getTable(), + node.getOutputSymbols(), + node.getAssignments(), + Optional.of(layout.get().getLayout().getHandle()), + layout.get().getLayout().getPredicate(), + computeEnforced(newDomain, layout.get().getUnenforcedConstraint())); + + // The order of the arguments to combineConjuncts matters: + // * Unenforced constraints go first because they can only be simple column references, + // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc. + // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should + // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet. + // * Short of implementing the previous bullet point, the current order of non-deterministic expressions + // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead + // to failures of previously successful queries. + Expression resultingPredicate = combineConjuncts( + domainTranslator.toPredicate(layout.get().getUnenforcedConstraint().transform(assignments::get)), + filterNonDeterministicConjuncts(predicate), + decomposedPredicate.getRemainingExpression()); + + if (!TRUE_LITERAL.equals(resultingPredicate)) { + return new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate); } - return layouts.stream() - .map(layout -> { - TableScanNode tableScan = new TableScanNode( - node.getId(), - node.getTable(), - node.getOutputSymbols(), - node.getAssignments(), - Optional.of(layout.getLayout().getHandle()), - layout.getLayout().getPredicate(), - computeEnforced(newDomain, layout.getUnenforcedConstraint())); - - // The order of the arguments to combineConjuncts matters: - // * Unenforced constraints go first because they can only be simple column references, - // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc. - // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should - // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet. - // * Short of implementing the previous bullet point, the current order of non-deterministic expressions - // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead - // to failures of previously successful queries. - Expression resultingPredicate = combineConjuncts( - domainTranslator.toPredicate(layout.getUnenforcedConstraint().transform(assignments::get)), - filterNonDeterministicConjuncts(predicate), - decomposedPredicate.getRemainingExpression()); - - if (!TRUE_LITERAL.equals(resultingPredicate)) { - return new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate); - } - - return tableScan; - }) - .collect(toImmutableList()); + return tableScan; } private static class LayoutConstraintEvaluator diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddExchanges.java index a728aeb92bdd0..873e6eadb058b 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/AddExchanges.java @@ -14,10 +14,6 @@ package io.prestosql.sql.planner.optimizations; import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; @@ -78,10 +74,7 @@ import io.prestosql.sql.tree.SymbolReference; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -89,7 +82,6 @@ import java.util.function.Function; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; @@ -540,26 +532,8 @@ else if (redistributeWrites) { private PlanWithProperties planTableScan(TableScanNode node, Expression predicate, PreferredProperties preferredProperties) { - List possiblePlans = PickTableLayout.pushFilterIntoTableScan(node, predicate, true, session, types, idAllocator, metadata, parser, domainTranslator); - List possiblePlansWithProperties = possiblePlans.stream() - .map(planNode -> new PlanWithProperties(planNode, derivePropertiesRecursively(planNode))) - .collect(toImmutableList()); - return pickPlan(possiblePlansWithProperties, preferredProperties); - } - - /** - * possiblePlans should be provided in layout preference order - */ - private PlanWithProperties pickPlan(List possiblePlans, PreferredProperties preferredProperties) - { - checkArgument(!possiblePlans.isEmpty()); - - if (preferStreamingOperators) { - possiblePlans = new ArrayList<>(possiblePlans); - Collections.sort(possiblePlans, Comparator.comparing(PlanWithProperties::getProperties, streamingExecutionPreference(preferredProperties))); // stable sort; is Collections.min() guaranteed to be stable? - } - - return possiblePlans.get(0); + PlanNode plan = PickTableLayout.pushFilterIntoTableScan(node, predicate, true, session, types, idAllocator, metadata, parser, domainTranslator); + return new PlanWithProperties(plan, derivePropertiesRecursively(plan)); } @Override @@ -1239,74 +1213,6 @@ private static Map computeIdentityTranslations(Assignments assig return outputToInput; } - @VisibleForTesting - static Comparator streamingExecutionPreference(PreferredProperties preferred) - { - // Calculating the matches can be a bit expensive, so cache the results between comparisons - LoadingCache>, List>>> matchCache = CacheBuilder.newBuilder() - .build(CacheLoader.from(actualProperties -> LocalProperties.match(actualProperties, preferred.getLocalProperties()))); - - return (actual1, actual2) -> { - List>> matchLayout1 = matchCache.getUnchecked(actual1.getLocalProperties()); - List>> matchLayout2 = matchCache.getUnchecked(actual2.getLocalProperties()); - - return ComparisonChain.start() - .compareTrueFirst(hasLocalOptimization(preferred.getLocalProperties(), matchLayout1), hasLocalOptimization(preferred.getLocalProperties(), matchLayout2)) - .compareTrueFirst(meetsPartitioningRequirements(preferred, actual1), meetsPartitioningRequirements(preferred, actual2)) - .compare(matchLayout1, matchLayout2, matchedLayoutPreference()) - .result(); - }; - } - - private static boolean hasLocalOptimization(List> desiredLayout, List>> matchResult) - { - checkArgument(desiredLayout.size() == matchResult.size()); - if (matchResult.isEmpty()) { - return false; - } - // Optimizations can be applied if the first LocalProperty has been modified in the match in any way - return !matchResult.get(0).equals(Optional.of(desiredLayout.get(0))); - } - - private static boolean meetsPartitioningRequirements(PreferredProperties preferred, ActualProperties actual) - { - if (!preferred.getGlobalProperties().isPresent()) { - return true; - } - PreferredProperties.Global preferredGlobal = preferred.getGlobalProperties().get(); - if (!preferredGlobal.isDistributed()) { - return actual.isSingleNode(); - } - if (!preferredGlobal.getPartitioningProperties().isPresent()) { - return !actual.isSingleNode(); - } - return actual.isStreamPartitionedOn(preferredGlobal.getPartitioningProperties().get().getPartitioningColumns()); - } - - // Prefer the match result that satisfied the most requirements - private static Comparator>>> matchedLayoutPreference() - { - return (matchLayout1, matchLayout2) -> { - Iterator>> match1Iterator = matchLayout1.iterator(); - Iterator>> match2Iterator = matchLayout2.iterator(); - while (match1Iterator.hasNext() && match2Iterator.hasNext()) { - Optional> match1 = match1Iterator.next(); - Optional> match2 = match2Iterator.next(); - if (match1.isPresent() && match2.isPresent()) { - return Integer.compare(match1.get().getColumns().size(), match2.get().getColumns().size()); - } - else if (match1.isPresent()) { - return 1; - } - else if (match2.isPresent()) { - return -1; - } - } - checkState(!match1Iterator.hasNext() && !match2Iterator.hasNext()); // Should be the same size - return 0; - }; - } - @VisibleForTesting static class PlanWithProperties { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java index 7057b2344e27e..4b35396aa74b1 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java @@ -41,12 +41,10 @@ import io.prestosql.sql.planner.plan.TableWriterNode; import io.prestosql.sql.planner.plan.UnionNode; -import java.util.List; import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; import static io.prestosql.metadata.TableLayoutResult.computeEnforced; import static io.prestosql.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; import static io.prestosql.sql.planner.plan.ChildReplacer.replaceChildren; @@ -197,22 +195,20 @@ private PlanNode rewriteDeleteTableScan(PlanNode node, TableHandle handle) TableScanNode scan = (TableScanNode) node; TupleDomain originalEnforcedConstraint = scan.getEnforcedConstraint(); - List layouts = metadata.getLayouts( + Optional layout = metadata.getLayout( session, handle, new Constraint<>(originalEnforcedConstraint), Optional.of(ImmutableSet.copyOf(scan.getAssignments().values()))); - verify(layouts.size() == 1, "Expected exactly one layout for delete"); - TableLayoutResult layoutResult = Iterables.getOnlyElement(layouts); return new TableScanNode( scan.getId(), handle, scan.getOutputSymbols(), scan.getAssignments(), - Optional.of(layoutResult.getLayout().getHandle()), - layoutResult.getLayout().getPredicate(), - computeEnforced(originalEnforcedConstraint, layoutResult.getUnenforcedConstraint())); + Optional.of(layout.get().getLayout().getHandle()), + layout.get().getLayout().getPredicate(), + computeEnforced(originalEnforcedConstraint, layout.get().getUnenforcedConstraint())); } if (node instanceof FilterNode) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/MetadataQueryOptimizer.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/MetadataQueryOptimizer.java index 3e70a8d9b9058..03d541629a75a 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/MetadataQueryOptimizer.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/MetadataQueryOptimizer.java @@ -139,10 +139,9 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext cont // with a Values node TableLayout layout = null; if (!tableScan.getLayout().isPresent()) { - List layouts = metadata.getLayouts(session, tableScan.getTable(), Constraint.alwaysTrue(), Optional.empty()); - if (layouts.size() == 1) { - layout = Iterables.getOnlyElement(layouts).getLayout(); - } + layout = metadata.getLayout(session, tableScan.getTable(), Constraint.alwaysTrue(), Optional.empty()) + .map(TableLayoutResult::getLayout) + .orElse(null); } else { layout = metadata.getLayout(session, tableScan.getLayout().get()); diff --git a/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java index 3af3a4c0c1689..f58e5a05db1aa 100644 --- a/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java @@ -114,7 +114,7 @@ public Optional getSystemTable(Session session, QualifiedObjectName } @Override - public List getLayouts(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns) + public Optional getLayout(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/optimizations/TestAddExchanges.java b/presto-main/src/test/java/io/prestosql/sql/planner/optimizations/TestAddExchanges.java deleted file mode 100644 index 78c343a81e038..0000000000000 --- a/presto-main/src/test/java/io/prestosql/sql/planner/optimizations/TestAddExchanges.java +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.sql.planner.optimizations; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import io.prestosql.spi.block.SortOrder; -import io.prestosql.spi.connector.ConstantProperty; -import io.prestosql.spi.connector.GroupingProperty; -import io.prestosql.spi.connector.SortingProperty; -import io.prestosql.sql.planner.Symbol; -import io.prestosql.sql.planner.optimizations.ActualProperties.Global; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.prestosql.spi.block.SortOrder.ASC_NULLS_FIRST; -import static io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; -import static io.prestosql.sql.planner.optimizations.ActualProperties.Global.arbitraryPartition; -import static io.prestosql.sql.planner.optimizations.ActualProperties.Global.partitionedOn; -import static io.prestosql.sql.planner.optimizations.ActualProperties.Global.singleStreamPartition; -import static io.prestosql.sql.planner.optimizations.ActualProperties.builder; -import static io.prestosql.sql.planner.optimizations.AddExchanges.streamingExecutionPreference; -import static org.testng.Assert.assertEquals; - -/** - * These are unit test for the internal logic in AddExchanges. - * For plan tests see {@link TestAddExchangesPlans} - */ -public class TestAddExchanges -{ - @Test - public void testPickLayoutAnyPreference() - { - Comparator preference = streamingExecutionPreference(PreferredProperties.any()); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a", "b")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .build(); - // Given no preferences, the original input order should be maintained - assertEquals(stableSort(input, preference), input); - } - - @Test - public void testPickLayoutPartitionedPreference() - { - Comparator preference = streamingExecutionPreference(PreferredProperties.distributed()); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutUnpartitionedPreference() - { - Comparator preference = streamingExecutionPreference(PreferredProperties.undistributed()); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutPartitionedOnSingle() - { - Comparator preference = streamingExecutionPreference( - PreferredProperties.partitioned(ImmutableSet.of(symbol("a")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutPartitionedOnMultiple() - { - Comparator preference = streamingExecutionPreference( - PreferredProperties.partitioned(ImmutableSet.of(symbol("a"), symbol("b")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutGrouped() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.local(ImmutableList.of(grouped("a")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutGroupedMultiple() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.local(ImmutableList.of(grouped("a", "b")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutGroupedMultipleProperties() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.local(ImmutableList.of(grouped("a"), grouped("b")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutGroupedWithSort() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.local(ImmutableList.of(grouped("a"), sorted("b", ASC_NULLS_FIRST)))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutUnpartitionedWithGroupAndSort() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.undistributedWithLocal(ImmutableList.of(grouped("a"), sorted("b", ASC_NULLS_FIRST)))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - @Test - public void testPickLayoutPartitionedWithGroup() - { - Comparator preference = streamingExecutionPreference - (PreferredProperties.partitionedWithLocal( - ImmutableSet.of(symbol("a")), - ImmutableList.of(grouped("a")))); - - List input = ImmutableList.builder() - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .build(); - - List expected = ImmutableList.builder() - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(streamPartitionedOn("a")) - .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(hashDistributedOn("a")) - .build()) - .add(builder() - .global(arbitraryPartition()) - .local(ImmutableList.of(grouped("a", "b"))) - .build()) - .add(builder() - .global(arbitraryPartition()) - .build()) - .build(); - assertEquals(stableSort(input, preference), expected); - } - - private static List stableSort(List list, Comparator comparator) - { - ArrayList copy = Lists.newArrayList(list); - Collections.sort(copy, comparator); - return copy; - } - - private static Global hashDistributedOn(String... columnNames) - { - return partitionedOn(FIXED_HASH_DISTRIBUTION, arguments(columnNames), Optional.of(arguments(columnNames))); - } - - public static Global singleStream() - { - return Global.streamPartitionedOn(ImmutableList.of()); - } - - private static Global streamPartitionedOn(String... columnNames) - { - return Global.streamPartitionedOn(arguments(columnNames)); - } - - private static ConstantProperty constant(String column) - { - return new ConstantProperty<>(symbol(column)); - } - - private static GroupingProperty grouped(String... columns) - { - return new GroupingProperty<>(Lists.transform(Arrays.asList(columns), Symbol::new)); - } - - private static SortingProperty sorted(String column, SortOrder order) - { - return new SortingProperty<>(symbol(column), order); - } - - private static Symbol symbol(String name) - { - return new Symbol(name); - } - - private static List arguments(String[] columnNames) - { - return Arrays.asList(columnNames).stream() - .map(Symbol::new) - .collect(toImmutableList()); - } -}