Skip to content

Commit

Permalink
Remove Type equalTo and hash uses in JoinCompiler
Browse files Browse the repository at this point in the history
  • Loading branch information
dain committed Oct 7, 2020
1 parent f551d80 commit e0c9a86
Show file tree
Hide file tree
Showing 23 changed files with 159 additions and 110 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import io.prestosql.operator.DriverFactory;
import io.prestosql.operator.OperatorFactory;
import io.prestosql.operator.TaskContext;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.sql.planner.plan.PlanNodeId;
import io.prestosql.testing.LocalQueryRunner;
import io.prestosql.testing.NullOutputOperator.NullOutputOperatorFactory;
Expand All @@ -28,14 +27,11 @@
import java.util.List;
import java.util.OptionalInt;

import static io.prestosql.metadata.MetadataManager.createTestMetadataManager;
import static io.prestosql.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION;

public abstract class AbstractSimpleOperatorBenchmark
extends AbstractOperatorBenchmark
{
protected static final JoinCompiler JOIN_COMPILER = new JoinCompiler(createTestMetadataManager());

protected AbstractSimpleOperatorBenchmark(
LocalQueryRunner localQueryRunner,
String benchmarkName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import io.prestosql.spi.PageBuilder;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.Type;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.sql.planner.plan.AggregationNode.Step;
import io.prestosql.sql.planner.plan.PlanNodeId;
import io.prestosql.sql.tree.QualifiedName;
Expand Down Expand Up @@ -124,7 +125,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
Optional.empty(),
10_000,
Optional.of(DataSize.of(16, MEGABYTE)),
JOIN_COMPILER,
new JoinCompiler(localQueryRunner.getTypeOperators()),
localQueryRunner.getBlockTypeOperators(),
false);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import io.prestosql.operator.OperatorFactory;
import io.prestosql.operator.aggregation.InternalAggregationFunction;
import io.prestosql.spi.type.Type;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.sql.planner.plan.AggregationNode.Step;
import io.prestosql.sql.planner.plan.PlanNodeId;
import io.prestosql.sql.tree.QualifiedName;
Expand Down Expand Up @@ -64,7 +65,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
Optional.empty(),
100_000,
Optional.of(DataSize.of(16, MEGABYTE)),
JOIN_COMPILER,
new JoinCompiler(localQueryRunner.getTypeOperators()),
localQueryRunner.getBlockTypeOperators(),
false);
return ImmutableList.of(tableScanOperator, aggregationOperator);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,14 @@
import io.prestosql.spi.type.SqlTimestampWithTimeZone;
import io.prestosql.spi.type.SqlVarbinary;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeOperators;
import io.prestosql.spi.type.VarcharType;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.testing.MaterializedResult;
import io.prestosql.testing.MaterializedRow;
import io.prestosql.testing.TestingConnectorSession;
import io.prestosql.testing.TestingNodeManager;
import io.prestosql.type.BlockTypeOperators;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
Expand Down Expand Up @@ -177,7 +179,6 @@
import static io.airlift.testing.Assertions.assertInstanceOf;
import static io.airlift.testing.Assertions.assertLessThanOrEqual;
import static io.airlift.units.DataSize.Unit.KILOBYTE;
import static io.prestosql.metadata.MetadataManager.createTestMetadataManager;
import static io.prestosql.plugin.hive.AbstractTestHive.TransactionDeleteInsertTestTag.COMMIT;
import static io.prestosql.plugin.hive.AbstractTestHive.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE;
import static io.prestosql.plugin.hive.AbstractTestHive.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT;
Expand Down Expand Up @@ -461,7 +462,9 @@ private static RowType toRowType(List<ColumnMetadata> columns)
// exclude formats that change table schema with serde
ImmutableSet.of(AVRO, CSV));

private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(createTestMetadataManager());
private static final TypeOperators TYPE_OPERATORS = new TypeOperators();
private static final BlockTypeOperators BLOCK_TYPE_OPERATORS = new BlockTypeOperators(TYPE_OPERATORS);
private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(TYPE_OPERATORS);

private static final List<ColumnMetadata> STATISTICS_TABLE_COLUMNS = ImmutableList.<ColumnMetadata>builder()
.add(new ColumnMetadata("t_boolean", BOOLEAN))
Expand Down Expand Up @@ -791,7 +794,7 @@ protected final void setup(String databaseName, HiveConfig hiveConfig, HiveMetas
hdfsEnvironment,
PAGE_SORTER,
metastoreClient,
new GroupByHashPageIndexerFactory(JOIN_COMPILER),
new GroupByHashPageIndexerFactory(JOIN_COMPILER, BLOCK_TYPE_OPERATORS),
TYPE_MANAGER,
getHiveConfig(),
locationService,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.security.ConnectorIdentity;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeOperators;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.testing.MaterializedResult;
import io.prestosql.testing.MaterializedRow;
import io.prestosql.testing.TestingNodeManager;
import io.prestosql.type.BlockTypeOperators;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
Expand All @@ -87,7 +89,6 @@
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
import static io.airlift.concurrent.MoreFutures.getFutureValue;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.prestosql.metadata.MetadataManager.createTestMetadataManager;
import static io.prestosql.plugin.hive.AbstractTestHive.createTableProperties;
import static io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnHandles;
import static io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata;
Expand Down Expand Up @@ -219,12 +220,14 @@ protected void setup(String host, int port, String databaseName, boolean s3Selec
config.getMaxSplitsPerSecond(),
config.getRecursiveDirWalkerEnabled(),
TYPE_MANAGER);
TypeOperators typeOperators = new TypeOperators();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
pageSinkProvider = new HivePageSinkProvider(
getDefaultHiveFileWriterFactories(config, hdfsEnvironment),
hdfsEnvironment,
PAGE_SORTER,
metastoreClient,
new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())),
new GroupByHashPageIndexerFactory(new JoinCompiler(typeOperators), blockTypeOperators),
TYPE_MANAGER,
config,
locationService,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import io.prestosql.spi.connector.DynamicFilter;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeOperators;
import io.prestosql.sql.gen.JoinCompiler;
import io.prestosql.testing.MaterializedResult;
import io.prestosql.testing.TestingNodeManager;
Expand All @@ -41,6 +42,7 @@
import io.prestosql.tpch.LineItemGenerator;
import io.prestosql.tpch.TpchColumnType;
import io.prestosql.tpch.TpchColumnTypes;
import io.prestosql.type.BlockTypeOperators;
import org.apache.hadoop.fs.Path;
import org.testng.annotations.Test;

Expand All @@ -58,7 +60,6 @@
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
import static io.airlift.concurrent.MoreFutures.getFutureValue;
import static io.airlift.testing.Assertions.assertGreaterThan;
import static io.prestosql.metadata.MetadataManager.createTestMetadataManager;
import static io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR;
import static io.prestosql.plugin.hive.HiveColumnHandle.createBaseColumn;
import static io.prestosql.plugin.hive.HiveCompressionCodec.NONE;
Expand Down Expand Up @@ -269,12 +270,14 @@ private static ConnectorPageSink createPageSink(HiveTransactionHandle transactio
false,
false);
JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
TypeOperators typeOperators = new TypeOperators();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
HivePageSinkProvider provider = new HivePageSinkProvider(
getDefaultHiveFileWriterFactories(config, HDFS_ENVIRONMENT),
HDFS_ENVIRONMENT,
PAGE_SORTER,
metastore,
new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())),
new GroupByHashPageIndexerFactory(new JoinCompiler(typeOperators), blockTypeOperators),
TYPE_MANAGER,
config,
new HiveLocationService(HDFS_ENVIRONMENT),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.airlift.units.DataSize;
import io.prestosql.Session;
import io.prestosql.geospatial.Rectangle;
import io.prestosql.metadata.Metadata;
import io.prestosql.operator.SpatialIndexBuilderOperator.SpatialPredicate;
import io.prestosql.spi.Page;
import io.prestosql.spi.PageBuilder;
Expand Down Expand Up @@ -58,7 +57,6 @@
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.metadata.MetadataManager.createTestMetadataManager;
import static io.prestosql.operator.SyntheticAddress.decodePosition;
import static io.prestosql.operator.SyntheticAddress.decodeSliceIndex;
import static io.prestosql.operator.SyntheticAddress.encodeSyntheticAddress;
Expand Down Expand Up @@ -130,10 +128,10 @@ public interface Factory
public static class TestingFactory
implements Factory
{
public static final TypeOperators TYPE_OPERATORS = new TypeOperators();
private static final OrderingCompiler ORDERING_COMPILER = new OrderingCompiler();
private static final Metadata METADATA = createTestMetadataManager();
private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(METADATA);
private static final BlockTypeOperators TYPE_OPERATOR_FACTORY = new BlockTypeOperators(new TypeOperators());
private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(TYPE_OPERATORS);
private static final BlockTypeOperators TYPE_OPERATOR_FACTORY = new BlockTypeOperators(TYPE_OPERATORS);
private final boolean eagerCompact;

public TestingFactory(boolean eagerCompact)
Expand Down
Loading

0 comments on commit e0c9a86

Please sign in to comment.