diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml
index 7e37ea38ff134..99e7c2f28560d 100644
--- a/presto-verifier/pom.xml
+++ b/presto-verifier/pom.xml
@@ -45,6 +45,11 @@
presto-hive-common
+
+ com.facebook.presto
+ presto-hive-metastore
+
+
com.facebook.presto
presto-jdbc
@@ -209,6 +214,11 @@
provided
+
+ joda-time
+ joda-time
+
+
com.facebook.presto
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
index 1e3c76d970e39..2caa6c5f86d66 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
@@ -13,6 +13,7 @@
*/
package com.facebook.presto.verifier.checksum;
+import com.facebook.presto.sql.tree.Expression;
import com.facebook.presto.sql.tree.FunctionCall;
import com.facebook.presto.sql.tree.GroupBy;
import com.facebook.presto.sql.tree.GroupingElement;
@@ -52,17 +53,17 @@ public ChecksumValidator(Map> columnValidato
this.columnValidators = columnValidators;
}
- public Query generateChecksumQuery(QualifiedName tableName, List columns)
+ public Query generateChecksumQuery(QualifiedName tableName, List columns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
for (Column column : columns) {
selectItems.addAll(columnValidators.get(column.getCategory()).get().generateChecksumColumns(column));
}
- return simpleQuery(new Select(false, selectItems.build()), new Table(tableName));
+ return simpleQuery(new Select(false, selectItems.build()), new Table(tableName), partitionPredicate, Optional.empty());
}
- public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns)
+ public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
@@ -79,7 +80,7 @@ public Query generatePartitionChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns)
+ public Query generateBucketChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
@@ -106,7 +107,7 @@ public Query generateBucketChecksumQuery(QualifiedName tableName, List p
return simpleQuery(
new Select(false, selectItems.build()),
new Table(tableName),
- Optional.empty(),
+ partitionPredicate,
Optional.of(new GroupBy(false, groupByList.build())),
Optional.empty(),
Optional.of(new OrderBy(orderByList.build())),
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java
index b9a3c3b3ddaa5..98fa2b1121980 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java
@@ -596,14 +596,16 @@ private void prepareReuseTableResubmit()
controlConfiguration.getUsername(),
controlConfiguration.getPassword(),
Optional.of(controlConfiguration.getSessionProperties()),
- false),
+ false,
+ Optional.empty()),
new QueryConfiguration(
testConfiguration.getCatalog(),
testConfiguration.getSchema(),
testConfiguration.getUsername(),
testConfiguration.getPassword(),
Optional.of(testConfiguration.getSessionProperties()),
- false));
+ false,
+ Optional.empty()));
}
protected static class QueryContext
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
index 73b63cc369058..cc20377c7a06a 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
@@ -107,7 +107,7 @@ public DataMatchResult verify(
ChecksumQueryContext testChecksumQueryContext)
{
List testColumns = getColumns(getHelperAction(), typeManager, test.getObjectName());
- Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns);
+ Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns, test.getPartitionsPredicate());
testChecksumQueryContext.setChecksumQuery(formatSql(testChecksumQuery));
List controlColumns = null;
@@ -115,7 +115,7 @@ public DataMatchResult verify(
if (isControlEnabled()) {
controlColumns = getColumns(getHelperAction(), typeManager, control.getObjectName());
- Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns);
+ Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns, control.getPartitionsPredicate());
controlChecksumQueryContext.setChecksumQuery(formatSql(controlChecksumQuery));
QueryResult controlChecksum = callAndConsume(
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
index 3283aaa367be5..dabf0121b3f1d 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
@@ -144,7 +144,7 @@ private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult co
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
// Run checksum query
- Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
+ Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns, Optional.empty());
ChecksumResult testChecksum = getOnlyElement(callAndConsume(
() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet),
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
index c36dd5f52afb8..fee119d50360f 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
@@ -273,7 +273,7 @@ private List runPartitionChecksum(
ChecksumQueryContext checksumQueryContext,
QueryStage queryStage)
{
- Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns);
+ Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns, bundle.getPartitionsPredicate());
checksumQueryContext.setPartitionChecksumQuery(formatSql(partitionChecksumQuery));
return callAndConsume(
() -> getHelperAction().execute(partitionChecksumQuery, queryStage, ChecksumResult::fromResultSet),
@@ -287,7 +287,7 @@ private List runBucketChecksum(
ChecksumQueryContext checksumQueryContext,
QueryStage queryStage)
{
- Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns);
+ Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns, bundle.getPartitionsPredicate());
List checksumResults = callAndConsume(
() -> getHelperAction().execute(bucketChecksumQuery, queryStage, ChecksumResult::fromResultSet),
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(checksumQueryContext::setBucketChecksumQueryId)).getResults();
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
index 3ae5f0e2d7718..eecee85523c8c 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
@@ -13,6 +13,7 @@
*/
package com.facebook.presto.verifier.framework;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.jdbi.v3.core.mapper.reflect.ColumnName;
import org.jdbi.v3.core.mapper.reflect.JdbiConstructor;
@@ -37,6 +38,7 @@ public class QueryConfiguration
private final Optional password;
private final Map sessionProperties;
private final boolean isReusableTable;
+ private final List partitions;
@JdbiConstructor
public QueryConfiguration(
@@ -45,9 +47,10 @@ public QueryConfiguration(
@ColumnName("username") Optional username,
@ColumnName("password") Optional password,
@ColumnName("session_properties") Optional