From 84e58cab95fac8f4610a2c8bb8c074d978424eb1 Mon Sep 17 00:00:00 2001 From: Zac Wen Date: Mon, 19 Jul 2021 17:01:37 -0700 Subject: [PATCH] Add configuration for materialized view consistency Adding a configuration to control the materialized view data consistency. When enabled and reading from materialized view, partition stitching is applied to achieve data consistency. --- .../facebook/presto/SystemSessionProperties.java | 11 +++++++++++ .../presto/sql/analyzer/FeaturesConfig.java | 14 ++++++++++++++ .../presto/sql/analyzer/StatementAnalyzer.java | 3 ++- .../presto/sql/analyzer/TestFeaturesConfig.java | 7 +++++-- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index af3e41cb369b..c6dd2e858705 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -197,6 +197,7 @@ public final class SystemSessionProperties public static final String PARTIAL_RESULTS_MAX_EXECUTION_TIME_MULTIPLIER = "partial_results_max_execution_time_multiplier"; public static final String OFFSET_CLAUSE_ENABLED = "offset_clause_enabled"; public static final String VERBOSE_EXCEEDED_MEMORY_LIMIT_ERRORS_ENABLED = "verbose_exceeded_memory_limit_errors_enabled"; + public static final String MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED = "materialized_view_data_consistency_enabled"; private final List> sessionProperties; @@ -1054,6 +1055,11 @@ public SystemSessionProperties( VERBOSE_EXCEEDED_MEMORY_LIMIT_ERRORS_ENABLED, "When enabled the error message for exceeded memory limit errors will contain additional operator memory allocation details", nodeMemoryConfig.isVerboseExceededMemoryLimitErrorsEnabled(), + false), + booleanProperty( + MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED, + "When enabled and reading from materialized view, partition stitching is applied to achieve data consistency", + featuresConfig.isMaterializedViewDataConsistencyEnabled(), false)); } @@ -1779,4 +1785,9 @@ public static boolean isVerboseExceededMemoryLimitErrorsEnabled(Session session) { return session.getSystemProperty(VERBOSE_EXCEEDED_MEMORY_LIMIT_ERRORS_ENABLED, Boolean.class); } + + public static boolean isMaterializedViewDataConsistencyEnabled(Session session) + { + return session.getSystemProperty(MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 64fafb886f71..3c551b6d491c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -200,6 +200,7 @@ public class FeaturesConfig private double partialResultsMaxExecutionTimeMultiplier = 2.0; private boolean offsetClauseEnabled; + private boolean materializedViewDataConsistencyEnabled = true; public enum PartitioningPrecisionStrategy { @@ -1757,4 +1758,17 @@ public FeaturesConfig setOffsetClauseEnabled(boolean offsetClauseEnabled) this.offsetClauseEnabled = offsetClauseEnabled; return this; } + + public boolean isMaterializedViewDataConsistencyEnabled() + { + return materializedViewDataConsistencyEnabled; + } + + @Config("materialized-view-data-consistency-enabled") + @ConfigDescription("When enabled and reading from materialized view, partition stitching is applied to achieve data consistency") + public FeaturesConfig setMaterializedViewDataConsistencyEnabled(boolean materializedViewDataConsistencyEnabled) + { + this.materializedViewDataConsistencyEnabled = materializedViewDataConsistencyEnabled; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index b0426167fdf7..d7de88f3ac1d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -175,6 +175,7 @@ import static com.facebook.presto.SystemSessionProperties.getMaxGroupingSets; import static com.facebook.presto.SystemSessionProperties.isAllowWindowOrderByLiterals; +import static com.facebook.presto.SystemSessionProperties.isMaterializedViewDataConsistencyEnabled; import static com.facebook.presto.common.predicate.TupleDomain.extractFixedValues; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -1207,7 +1208,7 @@ protected Scope visitTable(Table table, Optional scope) Optional optionalMaterializedView = metadata.getMaterializedView(session, name); Statement statement = analysis.getStatement(); - if (optionalMaterializedView.isPresent() && statement instanceof Query) { + if (isMaterializedViewDataConsistencyEnabled(session) && optionalMaterializedView.isPresent() && statement instanceof Query) { // When the materialized view has already been expanded, do not process it. Just use it as a table. MaterializedViewAnalysisState materializedViewAnalysisState = analysis.getMaterializedViewAnalysisState(table); QualifiedObjectName materializedViewName = createQualifiedObjectName(session, table, table.getName()); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 878e9e4c9809..2199da00fd0b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -172,7 +172,8 @@ public void testDefaults() .setPartialResultsEnabled(false) .setPartialResultsCompletionRatioThreshold(0.5) .setOffsetClauseEnabled(false) - .setPartialResultsMaxExecutionTimeMultiplier(2.0)); + .setPartialResultsMaxExecutionTimeMultiplier(2.0) + .setMaterializedViewDataConsistencyEnabled(true)); } @Test @@ -297,6 +298,7 @@ public void testExplicitPropertyMappings() .put("partial-results-completion-ratio-threshold", "0.9") .put("partial-results-max-execution-time-multiplier", "1.5") .put("offset-clause-enabled", "true") + .put("materialized-view-data-consistency-enabled", "false") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -418,7 +420,8 @@ public void testExplicitPropertyMappings() .setPartialResultsEnabled(true) .setPartialResultsCompletionRatioThreshold(0.9) .setOffsetClauseEnabled(true) - .setPartialResultsMaxExecutionTimeMultiplier(1.5); + .setPartialResultsMaxExecutionTimeMultiplier(1.5) + .setMaterializedViewDataConsistencyEnabled(false); assertFullMapping(properties, expected); }