diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java index 0e6f0f7051fa82..7f3aa05a79dbc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java @@ -20,31 +20,32 @@ import org.apache.doris.catalog.DynamicPartitionProperty; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; public class CloudPropertyAnalyzer extends PropertyAnalyzer { public CloudPropertyAnalyzer() { - forceProperties = ImmutableMap.builder() - .put(PropertyAnalyzer.PROPERTIES_INMEMORY, "true") - //.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, null) - .put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "") - .put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, "") - .put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, "") - .put(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM, "-1") - .put(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, "false") - .put(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE, "true") - .put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, - String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())) - .put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, + forceProperties = ImmutableList.of( + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_INMEMORY, "true"), + RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM), + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"), + RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY), + RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME), + RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM), + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, "false"), + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE, "true"), + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, + String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())), + RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, + ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt()), + RewriteProperty.delete(DynamicPartitionProperty.STORAGE_MEDIUM), + RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_NUM, + String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())), + RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_ALLOCATION, ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt()) - //.put(DynamicPartitionProperty.PROPERTIES_STORAGE_MEDIUM, "") - .put(DynamicPartitionProperty.REPLICATION_NUM, - String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())) - .put(DynamicPartitionProperty.REPLICATION_ALLOCATION, - ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt()) - .build(); + ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index afebd26ee06a39..957dd5a72f6942 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -49,7 +49,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; @@ -196,18 +196,61 @@ public class PropertyAnalyzer { public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600; public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5; - // Use forceProperties to rewrite olap's property. - // For a key-value pair in forceProperties, - // if the value is null, then delete this property from properties and skip check this property, - // otherwise rewrite this property into properties and check property using the force value. - // - // In most cases, specified a none-null force value is better then specified a null force value. - protected ImmutableMap forceProperties; + public enum RewriteType { + PUT, // always put property + REPLACE, // replace if exists property + DELETE, // delete property + } + + public static class RewriteProperty { + RewriteType rewriteType; + String key; + String value; + + private RewriteProperty(RewriteType rewriteType, String key, String value) { + this.rewriteType = rewriteType; + this.key = key; + this.value = value; + } + + public static RewriteProperty put(String key, String value) { + return new RewriteProperty(RewriteType.PUT, key, value); + } + + public static RewriteProperty replace(String key, String value) { + return new RewriteProperty(RewriteType.REPLACE, key, value); + } + + public static RewriteProperty delete(String key) { + return new RewriteProperty(RewriteType.DELETE, key, null); + } + + public void rewrite(Map properties) { + switch (rewriteType) { + case PUT: + properties.put(key, value); + break; + case REPLACE: + if (properties.containsKey(key)) { + properties.put(key, value); + } + break; + case DELETE: + properties.remove(key); + break; + default: + break; + } + } + + } + + protected ImmutableList forceProperties; public PropertyAnalyzer() { - forceProperties = ImmutableMap.builder() - .put(PROPERTIES_FILE_CACHE_TTL_SECONDS, "0") - .build(); + forceProperties = ImmutableList.of( + RewriteProperty.replace(PROPERTIES_FILE_CACHE_TTL_SECONDS, "0") + ); } private static class SingletonHolder { @@ -1319,13 +1362,7 @@ public Map rewriteOlapProperties( } private void rewriteForceProperties(Map properties) { - forceProperties.forEach((property, value) -> { - if (value == null) { - properties.remove(property); - } else { - properties.put(property, value); - } - }); + forceProperties.forEach(property -> property.rewrite(properties)); } private static Map rewriteReplicaAllocationProperties( diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java index 6c3d0d3ab9b514..e7625e228f1699 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java @@ -23,6 +23,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.PropertyAnalyzer; +import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -43,8 +44,10 @@ public void testCreate() throws Exception { Assert.assertTrue(envFactory.createTablet() instanceof CloudTablet); Assert.assertTrue(envFactory.createReplica() instanceof CloudReplica); - Map properties = PropertyAnalyzer.getInstance().rewriteOlapProperties( - "catalog_not_exist", "db_not_exist", null); + Map properties = Maps.newHashMap(); + properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "100"); + PropertyAnalyzer.getInstance().rewriteOlapProperties( + "catalog_not_exist", "db_not_exist", properties); Assert.assertEquals("1", properties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)); }