Skip to content

Commit

Permalink
[fix](merge-cloud) create cloud table failed (apache#30502)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yujun777
yujun777 authored Jan 29, 2024
1 parent 34f4061 commit f189e66
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 39 deletions.
39 changes: 20 additions & 19 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,31 +20,32 @@
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableList;

public class CloudPropertyAnalyzer extends PropertyAnalyzer {

public CloudPropertyAnalyzer() {
forceProperties = ImmutableMap.<String, String>builder()
.put(PropertyAnalyzer.PROPERTIES_INMEMORY, "true")
//.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, null)
.put(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "")
.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, "")
.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, "")
.put(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM, "-1")
.put(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, "false")
.put(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE, "true")
.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum()))
.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
forceProperties = ImmutableList.of(
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_INMEMORY, "true"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM),
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM),
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, "false"),
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE, "true"),
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())),
RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt()),
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_MEDIUM),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_ALLOCATION,
ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt())
//.put(DynamicPartitionProperty.PROPERTIES_STORAGE_MEDIUM, "")
.put(DynamicPartitionProperty.REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum()))
.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
ReplicaAllocation.DEFAULT_ALLOCATION.toCreateStmt())
.build();
);
}

}
73 changes: 55 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -196,18 +196,61 @@ public class PropertyAnalyzer {
public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600;
public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5;

// Use forceProperties to rewrite olap's property.
// For a key-value pair in forceProperties,
// if the value is null, then delete this property from properties and skip check this property,
// otherwise rewrite this property into properties and check property using the force value.
//
// In most cases, specified a none-null force value is better then specified a null force value.
protected ImmutableMap<String, String> forceProperties;
public enum RewriteType {
PUT, // always put property
REPLACE, // replace if exists property
DELETE, // delete property
}

public static class RewriteProperty {
RewriteType rewriteType;
String key;
String value;

private RewriteProperty(RewriteType rewriteType, String key, String value) {
this.rewriteType = rewriteType;
this.key = key;
this.value = value;
}

public static RewriteProperty put(String key, String value) {
return new RewriteProperty(RewriteType.PUT, key, value);
}

public static RewriteProperty replace(String key, String value) {
return new RewriteProperty(RewriteType.REPLACE, key, value);
}

public static RewriteProperty delete(String key) {
return new RewriteProperty(RewriteType.DELETE, key, null);
}

public void rewrite(Map<String, String> properties) {
switch (rewriteType) {
case PUT:
properties.put(key, value);
break;
case REPLACE:
if (properties.containsKey(key)) {
properties.put(key, value);
}
break;
case DELETE:
properties.remove(key);
break;
default:
break;
}
}

}

protected ImmutableList<RewriteProperty> forceProperties;

public PropertyAnalyzer() {
forceProperties = ImmutableMap.<String, String>builder()
.put(PROPERTIES_FILE_CACHE_TTL_SECONDS, "0")
.build();
forceProperties = ImmutableList.of(
RewriteProperty.replace(PROPERTIES_FILE_CACHE_TTL_SECONDS, "0")
);
}

private static class SingletonHolder {
Expand Down Expand Up @@ -1319,13 +1362,7 @@ public Map<String, String> rewriteOlapProperties(
}

private void rewriteForceProperties(Map<String, String> properties) {
forceProperties.forEach((property, value) -> {
if (value == null) {
properties.remove(property);
} else {
properties.put(property, value);
}
});
forceProperties.forEach(property -> property.rewrite(properties));
}

private static Map<String, String> rewriteReplicaAllocationProperties(
Expand Down
7 changes: 5 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.common.Config;
import org.apache.doris.common.util.PropertyAnalyzer;

import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Test;

Expand All @@ -43,8 +44,10 @@ public void testCreate() throws Exception {
Assert.assertTrue(envFactory.createTablet() instanceof CloudTablet);
Assert.assertTrue(envFactory.createReplica() instanceof CloudReplica);

Map<String, String> properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(
"catalog_not_exist", "db_not_exist", null);
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "100");
PropertyAnalyzer.getInstance().rewriteOlapProperties(
"catalog_not_exist", "db_not_exist", properties);
Assert.assertEquals("1", properties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM));
}

Expand Down

0 comments on commit f189e66

Please sign in to comment.