diff --git a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java old mode 100755 new mode 100644 similarity index 99% rename from ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java rename to ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java index 86f22f05d9..c312f84a3c --- a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.netease.arctic.server.utils; +package com.netease.arctic.api.config; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -38,12 +38,12 @@ import java.util.stream.IntStream; /** Utility class for {@link Configurations} related helper functions. */ -public class ConfigurationUtil { +public class ConfigHelpers { private static final String[] EMPTY = new String[0]; // Make sure that we cannot instantiate this class - private ConfigurationUtil() {} + private ConfigHelpers() {} /** * Creates a new {@link Configurations} from the given {@link Properties}. diff --git a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOption.java b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigOption.java similarity index 95% rename from ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOption.java rename to ams/api/src/main/java/com/netease/arctic/api/config/ConfigOption.java index 789f5e7a1d..1310eeddae 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOption.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigOption.java @@ -16,20 +16,15 @@ * limitations under the License. */ -package com.netease.arctic.server.utils; +package com.netease.arctic.api.config; import static com.google.common.base.Preconditions.checkNotNull; -import com.netease.arctic.server.ArcticManagementConf; - /** * A {@code ConfigOption} describes a configuration parameter. It encapsulates the configuration * key, deprecated older versions of the key, and an optional default value for the configuration * parameter. * - *

{@code ConfigOptions} are built via the {@link ArcticManagementConf} class. Once created, a - * config option is immutable. - * * @param The type of value associated with the configuration option. */ public class ConfigOption { diff --git a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOptions.java b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigOptions.java similarity index 99% rename from ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOptions.java rename to ams/api/src/main/java/com/netease/arctic/api/config/ConfigOptions.java index 0f850a5383..879000e70d 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigOptions.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigOptions.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.netease.arctic.server.utils; +package com.netease.arctic.api.config; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/ams/server/src/main/java/com/netease/arctic/server/utils/Configurations.java b/ams/api/src/main/java/com/netease/arctic/api/config/Configurations.java similarity index 98% rename from ams/server/src/main/java/com/netease/arctic/server/utils/Configurations.java rename to ams/api/src/main/java/com/netease/arctic/api/config/Configurations.java index 1b79f5a621..9f465e7793 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/utils/Configurations.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/Configurations.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.netease.arctic.server.utils; +package com.netease.arctic.api.config; import static com.google.common.base.Preconditions.checkNotNull; @@ -433,7 +433,7 @@ public > T getEnum( Object rawValue = getRawValueFromOption(configOption).orElseGet(configOption::defaultValue); try { - return ConfigurationUtil.convertToEnum(rawValue, enumClass); + return ConfigHelpers.convertToEnum(rawValue, enumClass); } catch (IllegalArgumentException ex) { final String errorMessage = String.format( @@ -538,9 +538,9 @@ public Optional getOptional(ConfigOption option) { try { if (option.isList()) { - return rawValue.map(v -> ConfigurationUtil.convertToList(v, clazz)); + return rawValue.map(v -> ConfigHelpers.convertToList(v, clazz)); } else { - return rawValue.map(v -> ConfigurationUtil.convertValue(v, clazz)); + return rawValue.map(v -> ConfigHelpers.convertValue(v, clazz)); } } catch (Exception e) { throw new IllegalArgumentException( @@ -562,7 +562,7 @@ public Map toMap() { synchronized (this.confData) { Map ret = new HashMap<>(this.confData.size()); for (Map.Entry entry : confData.entrySet()) { - ret.put(entry.getKey(), ConfigurationUtil.convertToString(entry.getValue())); + ret.put(entry.getKey(), ConfigHelpers.convertToString(entry.getValue())); } return ret; } diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java b/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java index aff7ba6233..24f8d916ff 100644 --- a/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java @@ -19,6 +19,10 @@ package com.netease.arctic.api.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.netease.arctic.table.ArcticTable; +import com.netease.arctic.table.TableProperties; +import com.netease.arctic.utils.CompatiblePropertyUtil; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Objects; @@ -30,6 +34,7 @@ import org.slf4j.LoggerFactory; import java.util.Locale; +import java.util.Map; import java.util.Set; /** Data expiration configuration. */ @@ -40,15 +45,19 @@ public class DataExpirationConfig { // data-expire.field private String expirationField; // data-expire.level + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT) private ExpireLevel expirationLevel; // data-expire.retention-time private long retentionTime; // data-expire.datetime-string-pattern + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT) private String dateTimePattern; // data-expire.datetime-number-format + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT) private String numberDateFormat; - // data-expire.since - private Since since; + // data-expire.base-on-rule + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT) + private BaseOnRule baseOnRule; @VisibleForTesting public enum ExpireLevel { @@ -66,14 +75,15 @@ public static ExpireLevel fromString(String level) { } @VisibleForTesting - public enum Since { - LATEST_SNAPSHOT, - CURRENT_TIMESTAMP; + public enum BaseOnRule { + LAST_COMMIT_TIME, + CURRENT_TIME; - public static Since fromString(String since) { - Preconditions.checkArgument(null != since, "data-expire.since is invalid: null"); + public static BaseOnRule fromString(String since) { + Preconditions.checkArgument( + null != since, TableProperties.DATA_EXPIRATION_BASE_ON_RULE + " is invalid: null"); try { - return Since.valueOf(since.toUpperCase(Locale.ENGLISH)); + return BaseOnRule.valueOf(since.toUpperCase(Locale.ENGLISH)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format("Unable to expire data since: %s", since), e); @@ -95,14 +105,111 @@ public DataExpirationConfig( long retentionTime, String dateTimePattern, String numberDateFormat, - Since since) { + BaseOnRule baseOnRule) { this.enabled = enabled; this.expirationField = expirationField; this.expirationLevel = expirationLevel; this.retentionTime = retentionTime; this.dateTimePattern = dateTimePattern; this.numberDateFormat = numberDateFormat; - this.since = since; + this.baseOnRule = baseOnRule; + } + + public DataExpirationConfig(ArcticTable table) { + Map properties = table.properties(); + expirationField = + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_FIELD, null); + Types.NestedField field = table.schema().findField(expirationField); + Preconditions.checkArgument( + StringUtils.isNoneBlank(expirationField) && null != field, + String.format( + "Field(%s) used to determine data expiration is illegal for table(%s)", + expirationField, table.name())); + Type.TypeID typeID = field.type().typeId(); + Preconditions.checkArgument( + FIELD_TYPES.contains(typeID), + String.format( + "The type(%s) of filed(%s) is incompatible for table(%s)", + typeID.name(), expirationField, table.name())); + + expirationLevel = + ExpireLevel.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_LEVEL, + TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)); + + String retention = + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); + if (StringUtils.isNotBlank(retention)) { + retentionTime = ConfigHelpers.TimeUtils.parseDuration(retention).toMillis(); + } + + dateTimePattern = + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT); + numberDateFormat = + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT); + baseOnRule = + BaseOnRule.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT)); + } + + public static DataExpirationConfig parse(Map properties) { + boolean gcEnabled = + CompatiblePropertyUtil.propertyAsBoolean( + properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); + DataExpirationConfig config = + new DataExpirationConfig() + .setEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_DATA_EXPIRATION, + TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT)) + .setExpirationLevel( + ExpireLevel.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_LEVEL, + TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT))) + .setExpirationField( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_FIELD, null)) + .setDateTimePattern( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)) + .setNumberDateFormat( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)) + .setBaseOnRule( + BaseOnRule.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT))); + String retention = + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); + if (StringUtils.isNotBlank(retention)) { + config.setRetentionTime(ConfigHelpers.TimeUtils.parseDuration(retention).toMillis()); + } + + return config; } public boolean isEnabled() { @@ -159,12 +266,12 @@ public DataExpirationConfig setNumberDateFormat(String numberDateFormat) { return this; } - public Since getSince() { - return since; + public BaseOnRule getBaseOnRule() { + return baseOnRule; } - public DataExpirationConfig setSince(Since since) { - this.since = since; + public DataExpirationConfig setBaseOnRule(BaseOnRule baseOnRule) { + this.baseOnRule = baseOnRule; return this; } @@ -183,7 +290,7 @@ public boolean equals(Object o) { && expirationLevel == config.expirationLevel && Objects.equal(dateTimePattern, config.dateTimePattern) && Objects.equal(numberDateFormat, config.numberDateFormat) - && since == config.since; + && baseOnRule == config.baseOnRule; } @Override @@ -195,7 +302,7 @@ public int hashCode() { retentionTime, dateTimePattern, numberDateFormat, - since); + baseOnRule); } public boolean isValid(Types.NestedField field, String name) { diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/OptimizingConfig.java b/ams/api/src/main/java/com/netease/arctic/api/config/OptimizingConfig.java index 48906b733b..0344b73f5f 100644 --- a/ams/api/src/main/java/com/netease/arctic/api/config/OptimizingConfig.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/OptimizingConfig.java @@ -19,8 +19,14 @@ package com.netease.arctic.api.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.netease.arctic.properties.HiveTableProperties; +import com.netease.arctic.table.TableProperties; +import com.netease.arctic.utils.CompatiblePropertyUtil; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Objects; +import org.apache.iceberg.util.PropertyUtil; + +import java.util.Map; /** Configuration for optimizing process scheduling and executing. */ @JsonIgnoreProperties(ignoreUnknown = true) @@ -28,42 +34,61 @@ public class OptimizingConfig { // self-optimizing.enabled private boolean enabled; + // self-optimizing.quota private double targetQuota; + // self-optimizing.group private String optimizerGroup; + // self-optimizing.execute.num-retries private int maxExecuteRetryCount; + // self-optimizing.commit.num-retries private int maxCommitRetryCount; + // self-optimizing.target-size private long targetSize; + // self-optimizing.max-task-size-bytes private long maxTaskSize; + // self-optimizing.max-file-count private int maxFileCount; + // read.split.open-file-cost private long openFileCost; + // self-optimizing.fragment-ratio private int fragmentRatio; + // self-optimizing.min-target-size-ratio private double minTargetSizeRatio; + // self-optimizing.minor.trigger.file-count private int minorLeastFileCount; + // self-optimizing.minor.trigger.interval private int minorLeastInterval; + // self-optimizing.major.trigger.duplicate-ratio private double majorDuplicateRatio; + // self-optimizing.full.trigger.interval private int fullTriggerInterval; + // self-optimizing.full.rewrite-all-files private boolean fullRewriteAllFiles; + // base.file-index.hash-bucket private int baseHashBucket; + // base.refresh-interval private long baseRefreshInterval; + // base.hive.refresh-interval private long hiveRefreshInterval; + // self-optimizing.min-plan-interval private long minPlanInterval; @@ -78,15 +103,6 @@ public OptimizingConfig setEnabled(boolean enabled) { return this; } - public int getMaxCommitRetryCount() { - return maxCommitRetryCount; - } - - public OptimizingConfig setMaxCommitRetryCount(int maxCommitRetryCount) { - this.maxCommitRetryCount = maxCommitRetryCount; - return this; - } - public double getTargetQuota() { return targetQuota; } @@ -271,6 +287,7 @@ public boolean equals(Object o) { && maxFileCount == that.maxFileCount && openFileCost == that.openFileCost && fragmentRatio == that.fragmentRatio + && Double.compare(minTargetSizeRatio, that.minTargetSizeRatio) == 0 && minorLeastFileCount == that.minorLeastFileCount && minorLeastInterval == that.minorLeastInterval && Double.compare(that.majorDuplicateRatio, majorDuplicateRatio) == 0 @@ -279,7 +296,7 @@ public boolean equals(Object o) { && baseHashBucket == that.baseHashBucket && baseRefreshInterval == that.baseRefreshInterval && hiveRefreshInterval == that.hiveRefreshInterval - && Objects.equal(optimizerGroup, that.optimizerGroup); + && com.google.common.base.Objects.equal(optimizerGroup, that.optimizerGroup); } @Override @@ -295,6 +312,7 @@ public int hashCode() { maxFileCount, openFileCost, fragmentRatio, + minTargetSizeRatio, minorLeastFileCount, minorLeastInterval, majorDuplicateRatio, @@ -329,7 +347,102 @@ public String toString() { .toString(); } - public long getRefreshMinInterval() { - return Math.min(baseRefreshInterval, hiveRefreshInterval); + public static OptimizingConfig parse(Map properties) { + return new OptimizingConfig() + .setEnabled( + CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_SELF_OPTIMIZING, + TableProperties.ENABLE_SELF_OPTIMIZING_DEFAULT)) + .setMaxExecuteRetryCount( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_EXECUTE_RETRY_NUMBER, + TableProperties.SELF_OPTIMIZING_EXECUTE_RETRY_NUMBER_DEFAULT)) + .setOptimizerGroup( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.SELF_OPTIMIZING_GROUP, + TableProperties.SELF_OPTIMIZING_GROUP_DEFAULT)) + .setFragmentRatio( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_FRAGMENT_RATIO, + TableProperties.SELF_OPTIMIZING_FRAGMENT_RATIO_DEFAULT)) + .setMinTargetSizeRatio( + CompatiblePropertyUtil.propertyAsDouble( + properties, + TableProperties.SELF_OPTIMIZING_MIN_TARGET_SIZE_RATIO, + TableProperties.SELF_OPTIMIZING_MIN_TARGET_SIZE_RATIO_DEFAULT)) + .setMaxFileCount( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_MAX_FILE_CNT, + TableProperties.SELF_OPTIMIZING_MAX_FILE_CNT_DEFAULT)) + .setOpenFileCost( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.SPLIT_OPEN_FILE_COST, + TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)) + .setTargetSize( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.SELF_OPTIMIZING_TARGET_SIZE, + TableProperties.SELF_OPTIMIZING_TARGET_SIZE_DEFAULT)) + .setMaxTaskSize( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.SELF_OPTIMIZING_MAX_TASK_SIZE, + TableProperties.SELF_OPTIMIZING_MAX_TASK_SIZE_DEFAULT)) + .setTargetQuota( + CompatiblePropertyUtil.propertyAsDouble( + properties, + TableProperties.SELF_OPTIMIZING_QUOTA, + TableProperties.SELF_OPTIMIZING_QUOTA_DEFAULT)) + .setMinorLeastFileCount( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT, + TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT)) + .setMinorLeastInterval( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, + TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT)) + .setMajorDuplicateRatio( + CompatiblePropertyUtil.propertyAsDouble( + properties, + TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO, + TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT)) + .setFullTriggerInterval( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, + TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT)) + .setFullRewriteAllFiles( + CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, + TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES_DEFAULT)) + .setBaseHashBucket( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.BASE_FILE_INDEX_HASH_BUCKET, + TableProperties.BASE_FILE_INDEX_HASH_BUCKET_DEFAULT)) + .setBaseRefreshInterval( + PropertyUtil.propertyAsLong( + properties, + TableProperties.BASE_REFRESH_INTERVAL, + TableProperties.BASE_REFRESH_INTERVAL_DEFAULT)) + .setHiveRefreshInterval( + PropertyUtil.propertyAsLong( + properties, + HiveTableProperties.REFRESH_HIVE_INTERVAL, + HiveTableProperties.REFRESH_HIVE_INTERVAL_DEFAULT)) + .setMinPlanInterval( + PropertyUtil.propertyAsLong( + properties, + TableProperties.SELF_OPTIMIZING_MIN_PLAN_INTERVAL, + TableProperties.SELF_OPTIMIZING_MIN_PLAN_INTERVAL_DEFAULT)); } } diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/StructuredOptionsSplitter.java b/ams/api/src/main/java/com/netease/arctic/api/config/StructuredOptionsSplitter.java new file mode 100644 index 0000000000..e8eb0414b0 --- /dev/null +++ b/ams/api/src/main/java/com/netease/arctic/api/config/StructuredOptionsSplitter.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netease.arctic.api.config; + +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** Helper class for splitting a string on a given delimiter with quoting logic. */ +public class StructuredOptionsSplitter { + + private StructuredOptionsSplitter() {} + + /** + * Splits the given string on the given delimiter. It supports quoting parts of the string with + * either single (') or double quotes ("). Quotes can be escaped by doubling the quotes. + * + *

Examples: + * + *

+ * + *

For more examples check the tests. + * + * @param string a string to split + * @param delimiter delimiter to split on + * @return a list of splits + */ + public static List splitEscaped(String string, char delimiter) { + List tokens = tokenize(Preconditions.checkNotNull(string), delimiter); + return processTokens(tokens); + } + + /** + * Escapes the given string with single quotes, if the input string contains a double quote or any + * of the given {@code charsToEscape}. Any single quotes in the input string will be escaped by + * doubling. + * + *

Given that the escapeChar is (;) + * + *

Examples: + * + *

+ * + * @param string a string which needs to be escaped + * @param charsToEscape escape chars for the escape conditions + * @return escaped string by single quote + */ + public static String escapeWithSingleQuote(String string, String... charsToEscape) { + boolean escape = + Arrays.stream(charsToEscape).anyMatch(string::contains) + || string.contains("\"") + || string.contains("'"); + + if (escape) { + return "'" + string.replaceAll("'", "''") + "'"; + } + + return string; + } + + private static List processTokens(List tokens) { + final List splits = new ArrayList<>(); + for (int i = 0; i < tokens.size(); i++) { + Token token = tokens.get(i); + switch (token.getTokenType()) { + case DOUBLE_QUOTED: + case SINGLE_QUOTED: + if (i + 1 < tokens.size() && tokens.get(i + 1).getTokenType() != TokenType.DELIMITER) { + int illegalPosition = tokens.get(i + 1).getPosition() - 1; + throw new IllegalArgumentException( + "Could not split string. Illegal quoting at position: " + illegalPosition); + } + splits.add(token.getString()); + break; + case UNQUOTED: + splits.add(token.getString()); + break; + case DELIMITER: + if (i + 1 < tokens.size() && tokens.get(i + 1).getTokenType() == TokenType.DELIMITER) { + splits.add(""); + } + break; + } + } + + return splits; + } + + private static List tokenize(String string, char delimiter) { + final List tokens = new ArrayList<>(); + final StringBuilder builder = new StringBuilder(); + for (int cursor = 0; cursor < string.length(); ) { + final char c = string.charAt(cursor); + + int nextChar = cursor + 1; + if (c == '\'') { + nextChar = consumeInQuotes(string, '\'', cursor, builder); + tokens.add(new Token(TokenType.SINGLE_QUOTED, builder.toString(), cursor)); + } else if (c == '"') { + nextChar = consumeInQuotes(string, '"', cursor, builder); + tokens.add(new Token(TokenType.DOUBLE_QUOTED, builder.toString(), cursor)); + } else if (c == delimiter) { + tokens.add(new Token(TokenType.DELIMITER, String.valueOf(c), cursor)); + } else if (!Character.isWhitespace(c)) { + nextChar = consumeUnquoted(string, delimiter, cursor, builder); + tokens.add(new Token(TokenType.UNQUOTED, builder.toString().trim(), cursor)); + } + builder.setLength(0); + cursor = nextChar; + } + + return tokens; + } + + private static int consumeInQuotes(String string, char quote, int cursor, StringBuilder builder) { + for (int i = cursor + 1; i < string.length(); i++) { + char c = string.charAt(i); + if (c == quote) { + if (i + 1 < string.length() && string.charAt(i + 1) == quote) { + builder.append(c); + i += 1; + } else { + return i + 1; + } + } else { + builder.append(c); + } + } + + throw new IllegalArgumentException("Could not split string. Quoting was not closed properly."); + } + + private static int consumeUnquoted( + String string, char delimiter, int cursor, StringBuilder builder) { + int i; + for (i = cursor; i < string.length(); i++) { + char c = string.charAt(i); + if (c == delimiter) { + return i; + } + + builder.append(c); + } + + return i; + } + + private enum TokenType { + DOUBLE_QUOTED, + SINGLE_QUOTED, + UNQUOTED, + DELIMITER + } + + private static class Token { + private final TokenType tokenType; + private final String string; + private final int position; + + private Token(TokenType tokenType, String string, int position) { + this.tokenType = tokenType; + this.string = string; + this.position = position; + } + + public TokenType getTokenType() { + return tokenType; + } + + public String getString() { + return string; + } + + public int getPosition() { + return position; + } + } +} diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/TableConfiguration.java b/ams/api/src/main/java/com/netease/arctic/api/config/TableConfiguration.java index e8b822ae47..369f1df3e5 100644 --- a/ams/api/src/main/java/com/netease/arctic/api/config/TableConfiguration.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/TableConfiguration.java @@ -19,67 +19,29 @@ package com.netease.arctic.api.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.netease.arctic.api.Action; +import com.netease.arctic.table.TableProperties; +import com.netease.arctic.utils.CompatiblePropertyUtil; import org.apache.iceberg.relocated.com.google.common.base.Objects; -import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; -/** - * Configuration for a table, containing {@link OptimizingConfig}, {@link DataExpirationConfig}, and - * {@link TagConfiguration}. - */ @JsonIgnoreProperties(ignoreUnknown = true) public class TableConfiguration { - - // The maximum retry count for executing process. - private int maxExecuteRetryCount; - // Whether to expire snapshots. - private boolean expireOperationEnabled; - // The time to live for snapshots. + private boolean expireSnapshotEnabled; private long snapshotTTLMinutes; - // The time to live for change store data. private long changeDataTTLMinutes; - // Whether to clean orphaned files. private boolean cleanOrphanEnabled; - // The time to live for orphaned files. private long orphanExistingMinutes; - // Whether to delete dangling delete files. private boolean deleteDanglingDeleteFilesEnabled; - // The optimizing configuration. private OptimizingConfig optimizingConfig; - // The data expiration configuration. private DataExpirationConfig expiringDataConfig; - // The tag configuration. private TagConfiguration tagConfiguration; public TableConfiguration() {} - public Map getActionMinIntervals(Set actions) { - Map minIntervals = new HashMap<>(); - if (actions.contains(Action.REFRESH_METADATA)) { - minIntervals.put(Action.REFRESH_METADATA, optimizingConfig.getRefreshMinInterval()); - } else if (actions.contains(Action.EXPIRE_DATA) && isExpireOperationEnabled()) { - minIntervals.put(Action.EXPIRE_DATA, getSnapshotTTLMinutes() * 60 * 1000); - } else if (actions.contains(Action.DELETE_ORPHAN_FILES)) { - minIntervals.put(Action.DELETE_ORPHAN_FILES, getOrphanExistingMinutes() * 60 * 1000); - } - return minIntervals; - } - - /** - * Get the maximum retry count for executing process. - * - * @return the maximum retry count - */ - public int getMaxExecuteRetryCount() { - return maxExecuteRetryCount; - } - - public boolean isExpireOperationEnabled() { - return expireOperationEnabled; + public boolean isExpireSnapshotEnabled() { + return expireSnapshotEnabled; } public long getSnapshotTTLMinutes() { @@ -102,18 +64,13 @@ public OptimizingConfig getOptimizingConfig() { return optimizingConfig; } - public TableConfiguration setMaxExecuteRetryCount(int maxExecuteRetryCount) { - this.maxExecuteRetryCount = maxExecuteRetryCount; - return this; - } - public TableConfiguration setOptimizingConfig(OptimizingConfig optimizingConfig) { this.optimizingConfig = optimizingConfig; return this; } - public TableConfiguration setExpireOperationEnabled(boolean expireOperationEnabled) { - this.expireOperationEnabled = expireOperationEnabled; + public TableConfiguration setExpireSnapshotEnabled(boolean expireSnapshotEnabled) { + this.expireSnapshotEnabled = expireSnapshotEnabled; return this; } @@ -170,7 +127,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TableConfiguration that = (TableConfiguration) o; - return expireOperationEnabled == that.expireOperationEnabled + return expireSnapshotEnabled == that.expireSnapshotEnabled && snapshotTTLMinutes == that.snapshotTTLMinutes && changeDataTTLMinutes == that.changeDataTTLMinutes && cleanOrphanEnabled == that.cleanOrphanEnabled @@ -184,7 +141,7 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hashCode( - expireOperationEnabled, + expireSnapshotEnabled, snapshotTTLMinutes, changeDataTTLMinutes, cleanOrphanEnabled, @@ -194,4 +151,47 @@ public int hashCode() { expiringDataConfig, tagConfiguration); } + + public static TableConfiguration parseConfig(Map properties) { + boolean gcEnabled = + CompatiblePropertyUtil.propertyAsBoolean( + properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); + return new TableConfiguration() + .setExpireSnapshotEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_TABLE_EXPIRE, + TableProperties.ENABLE_TABLE_EXPIRE_DEFAULT)) + .setSnapshotTTLMinutes( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.BASE_SNAPSHOT_KEEP_MINUTES, + TableProperties.BASE_SNAPSHOT_KEEP_MINUTES_DEFAULT)) + .setChangeDataTTLMinutes( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.CHANGE_DATA_TTL, + TableProperties.CHANGE_DATA_TTL_DEFAULT)) + .setCleanOrphanEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_ORPHAN_CLEAN, + TableProperties.ENABLE_ORPHAN_CLEAN_DEFAULT)) + .setOrphanExistingMinutes( + CompatiblePropertyUtil.propertyAsLong( + properties, + TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME, + TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT)) + .setDeleteDanglingDeleteFilesEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_DANGLING_DELETE_FILES_CLEAN, + TableProperties.ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT)) + .setOptimizingConfig(OptimizingConfig.parse(properties)) + .setExpiringDataConfig(DataExpirationConfig.parse(properties)) + .setTagConfiguration(TagConfiguration.parse(properties)); + } } diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/TagConfiguration.java b/ams/api/src/main/java/com/netease/arctic/api/config/TagConfiguration.java index 489a79770a..4e5894425b 100644 --- a/ams/api/src/main/java/com/netease/arctic/api/config/TagConfiguration.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/TagConfiguration.java @@ -19,19 +19,24 @@ package com.netease.arctic.api.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.netease.arctic.table.TableProperties; +import com.netease.arctic.utils.CompatiblePropertyUtil; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Objects; +import java.time.Duration; import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Locale; +import java.util.Map; /** Configuration for auto creating tags. */ @JsonIgnoreProperties(ignoreUnknown = true) public class TagConfiguration { // tag.auto-create.enabled private boolean autoCreateTag = false; - // tag.auto-create.daily.tag-format + // tag.auto-create.tag-format private String tagFormat; // tag.auto-create.trigger.period private Period triggerPeriod; @@ -39,15 +44,32 @@ public class TagConfiguration { private int triggerOffsetMinutes; // tag.auto-create.trigger.max-delay.minutes private int maxDelayMinutes; + // tag.auto-create.max-age-ms + private long tagMaxAgeMs; /** The interval for periodically triggering creating tags */ public enum Period { DAILY("daily") { @Override - public long getTagTriggerTime(LocalDateTime checkTime, int triggerOffsetMinutes) { - LocalTime offsetTime = LocalTime.ofSecondOfDay(triggerOffsetMinutes * 60L); - LocalDateTime triggerTime = LocalDateTime.of(checkTime.toLocalDate(), offsetTime); - return triggerTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + protected Duration periodDuration() { + return Duration.ofDays(1); + } + + @Override + public LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes) { + return checkTime.minusMinutes(triggerOffsetMinutes).truncatedTo(ChronoUnit.DAYS); + } + }, + + HOURLY("hourly") { + @Override + protected Duration periodDuration() { + return Duration.ofHours(1); + } + + @Override + public LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes) { + return checkTime.minusMinutes(triggerOffsetMinutes).truncatedTo(ChronoUnit.HOURS); } }; @@ -61,14 +83,73 @@ public String propertyName() { return propertyName; } + protected abstract Duration periodDuration(); + /** - * Obtain the trigger time for creating a tag, which is the idea time of the last tag before the + * Obtain the tag time for creating a tag, which is the ideal time of the last tag before the * check time. * *

For example, when creating a daily tag, the check time is 2022-08-08 11:00:00 and the - * offset is set to be 5 min, the idea trigger time is 2022-08-08 00:05:00. + * offset is set to be 5 min, the idea tag time is 2022-08-08 00:00:00. + * + *

For example, when creating a daily tag, the offset is set to be 30 min, if the check time + * is 2022-08-08 02:00:00, the ideal tag time is 2022-08-08 00:00:00; if the check time is + * 2022-08-09 00:20:00 (before 00:30 of the next day), the ideal tag time is still 2022-08-08 + * 00:00:00. */ - public abstract long getTagTriggerTime(LocalDateTime checkTime, int triggerOffsetMinutes); + public abstract LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes); + + public String generateTagName(LocalDateTime tagTime, String tagFormat) { + return tagTime.minus(periodDuration()).format(DateTimeFormatter.ofPattern(tagFormat)); + } + } + + public static TagConfiguration parse(Map tableProperties) { + TagConfiguration tagConfig = new TagConfiguration(); + tagConfig.setAutoCreateTag( + CompatiblePropertyUtil.propertyAsBoolean( + tableProperties, + TableProperties.ENABLE_AUTO_CREATE_TAG, + TableProperties.ENABLE_AUTO_CREATE_TAG_DEFAULT)); + tagConfig.setTriggerPeriod( + Period.valueOf( + CompatiblePropertyUtil.propertyAsString( + tableProperties, + TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD, + TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD_DEFAULT) + .toUpperCase(Locale.ROOT))); + + String defaultFormat; + switch (tagConfig.getTriggerPeriod()) { + case DAILY: + defaultFormat = TableProperties.AUTO_CREATE_TAG_FORMAT_DAILY_DEFAULT; + break; + case HOURLY: + defaultFormat = TableProperties.AUTO_CREATE_TAG_FORMAT_HOURLY_DEFAULT; + break; + default: + throw new IllegalArgumentException( + "Unsupported trigger period: " + tagConfig.getTriggerPeriod()); + } + tagConfig.setTagFormat( + CompatiblePropertyUtil.propertyAsString( + tableProperties, TableProperties.AUTO_CREATE_TAG_FORMAT, defaultFormat)); + tagConfig.setTriggerOffsetMinutes( + CompatiblePropertyUtil.propertyAsInt( + tableProperties, + TableProperties.AUTO_CREATE_TAG_TRIGGER_OFFSET_MINUTES, + TableProperties.AUTO_CREATE_TAG_TRIGGER_OFFSET_MINUTES_DEFAULT)); + tagConfig.setMaxDelayMinutes( + CompatiblePropertyUtil.propertyAsInt( + tableProperties, + TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES, + TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT)); + tagConfig.setTagMaxAgeMs( + CompatiblePropertyUtil.propertyAsLong( + tableProperties, + TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS, + TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT)); + return tagConfig; } public boolean isAutoCreateTag() { @@ -111,6 +192,14 @@ public void setMaxDelayMinutes(int maxDelayMinutes) { this.maxDelayMinutes = maxDelayMinutes; } + public long getTagMaxAgeMs() { + return tagMaxAgeMs; + } + + public void setTagMaxAgeMs(long tagMaxAgeMs) { + this.tagMaxAgeMs = tagMaxAgeMs; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -119,14 +208,20 @@ public boolean equals(Object o) { return autoCreateTag == that.autoCreateTag && triggerOffsetMinutes == that.triggerOffsetMinutes && maxDelayMinutes == that.maxDelayMinutes - && Objects.equal(tagFormat, that.tagFormat) - && triggerPeriod == that.triggerPeriod; + && com.google.common.base.Objects.equal(tagFormat, that.tagFormat) + && triggerPeriod == that.triggerPeriod + && tagMaxAgeMs == that.tagMaxAgeMs; } @Override public int hashCode() { return Objects.hashCode( - autoCreateTag, tagFormat, triggerPeriod, triggerOffsetMinutes, maxDelayMinutes); + autoCreateTag, + tagFormat, + triggerPeriod, + triggerOffsetMinutes, + maxDelayMinutes, + tagMaxAgeMs); } @Override @@ -137,6 +232,7 @@ public String toString() { .add("triggerPeriod", triggerPeriod) .add("triggerOffsetMinutes", triggerOffsetMinutes) .add("maxDelayMinutes", maxDelayMinutes) + .add("tagMaxAgeMs", tagMaxAgeMs) .toString(); } } diff --git a/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java b/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java index 85dde96c1a..0a50b4be91 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java +++ b/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java @@ -18,8 +18,8 @@ package com.netease.arctic.server; -import com.netease.arctic.server.utils.ConfigOption; -import com.netease.arctic.server.utils.ConfigOptions; +import com.netease.arctic.api.config.ConfigOption; +import com.netease.arctic.api.config.ConfigOptions; import java.time.Duration; diff --git a/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceContainer.java b/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceContainer.java index bc7822ae9b..d01956d99e 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceContainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceContainer.java @@ -24,6 +24,9 @@ import com.netease.arctic.api.ArcticTableMetastore; import com.netease.arctic.api.OptimizerProperties; import com.netease.arctic.api.OptimizingService; +import com.netease.arctic.api.config.ConfigHelpers; +import com.netease.arctic.api.config.ConfigOption; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.dashboard.DashboardServer; import com.netease.arctic.server.dashboard.response.ErrorResponse; import com.netease.arctic.server.dashboard.utils.AmsUtil; @@ -40,9 +43,6 @@ import com.netease.arctic.server.table.TableService; import com.netease.arctic.server.table.executor.AsyncTableExecutors; import com.netease.arctic.server.terminal.TerminalManager; -import com.netease.arctic.server.utils.ConfigOption; -import com.netease.arctic.server.utils.ConfigurationUtil; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.server.utils.ThriftServiceProxy; import com.netease.arctic.utils.JacksonUtil; import io.javalin.Javalin; @@ -412,7 +412,7 @@ private void initServiceConfig(Map envConfig) throws IOException private Map initEnvConfig() { LOG.info("initializing system env configuration..."); String prefix = ArcticManagementConf.SYSTEM_CONFIG.toUpperCase(); - return ConfigurationUtil.convertConfigurationKeys(prefix, System.getenv()); + return ConfigHelpers.convertConfigurationKeys(prefix, System.getenv()); } private void validateConfig(Map systemConfig) { diff --git a/ams/server/src/main/java/com/netease/arctic/server/DefaultOptimizingService.java b/ams/server/src/main/java/com/netease/arctic/server/DefaultOptimizingService.java index c6a1ee3149..c70675d169 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/DefaultOptimizingService.java +++ b/ams/server/src/main/java/com/netease/arctic/server/DefaultOptimizingService.java @@ -26,6 +26,9 @@ import com.netease.arctic.api.OptimizingTask; import com.netease.arctic.api.OptimizingTaskId; import com.netease.arctic.api.OptimizingTaskResult; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.Configurations; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.api.resource.Resource; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.properties.CatalogMetaProperties; @@ -45,12 +48,9 @@ import com.netease.arctic.server.resource.QuotaProvider; import com.netease.arctic.server.table.DefaultTableService; import com.netease.arctic.server.table.RuntimeHandlerChain; -import com.netease.arctic.server.table.ServerTableIdentifier; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableRuntimeMeta; import com.netease.arctic.server.table.TableService; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableProperties; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; diff --git a/ams/server/src/main/java/com/netease/arctic/server/HighAvailabilityContainer.java b/ams/server/src/main/java/com/netease/arctic/server/HighAvailabilityContainer.java index 9ca9874924..723adffa57 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/HighAvailabilityContainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/HighAvailabilityContainer.java @@ -18,9 +18,9 @@ package com.netease.arctic.server; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.client.AmsServerInfo; import com.netease.arctic.properties.AmsHAProperties; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.utils.JacksonUtil; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; diff --git a/ams/server/src/main/java/com/netease/arctic/server/RestCatalogService.java b/ams/server/src/main/java/com/netease/arctic/server/RestCatalogService.java index e9cee3a248..1dd7079a78 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/RestCatalogService.java +++ b/ams/server/src/main/java/com/netease/arctic/server/RestCatalogService.java @@ -32,6 +32,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.netease.arctic.TableFormat; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.events.IcebergReportEvent; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.server.catalog.InternalCatalog; @@ -39,7 +40,6 @@ import com.netease.arctic.server.exception.ObjectNotExistsException; import com.netease.arctic.server.manager.EventsManager; import com.netease.arctic.server.persistence.PersistentBase; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableService; import com.netease.arctic.server.table.internal.InternalTableCreator; import com.netease.arctic.server.table.internal.InternalTableHandler; diff --git a/ams/server/src/main/java/com/netease/arctic/server/TableManagementService.java b/ams/server/src/main/java/com/netease/arctic/server/TableManagementService.java index 8d2454bbf8..ea4119dee4 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/TableManagementService.java +++ b/ams/server/src/main/java/com/netease/arctic/server/TableManagementService.java @@ -25,10 +25,10 @@ import com.netease.arctic.api.CatalogMeta; import com.netease.arctic.api.NoSuchObjectException; import com.netease.arctic.api.OperationConflictException; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableCommitMeta; import com.netease.arctic.api.TableIdentifier; import com.netease.arctic.api.TableMeta; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import com.netease.arctic.server.table.TableService; import com.netease.arctic.server.utils.InternalTableUtil; diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/CatalogBuilder.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/CatalogBuilder.java index cc326377d1..013a84ea80 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/CatalogBuilder.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/CatalogBuilder.java @@ -26,9 +26,9 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.server.ArcticManagementConf; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.utils.ArcticCatalogUtil; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java index c58abe330e..1ab93928c9 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java @@ -24,9 +24,9 @@ import com.netease.arctic.TableIDWithFormat; import com.netease.arctic.UnifiedCatalog; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.table.TableMetaStore; import com.netease.arctic.utils.ArcticCatalogUtil; import org.apache.iceberg.relocated.com.google.common.collect.Maps; diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalCatalog.java index 3157366cf3..25dc956128 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalCatalog.java @@ -21,6 +21,7 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.TableIDWithFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableIdentifier; import com.netease.arctic.server.exception.AlreadyExistsException; import com.netease.arctic.server.exception.IllegalMetadataException; @@ -28,7 +29,6 @@ import com.netease.arctic.server.persistence.mapper.CatalogMetaMapper; import com.netease.arctic.server.persistence.mapper.TableBlockerMapper; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import com.netease.arctic.server.table.internal.InternalTableCreator; import com.netease.arctic.server.table.internal.InternalTableHandler; diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalIcebergCatalogImpl.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalIcebergCatalogImpl.java index 5777fee7c7..75ac1bbe58 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalIcebergCatalogImpl.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalIcebergCatalogImpl.java @@ -21,6 +21,7 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.formats.iceberg.IcebergTable; import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.server.ArcticManagementConf; @@ -31,7 +32,6 @@ import com.netease.arctic.server.table.internal.InternalIcebergHandler; import com.netease.arctic.server.table.internal.InternalTableCreator; import com.netease.arctic.server.table.internal.InternalTableHandler; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.server.utils.InternalTableUtil; import com.netease.arctic.utils.ArcticCatalogUtil; import org.apache.iceberg.BaseTable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalMixedCatalogImpl.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalMixedCatalogImpl.java index 022ecda7a3..a44195af97 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalMixedCatalogImpl.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/InternalMixedCatalogImpl.java @@ -23,6 +23,7 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.formats.mixed.MixedTable; import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.mixed.InternalMixedIcebergCatalog; @@ -32,7 +33,6 @@ import com.netease.arctic.server.table.internal.InternalMixedIcebergHandler; import com.netease.arctic.server.table.internal.InternalTableCreator; import com.netease.arctic.server.table.internal.InternalTableHandler; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.server.utils.InternalTableUtil; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.BasicKeyedTable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/DashboardServer.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/DashboardServer.java index af01730d4d..cf7a91030c 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/DashboardServer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/DashboardServer.java @@ -24,6 +24,7 @@ import static io.javalin.apibuilder.ApiBuilder.post; import static io.javalin.apibuilder.ApiBuilder.put; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.DefaultOptimizingService; import com.netease.arctic.server.RestCatalogService; import com.netease.arctic.server.dashboard.controller.CatalogController; @@ -41,7 +42,6 @@ import com.netease.arctic.server.exception.SignatureCheckException; import com.netease.arctic.server.table.TableService; import com.netease.arctic.server.terminal.TerminalManager; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.utils.JacksonUtil; import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.ContentType; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/ServerTableDescriptor.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/ServerTableDescriptor.java index 9993e96919..fae91f1ad8 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/ServerTableDescriptor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/ServerTableDescriptor.java @@ -21,6 +21,7 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; import com.netease.arctic.api.TableIdentifier; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.catalog.ServerCatalog; import com.netease.arctic.server.dashboard.model.AmoroSnapshotsOfTable; import com.netease.arctic.server.dashboard.model.DDLInfo; @@ -33,7 +34,6 @@ import com.netease.arctic.server.dashboard.model.TagOrBranchInfo; import com.netease.arctic.server.persistence.PersistentBase; import com.netease.arctic.server.table.TableService; -import com.netease.arctic.server.utils.Configurations; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.ThreadPools; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/LoginController.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/LoginController.java index 5257cf8354..e8f6a7821e 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/LoginController.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/LoginController.java @@ -18,9 +18,9 @@ package com.netease.arctic.server.dashboard.controller; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.dashboard.response.OkResponse; -import com.netease.arctic.server.utils.Configurations; import io.javalin.http.Context; import java.io.Serializable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/OptimizerController.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/OptimizerController.java index 8c3706cba9..aac56fa0bb 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/OptimizerController.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/OptimizerController.java @@ -19,6 +19,7 @@ package com.netease.arctic.server.dashboard.controller; import com.google.common.base.Preconditions; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.resource.Resource; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.api.resource.ResourceType; @@ -32,7 +33,6 @@ import com.netease.arctic.server.resource.ContainerMetadata; import com.netease.arctic.server.resource.OptimizerInstance; import com.netease.arctic.server.resource.ResourceContainers; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableService; import io.javalin.http.Context; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/SettingController.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/SettingController.java index ebcc1bcb17..ba58c8a45a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/SettingController.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/SettingController.java @@ -18,13 +18,13 @@ package com.netease.arctic.server.dashboard.controller; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.dashboard.response.OkResponse; import com.netease.arctic.server.resource.ContainerMetadata; import com.netease.arctic.server.resource.OptimizerManager; import com.netease.arctic.server.resource.ResourceContainers; -import com.netease.arctic.server.utils.Configurations; import io.javalin.http.Context; import org.glassfish.jersey.internal.guava.Sets; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/TableController.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/TableController.java index 2cc068afe1..d0ad2b3ac5 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/TableController.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/TableController.java @@ -23,14 +23,16 @@ import com.netease.arctic.Constants; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.catalog.CatalogLoader; import com.netease.arctic.hive.CachedHiveClientPool; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.catalog.ArcticHiveCatalog; import com.netease.arctic.hive.utils.HiveTableUtil; import com.netease.arctic.hive.utils.UpgradeHiveTableUtil; import com.netease.arctic.properties.CatalogMetaProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.server.catalog.ServerCatalog; import com.netease.arctic.server.dashboard.ServerTableDescriptor; import com.netease.arctic.server.dashboard.ServerTableProperties; @@ -54,10 +56,8 @@ import com.netease.arctic.server.dashboard.response.PageResult; import com.netease.arctic.server.dashboard.utils.AmsUtil; import com.netease.arctic.server.dashboard.utils.CommonUtil; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableService; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.table.TableMetaStore; import com.netease.arctic.table.TableProperties; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/model/TableOptimizingInfo.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/model/TableOptimizingInfo.java index 3b60935796..d8b3682386 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/model/TableOptimizingInfo.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/model/TableOptimizingInfo.java @@ -18,8 +18,8 @@ package com.netease.arctic.server.dashboard.model; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.optimizing.OptimizingStatus; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.table.TableProperties; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/utils/AmsUtil.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/utils/AmsUtil.java index fad3aa0033..33fe0e78d6 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/dashboard/utils/AmsUtil.java +++ b/ams/server/src/main/java/com/netease/arctic/server/dashboard/utils/AmsUtil.java @@ -18,16 +18,10 @@ package com.netease.arctic.server.dashboard.utils; -import static com.netease.arctic.server.ArcticManagementConf.HA_CLUSTER_NAME; -import static com.netease.arctic.server.ArcticManagementConf.HA_ENABLE; -import static com.netease.arctic.server.ArcticManagementConf.HA_ZOOKEEPER_ADDRESS; -import static com.netease.arctic.server.ArcticManagementConf.OPTIMIZING_SERVICE_THRIFT_BIND_PORT; -import static com.netease.arctic.server.ArcticManagementConf.SERVER_EXPOSE_HOST; -import static com.netease.arctic.server.ArcticManagementConf.TABLE_SERVICE_THRIFT_BIND_PORT; - import com.netease.arctic.Constants; import com.netease.arctic.api.TableIdentifier; -import com.netease.arctic.server.utils.Configurations; +import com.netease.arctic.api.config.Configurations; +import com.netease.arctic.server.ArcticManagementConf; import java.io.File; import java.io.PrintWriter; @@ -120,7 +114,8 @@ public static String getStackTrace(Throwable throwable) { public static InetAddress lookForBindHost(String prefix) { if (prefix.startsWith("0")) { - throw new RuntimeException("config " + SERVER_EXPOSE_HOST.key() + " can't start with 0"); + throw new RuntimeException( + "config " + ArcticManagementConf.SERVER_EXPOSE_HOST.key() + " can't start with 0"); } try { Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); @@ -141,22 +136,22 @@ public static InetAddress lookForBindHost(String prefix) { } public static String getAMSThriftAddress(Configurations conf, String serviceName) { - if (conf.getBoolean(HA_ENABLE)) { + if (conf.getBoolean(ArcticManagementConf.HA_ENABLE)) { return String.format( ZOOKEEPER_ADDRESS_FORMAT, - conf.getString(HA_ZOOKEEPER_ADDRESS), - conf.getString(HA_CLUSTER_NAME)); + conf.getString(ArcticManagementConf.HA_ZOOKEEPER_ADDRESS), + conf.getString(ArcticManagementConf.HA_CLUSTER_NAME)); } else { if (Constants.THRIFT_TABLE_SERVICE_NAME.equals(serviceName)) { return String.format( THRIFT_ADDRESS_FORMAT, - conf.getString(SERVER_EXPOSE_HOST), - conf.getInteger(TABLE_SERVICE_THRIFT_BIND_PORT)); + conf.getString(ArcticManagementConf.SERVER_EXPOSE_HOST), + conf.getInteger(ArcticManagementConf.TABLE_SERVICE_THRIFT_BIND_PORT)); } else if (Constants.THRIFT_OPTIMIZING_SERVICE_NAME.equals(serviceName)) { return String.format( THRIFT_ADDRESS_FORMAT, - conf.getString(SERVER_EXPOSE_HOST), - conf.getInteger(OPTIMIZING_SERVICE_THRIFT_BIND_PORT)); + conf.getString(ArcticManagementConf.SERVER_EXPOSE_HOST), + conf.getInteger(ArcticManagementConf.OPTIMIZING_SERVICE_THRIFT_BIND_PORT)); } else { throw new IllegalArgumentException(String.format("Unknown service name %s", serviceName)); } diff --git a/ams/server/src/main/java/com/netease/arctic/server/exception/ArcticRuntimeException.java b/ams/server/src/main/java/com/netease/arctic/server/exception/ArcticRuntimeException.java index 88ba36ef32..e0e14ff472 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/exception/ArcticRuntimeException.java +++ b/ams/server/src/main/java/com/netease/arctic/server/exception/ArcticRuntimeException.java @@ -24,8 +24,8 @@ import com.netease.arctic.api.MetaException; import com.netease.arctic.api.NoSuchObjectException; import com.netease.arctic.api.OperationConflictException; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableIdentifier; -import com.netease.arctic.server.table.ServerTableIdentifier; import org.apache.thrift.TException; import java.util.HashMap; diff --git a/ams/server/src/main/java/com/netease/arctic/server/exception/ObjectNotExistsException.java b/ams/server/src/main/java/com/netease/arctic/server/exception/ObjectNotExistsException.java index 4cc83b8dce..56df4f33af 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/exception/ObjectNotExistsException.java +++ b/ams/server/src/main/java/com/netease/arctic/server/exception/ObjectNotExistsException.java @@ -18,8 +18,8 @@ package com.netease.arctic.server.exception; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableIdentifier; -import com.netease.arctic.server.table.ServerTableIdentifier; public class ObjectNotExistsException extends ArcticRuntimeException { public ObjectNotExistsException(String object) { diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingConfig.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingConfig.java deleted file mode 100644 index af23257221..0000000000 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingConfig.java +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.optimizing; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.google.common.base.Objects; -import com.netease.arctic.hive.HiveTableProperties; -import com.netease.arctic.table.TableProperties; -import com.netease.arctic.utils.CompatiblePropertyUtil; -import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; -import org.apache.iceberg.util.PropertyUtil; - -import java.util.Map; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class OptimizingConfig { - - // self-optimizing.enabled - private boolean enabled; - - // self-optimizing.quota - private double targetQuota; - - // self-optimizing.group - private String optimizerGroup; - - // self-optimizing.execute.num-retries - private int maxExecuteRetryCount; - - // self-optimizing.commit.num-retries - private int maxCommitRetryCount; - - // self-optimizing.target-size - private long targetSize; - - // self-optimizing.max-task-size-bytes - private long maxTaskSize; - - // self-optimizing.max-file-count - private int maxFileCount; - - // read.split.open-file-cost - private long openFileCost; - - // self-optimizing.fragment-ratio - private int fragmentRatio; - - // self-optimizing.min-target-size-ratio - private double minTargetSizeRatio; - - // self-optimizing.minor.trigger.file-count - private int minorLeastFileCount; - - // self-optimizing.minor.trigger.interval - private int minorLeastInterval; - - // self-optimizing.major.trigger.duplicate-ratio - private double majorDuplicateRatio; - - // self-optimizing.full.trigger.interval - private int fullTriggerInterval; - - // self-optimizing.full.rewrite-all-files - private boolean fullRewriteAllFiles; - - // base.file-index.hash-bucket - private int baseHashBucket; - - // base.refresh-interval - private long baseRefreshInterval; - - // base.hive.refresh-interval - private long hiveRefreshInterval; - - // self-optimizing.min-plan-interval - private long minPlanInterval; - - public OptimizingConfig() {} - - public boolean isEnabled() { - return enabled; - } - - public OptimizingConfig setEnabled(boolean enabled) { - this.enabled = enabled; - return this; - } - - public double getTargetQuota() { - return targetQuota; - } - - public OptimizingConfig setTargetQuota(double targetQuota) { - this.targetQuota = targetQuota; - return this; - } - - public OptimizingConfig setMinPlanInterval(long minPlanInterval) { - this.minPlanInterval = minPlanInterval; - return this; - } - - public long getMinPlanInterval() { - return minPlanInterval; - } - - public String getOptimizerGroup() { - return optimizerGroup; - } - - public OptimizingConfig setOptimizerGroup(String optimizerGroup) { - this.optimizerGroup = optimizerGroup; - return this; - } - - public int getMaxExecuteRetryCount() { - return maxExecuteRetryCount; - } - - public OptimizingConfig setMaxExecuteRetryCount(int maxExecuteRetryCount) { - this.maxExecuteRetryCount = maxExecuteRetryCount; - return this; - } - - public long getTargetSize() { - return targetSize; - } - - public OptimizingConfig setTargetSize(long targetSize) { - this.targetSize = targetSize; - return this; - } - - public long getMaxTaskSize() { - return maxTaskSize; - } - - public OptimizingConfig setMaxTaskSize(long maxTaskSize) { - this.maxTaskSize = maxTaskSize; - return this; - } - - public int getMaxFileCount() { - return maxFileCount; - } - - public OptimizingConfig setMaxFileCount(int maxFileCount) { - this.maxFileCount = maxFileCount; - return this; - } - - public long getOpenFileCost() { - return openFileCost; - } - - public OptimizingConfig setOpenFileCost(long openFileCost) { - this.openFileCost = openFileCost; - return this; - } - - public int getFragmentRatio() { - return fragmentRatio; - } - - public double getMinTargetSizeRatio() { - return minTargetSizeRatio; - } - - public long maxFragmentSize() { - return targetSize / fragmentRatio; - } - - public long maxDuplicateSize() { - return (long) (maxFragmentSize() * majorDuplicateRatio); - } - - public OptimizingConfig setFragmentRatio(int fragmentRatio) { - this.fragmentRatio = fragmentRatio; - return this; - } - - public OptimizingConfig setMinTargetSizeRatio(double minTargetSizeRatio) { - this.minTargetSizeRatio = minTargetSizeRatio; - return this; - } - - public int getMinorLeastFileCount() { - return minorLeastFileCount; - } - - public OptimizingConfig setMinorLeastFileCount(int minorLeastFileCount) { - this.minorLeastFileCount = minorLeastFileCount; - return this; - } - - public int getMinorLeastInterval() { - return minorLeastInterval; - } - - public OptimizingConfig setMinorLeastInterval(int minorLeastInterval) { - this.minorLeastInterval = minorLeastInterval; - return this; - } - - public double getMajorDuplicateRatio() { - return majorDuplicateRatio; - } - - public OptimizingConfig setMajorDuplicateRatio(double majorDuplicateRatio) { - this.majorDuplicateRatio = majorDuplicateRatio; - return this; - } - - public int getFullTriggerInterval() { - return fullTriggerInterval; - } - - public OptimizingConfig setFullTriggerInterval(int fullTriggerInterval) { - this.fullTriggerInterval = fullTriggerInterval; - return this; - } - - public boolean isFullRewriteAllFiles() { - return fullRewriteAllFiles; - } - - public OptimizingConfig setFullRewriteAllFiles(boolean fullRewriteAllFiles) { - this.fullRewriteAllFiles = fullRewriteAllFiles; - return this; - } - - public int getBaseHashBucket() { - return baseHashBucket; - } - - public OptimizingConfig setBaseHashBucket(int baseHashBucket) { - this.baseHashBucket = baseHashBucket; - return this; - } - - public long getBaseRefreshInterval() { - return baseRefreshInterval; - } - - public OptimizingConfig setBaseRefreshInterval(long baseRefreshInterval) { - this.baseRefreshInterval = baseRefreshInterval; - return this; - } - - public long getHiveRefreshInterval() { - return hiveRefreshInterval; - } - - public OptimizingConfig setHiveRefreshInterval(long hiveRefreshInterval) { - this.hiveRefreshInterval = hiveRefreshInterval; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - OptimizingConfig that = (OptimizingConfig) o; - return enabled == that.enabled - && Double.compare(that.targetQuota, targetQuota) == 0 - && maxExecuteRetryCount == that.maxExecuteRetryCount - && maxCommitRetryCount == that.maxCommitRetryCount - && targetSize == that.targetSize - && maxTaskSize == that.maxTaskSize - && maxFileCount == that.maxFileCount - && openFileCost == that.openFileCost - && fragmentRatio == that.fragmentRatio - && Double.compare(minTargetSizeRatio, that.minTargetSizeRatio) == 0 - && minorLeastFileCount == that.minorLeastFileCount - && minorLeastInterval == that.minorLeastInterval - && Double.compare(that.majorDuplicateRatio, majorDuplicateRatio) == 0 - && fullTriggerInterval == that.fullTriggerInterval - && fullRewriteAllFiles == that.fullRewriteAllFiles - && baseHashBucket == that.baseHashBucket - && baseRefreshInterval == that.baseRefreshInterval - && hiveRefreshInterval == that.hiveRefreshInterval - && Objects.equal(optimizerGroup, that.optimizerGroup); - } - - @Override - public int hashCode() { - return Objects.hashCode( - enabled, - targetQuota, - optimizerGroup, - maxExecuteRetryCount, - maxCommitRetryCount, - targetSize, - maxTaskSize, - maxFileCount, - openFileCost, - fragmentRatio, - minTargetSizeRatio, - minorLeastFileCount, - minorLeastInterval, - majorDuplicateRatio, - fullTriggerInterval, - fullRewriteAllFiles, - baseHashBucket, - baseRefreshInterval, - hiveRefreshInterval); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("enabled", enabled) - .add("targetQuota", targetQuota) - .add("optimizerGroup", optimizerGroup) - .add("maxExecuteRetryCount", maxExecuteRetryCount) - .add("maxCommitRetryCount", maxCommitRetryCount) - .add("targetSize", targetSize) - .add("maxTaskSize", maxTaskSize) - .add("maxFileCount", maxFileCount) - .add("openFileCost", openFileCost) - .add("fragmentRatio", fragmentRatio) - .add("minorLeastFileCount", minorLeastFileCount) - .add("minorLeastInterval", minorLeastInterval) - .add("majorDuplicateRatio", majorDuplicateRatio) - .add("fullTriggerInterval", fullTriggerInterval) - .add("fullRewriteAllFiles", fullRewriteAllFiles) - .add("baseHashBucket", baseHashBucket) - .add("baseRefreshInterval", baseRefreshInterval) - .add("hiveRefreshInterval", hiveRefreshInterval) - .toString(); - } - - public static OptimizingConfig parseOptimizingConfig(Map properties) { - return new OptimizingConfig() - .setEnabled( - CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_SELF_OPTIMIZING, - TableProperties.ENABLE_SELF_OPTIMIZING_DEFAULT)) - .setMaxExecuteRetryCount( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_EXECUTE_RETRY_NUMBER, - TableProperties.SELF_OPTIMIZING_EXECUTE_RETRY_NUMBER_DEFAULT)) - .setOptimizerGroup( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.SELF_OPTIMIZING_GROUP, - TableProperties.SELF_OPTIMIZING_GROUP_DEFAULT)) - .setFragmentRatio( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_FRAGMENT_RATIO, - TableProperties.SELF_OPTIMIZING_FRAGMENT_RATIO_DEFAULT)) - .setMinTargetSizeRatio( - CompatiblePropertyUtil.propertyAsDouble( - properties, - TableProperties.SELF_OPTIMIZING_MIN_TARGET_SIZE_RATIO, - TableProperties.SELF_OPTIMIZING_MIN_TARGET_SIZE_RATIO_DEFAULT)) - .setMaxFileCount( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_MAX_FILE_CNT, - TableProperties.SELF_OPTIMIZING_MAX_FILE_CNT_DEFAULT)) - .setOpenFileCost( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.SPLIT_OPEN_FILE_COST, - TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)) - .setTargetSize( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.SELF_OPTIMIZING_TARGET_SIZE, - TableProperties.SELF_OPTIMIZING_TARGET_SIZE_DEFAULT)) - .setMaxTaskSize( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.SELF_OPTIMIZING_MAX_TASK_SIZE, - TableProperties.SELF_OPTIMIZING_MAX_TASK_SIZE_DEFAULT)) - .setTargetQuota( - CompatiblePropertyUtil.propertyAsDouble( - properties, - TableProperties.SELF_OPTIMIZING_QUOTA, - TableProperties.SELF_OPTIMIZING_QUOTA_DEFAULT)) - .setMinorLeastFileCount( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT)) - .setMinorLeastInterval( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT)) - .setMajorDuplicateRatio( - CompatiblePropertyUtil.propertyAsDouble( - properties, - TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO, - TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT)) - .setFullTriggerInterval( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, - TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT)) - .setFullRewriteAllFiles( - CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, - TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES_DEFAULT)) - .setBaseHashBucket( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.BASE_FILE_INDEX_HASH_BUCKET, - TableProperties.BASE_FILE_INDEX_HASH_BUCKET_DEFAULT)) - .setBaseRefreshInterval( - PropertyUtil.propertyAsLong( - properties, - TableProperties.BASE_REFRESH_INTERVAL, - TableProperties.BASE_REFRESH_INTERVAL_DEFAULT)) - .setHiveRefreshInterval( - PropertyUtil.propertyAsLong( - properties, - HiveTableProperties.REFRESH_HIVE_INTERVAL, - HiveTableProperties.REFRESH_HIVE_INTERVAL_DEFAULT)) - .setMinPlanInterval( - PropertyUtil.propertyAsLong( - properties, - TableProperties.SELF_OPTIMIZING_MIN_PLAN_INTERVAL, - TableProperties.SELF_OPTIMIZING_MIN_PLAN_INTERVAL_DEFAULT)); - } -} diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java index 12322fcba0..39bfdb54e7 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java @@ -21,6 +21,7 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.api.OptimizerProperties; import com.netease.arctic.api.OptimizingTaskId; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.optimizing.RewriteFilesInput; import com.netease.arctic.server.ArcticServiceConstants; @@ -33,7 +34,6 @@ import com.netease.arctic.server.persistence.mapper.OptimizingMapper; import com.netease.arctic.server.resource.OptimizerInstance; import com.netease.arctic.server.resource.QuotaProvider; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableRuntimeMeta; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/SchedulingPolicy.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/SchedulingPolicy.java index f385f0f299..96efdf954f 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/SchedulingPolicy.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/SchedulingPolicy.java @@ -20,8 +20,8 @@ import com.google.common.collect.Maps; import com.netease.arctic.api.BlockableOperation; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.resource.ResourceGroup; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableRuntime; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/TaskRuntime.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/TaskRuntime.java index eab409f205..2adc9b7b66 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/TaskRuntime.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/TaskRuntime.java @@ -23,6 +23,7 @@ import com.netease.arctic.api.OptimizingTask; import com.netease.arctic.api.OptimizingTaskId; import com.netease.arctic.api.OptimizingTaskResult; +import com.netease.arctic.api.StateField; import com.netease.arctic.optimizing.RewriteFilesInput; import com.netease.arctic.optimizing.RewriteFilesOutput; import com.netease.arctic.server.ArcticServiceConstants; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java index 8d72b67f6f..22227080e8 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java @@ -25,7 +25,6 @@ import com.netease.arctic.api.CommitMetaProducer; import com.netease.arctic.data.FileNameRules; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.hive.utils.HivePartitionUtil; import com.netease.arctic.hive.utils.HiveTableUtil; @@ -33,6 +32,7 @@ import com.netease.arctic.op.SnapshotSummary; import com.netease.arctic.optimizing.OptimizingInputProperties; import com.netease.arctic.optimizing.RewriteFilesOutput; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.server.ArcticServiceConstants; import com.netease.arctic.server.exception.OptimizingCommitException; import com.netease.arctic.server.utils.IcebergTableUtil; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java index f99aee7c29..f50c4898f7 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java @@ -18,7 +18,7 @@ package com.netease.arctic.server.optimizing.maintainer; -import com.netease.arctic.server.table.TagConfiguration; +import com.netease.arctic.api.config.TagConfiguration; import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java index a16e6a5b00..ae9f181e02 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java @@ -21,12 +21,12 @@ import static org.apache.iceberg.relocated.com.google.common.primitives.Longs.min; import com.netease.arctic.api.CommitMetaProducer; +import com.netease.arctic.api.config.DataExpirationConfig; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.io.PathInfo; import com.netease.arctic.io.SupportsFileSystemOperations; import com.netease.arctic.server.ArcticServiceConstants; -import com.netease.arctic.server.table.DataExpirationConfig; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.utils.IcebergTableUtil; import com.netease.arctic.utils.TableFileUtil; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java index e97e1a4226..27158ea9fc 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java @@ -22,10 +22,10 @@ import static org.apache.iceberg.relocated.com.google.common.primitives.Longs.min; import com.netease.arctic.IcebergFileEntry; +import com.netease.arctic.api.config.DataExpirationConfig; import com.netease.arctic.data.FileNameRules; import com.netease.arctic.hive.utils.TableTypeUtil; import com.netease.arctic.scan.TableEntriesScan; -import com.netease.arctic.server.table.DataExpirationConfig; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.utils.HiveLocationUtil; import com.netease.arctic.server.utils.IcebergTableUtil; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/AbstractPartitionPlan.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/AbstractPartitionPlan.java index 376e22ec71..d76fc67596 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/AbstractPartitionPlan.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/AbstractPartitionPlan.java @@ -18,9 +18,9 @@ package com.netease.arctic.server.optimizing.plan; +import com.netease.arctic.api.config.OptimizingConfig; import com.netease.arctic.optimizing.OptimizingInputProperties; import com.netease.arctic.optimizing.RewriteFilesInput; -import com.netease.arctic.server.optimizing.OptimizingConfig; import com.netease.arctic.server.optimizing.OptimizingType; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.table.ArcticTable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java index d592a4767a..d5c00ec071 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java @@ -18,7 +18,7 @@ package com.netease.arctic.server.optimizing.plan; -import com.netease.arctic.server.optimizing.OptimizingConfig; +import com.netease.arctic.api.config.OptimizingConfig; import com.netease.arctic.server.optimizing.OptimizingType; import com.netease.arctic.server.table.TableRuntime; import org.apache.iceberg.ContentFile; diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/MixedHivePartitionPlan.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/MixedHivePartitionPlan.java index 24edbfc686..5d55b5a025 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/MixedHivePartitionPlan.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/MixedHivePartitionPlan.java @@ -20,9 +20,9 @@ import com.netease.arctic.data.DataFileType; import com.netease.arctic.data.PrimaryKeyedFile; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.utils.HiveTableUtil; import com.netease.arctic.optimizing.OptimizingInputProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.table.ArcticTable; import org.apache.iceberg.ContentFile; diff --git a/ams/server/src/main/java/com/netease/arctic/server/persistence/SqlSessionFactoryProvider.java b/ams/server/src/main/java/com/netease/arctic/server/persistence/SqlSessionFactoryProvider.java index 195e9dd70b..92b185d570 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/persistence/SqlSessionFactoryProvider.java +++ b/ams/server/src/main/java/com/netease/arctic/server/persistence/SqlSessionFactoryProvider.java @@ -18,6 +18,7 @@ package com.netease.arctic.server.persistence; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.persistence.mapper.ApiTokensMapper; import com.netease.arctic.server.persistence.mapper.CatalogMetaMapper; @@ -27,7 +28,6 @@ import com.netease.arctic.server.persistence.mapper.ResourceMapper; import com.netease.arctic.server.persistence.mapper.TableBlockerMapper; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; -import com.netease.arctic.server.utils.Configurations; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.BaseObjectPoolConfig; diff --git a/ams/server/src/main/java/com/netease/arctic/server/persistence/StatedPersistentBase.java b/ams/server/src/main/java/com/netease/arctic/server/persistence/StatedPersistentBase.java index 1cbd67d4fd..184479837a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/persistence/StatedPersistentBase.java +++ b/ams/server/src/main/java/com/netease/arctic/server/persistence/StatedPersistentBase.java @@ -18,10 +18,9 @@ package com.netease.arctic.server.persistence; +import com.netease.arctic.api.StateField; import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; @@ -120,9 +119,4 @@ void setValue(Field field, Object value) { throw new IllegalStateException(e); } } - - /** TODO Use StateField in API module, this class shall be removed after 0.7.0 */ - @Deprecated - @Retention(RetentionPolicy.RUNTIME) - public @interface StateField {} } diff --git a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/OptimizingMapper.java b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/OptimizingMapper.java index 8c05f891e0..4b2fc40a85 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/OptimizingMapper.java +++ b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/OptimizingMapper.java @@ -18,6 +18,7 @@ package com.netease.arctic.server.persistence.mapper; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.optimizing.RewriteFilesInput; import com.netease.arctic.server.optimizing.MetricsSummary; import com.netease.arctic.server.optimizing.OptimizingProcess; @@ -30,7 +31,6 @@ import com.netease.arctic.server.persistence.converter.Map2StringConverter; import com.netease.arctic.server.persistence.converter.MapLong2StringConverter; import com.netease.arctic.server.persistence.converter.Object2ByteArrayConvert; -import com.netease.arctic.server.table.ServerTableIdentifier; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; diff --git a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableBlockerMapper.java b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableBlockerMapper.java index fe1882e538..cdd05acfe6 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableBlockerMapper.java +++ b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableBlockerMapper.java @@ -18,10 +18,10 @@ package com.netease.arctic.server.persistence.mapper; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.persistence.converter.List2StringConverter; import com.netease.arctic.server.persistence.converter.Long2TsConverter; import com.netease.arctic.server.persistence.converter.Map2StringConverter; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.blocker.TableBlocker; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; diff --git a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableMetaMapper.java b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableMetaMapper.java index d561b209a4..437d882f3a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableMetaMapper.java +++ b/ams/server/src/main/java/com/netease/arctic/server/persistence/mapper/TableMetaMapper.java @@ -18,11 +18,11 @@ package com.netease.arctic.server.persistence.mapper; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.persistence.converter.JsonObjectConverter; import com.netease.arctic.server.persistence.converter.Long2TsConverter; import com.netease.arctic.server.persistence.converter.Map2StringConverter; import com.netease.arctic.server.persistence.converter.MapLong2StringConverter; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableRuntimeMeta; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java b/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java deleted file mode 100644 index e53b2d45e4..0000000000 --- a/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.table; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import com.netease.arctic.server.utils.ConfigurationUtil; -import com.netease.arctic.table.ArcticTable; -import com.netease.arctic.table.TableProperties; -import com.netease.arctic.utils.CompatiblePropertyUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -/** TODO Use DataExpirationConfig class in API module, this class shall be removed after 0.7.0 */ -@Deprecated -@JsonIgnoreProperties(ignoreUnknown = true) -public class DataExpirationConfig { - // data-expire.enabled - private boolean enabled; - // data-expire.field - private String expirationField; - // data-expire.level - @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT) - private ExpireLevel expirationLevel; - // data-expire.retention-time - private long retentionTime; - // data-expire.datetime-string-pattern - @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT) - private String dateTimePattern; - // data-expire.datetime-number-format - @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT) - private String numberDateFormat; - // data-expire.base-on-rule - @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT) - private BaseOnRule baseOnRule; - - @VisibleForTesting - public enum ExpireLevel { - PARTITION, - FILE; - - public static ExpireLevel fromString(String level) { - Preconditions.checkArgument(null != level, "Invalid level type: null"); - try { - return ExpireLevel.valueOf(level.toUpperCase(Locale.ENGLISH)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(String.format("Invalid level type: %s", level), e); - } - } - } - - @VisibleForTesting - public enum BaseOnRule { - LAST_COMMIT_TIME, - CURRENT_TIME; - - public static BaseOnRule fromString(String since) { - Preconditions.checkArgument( - null != since, TableProperties.DATA_EXPIRATION_BASE_ON_RULE + " is invalid: null"); - try { - return BaseOnRule.valueOf(since.toUpperCase(Locale.ENGLISH)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - String.format("Unable to expire data since: %s", since), e); - } - } - } - - public static final Set FIELD_TYPES = - Sets.newHashSet(Type.TypeID.TIMESTAMP, Type.TypeID.STRING, Type.TypeID.LONG); - - private static final Logger LOG = LoggerFactory.getLogger(DataExpirationConfig.class); - - public DataExpirationConfig() {} - - public DataExpirationConfig( - boolean enabled, - String expirationField, - ExpireLevel expirationLevel, - long retentionTime, - String dateTimePattern, - String numberDateFormat, - BaseOnRule baseOnRule) { - this.enabled = enabled; - this.expirationField = expirationField; - this.expirationLevel = expirationLevel; - this.retentionTime = retentionTime; - this.dateTimePattern = dateTimePattern; - this.numberDateFormat = numberDateFormat; - this.baseOnRule = baseOnRule; - } - - public DataExpirationConfig(ArcticTable table) { - Map properties = table.properties(); - expirationField = - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_FIELD, null); - Types.NestedField field = table.schema().findField(expirationField); - Preconditions.checkArgument( - StringUtils.isNoneBlank(expirationField) && null != field, - String.format( - "Field(%s) used to determine data expiration is illegal for table(%s)", - expirationField, table.name())); - Type.TypeID typeID = field.type().typeId(); - Preconditions.checkArgument( - FIELD_TYPES.contains(typeID), - String.format( - "The type(%s) of filed(%s) is incompatible for table(%s)", - typeID.name(), expirationField, table.name())); - - expirationLevel = - ExpireLevel.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_LEVEL, - TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)); - - String retention = - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); - if (StringUtils.isNotBlank(retention)) { - retentionTime = ConfigurationUtil.TimeUtils.parseDuration(retention).toMillis(); - } - - dateTimePattern = - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT); - numberDateFormat = - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT); - baseOnRule = - BaseOnRule.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT)); - } - - public static DataExpirationConfig parse(Map properties) { - boolean gcEnabled = - CompatiblePropertyUtil.propertyAsBoolean( - properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); - DataExpirationConfig config = - new DataExpirationConfig() - .setEnabled( - gcEnabled - && CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_DATA_EXPIRATION, - TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT)) - .setExpirationLevel( - ExpireLevel.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_LEVEL, - TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT))) - .setExpirationField( - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_FIELD, null)) - .setDateTimePattern( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)) - .setNumberDateFormat( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)) - .setBaseOnRule( - BaseOnRule.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT))); - String retention = - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); - if (StringUtils.isNotBlank(retention)) { - config.setRetentionTime(ConfigurationUtil.TimeUtils.parseDuration(retention).toMillis()); - } - - return config; - } - - public boolean isEnabled() { - return enabled; - } - - public DataExpirationConfig setEnabled(boolean enabled) { - this.enabled = enabled; - return this; - } - - public String getExpirationField() { - return expirationField; - } - - public DataExpirationConfig setExpirationField(String expirationField) { - this.expirationField = expirationField; - return this; - } - - public ExpireLevel getExpirationLevel() { - return expirationLevel; - } - - public DataExpirationConfig setExpirationLevel(ExpireLevel expirationLevel) { - this.expirationLevel = expirationLevel; - return this; - } - - public long getRetentionTime() { - return retentionTime; - } - - public DataExpirationConfig setRetentionTime(long retentionTime) { - this.retentionTime = retentionTime; - return this; - } - - public String getDateTimePattern() { - return dateTimePattern; - } - - public DataExpirationConfig setDateTimePattern(String dateTimePattern) { - this.dateTimePattern = dateTimePattern; - return this; - } - - public String getNumberDateFormat() { - return numberDateFormat; - } - - public DataExpirationConfig setNumberDateFormat(String numberDateFormat) { - this.numberDateFormat = numberDateFormat; - return this; - } - - public BaseOnRule getBaseOnRule() { - return baseOnRule; - } - - public DataExpirationConfig setBaseOnRule(BaseOnRule baseOnRule) { - this.baseOnRule = baseOnRule; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof DataExpirationConfig)) { - return false; - } - DataExpirationConfig config = (DataExpirationConfig) o; - return enabled == config.enabled - && retentionTime == config.retentionTime - && Objects.equal(expirationField, config.expirationField) - && expirationLevel == config.expirationLevel - && Objects.equal(dateTimePattern, config.dateTimePattern) - && Objects.equal(numberDateFormat, config.numberDateFormat) - && baseOnRule == config.baseOnRule; - } - - @Override - public int hashCode() { - return Objects.hashCode( - enabled, - expirationField, - expirationLevel, - retentionTime, - dateTimePattern, - numberDateFormat, - baseOnRule); - } - - public boolean isValid(Types.NestedField field, String name) { - return isEnabled() - && getRetentionTime() > 0 - && validateExpirationField(field, name, getExpirationField()); - } - - private boolean validateExpirationField( - Types.NestedField field, String name, String expirationField) { - if (StringUtils.isBlank(expirationField) || null == field) { - LOG.warn( - String.format( - "Field(%s) used to determine data expiration is illegal for table(%s)", - expirationField, name)); - return false; - } - Type.TypeID typeID = field.type().typeId(); - if (!DataExpirationConfig.FIELD_TYPES.contains(typeID)) { - LOG.warn( - String.format( - "Table(%s) field(%s) type(%s) is not supported for data expiration, please use the " - + "following types: %s", - name, - expirationField, - typeID.name(), - StringUtils.join(DataExpirationConfig.FIELD_TYPES, ", "))); - return false; - } - - return true; - } -} diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/DefaultTableService.java b/ams/server/src/main/java/com/netease/arctic/server/table/DefaultTableService.java index 86e572222d..59f6f54e8b 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/DefaultTableService.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/DefaultTableService.java @@ -30,7 +30,10 @@ import com.netease.arctic.api.BlockableOperation; import com.netease.arctic.api.Blocker; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableIdentifier; +import com.netease.arctic.api.config.Configurations; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.catalog.CatalogBuilder; import com.netease.arctic.server.catalog.ExternalCatalog; @@ -45,7 +48,6 @@ import com.netease.arctic.server.persistence.mapper.CatalogMetaMapper; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; import com.netease.arctic.server.table.blocker.TableBlocker; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.utils.TablePropertyUtil; import org.apache.commons.lang.StringUtils; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; @@ -78,7 +80,6 @@ public class DefaultTableService extends StatedPersistentBase implements TableSe private final Map internalCatalogMap = new ConcurrentHashMap<>(); private final Map externalCatalogMap = new ConcurrentHashMap<>(); - @StateField private final Map tableRuntimeMap = new ConcurrentHashMap<>(); diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/RuntimeHandlerChain.java b/ams/server/src/main/java/com/netease/arctic/server/table/RuntimeHandlerChain.java index 24fb8844a9..0aa95e40f2 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/RuntimeHandlerChain.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/RuntimeHandlerChain.java @@ -20,6 +20,7 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.OptimizingStatus; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/ServerTableIdentifier.java b/ams/server/src/main/java/com/netease/arctic/server/table/ServerTableIdentifier.java deleted file mode 100644 index fa52ff2069..0000000000 --- a/ams/server/src/main/java/com/netease/arctic/server/table/ServerTableIdentifier.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.table; - -import com.netease.arctic.TableFormat; -import com.netease.arctic.api.TableIdentifier; - -import java.util.Objects; - -/** TODO Use ServerTableIdentifier class in API module, this class shall be removed after 0.7.0 */ -@Deprecated -public class ServerTableIdentifier { - - private Long id; - private String catalog; - private String database; - private String tableName; - private TableFormat format; - - // used by the MyBatis framework. - private ServerTableIdentifier() {} - - private ServerTableIdentifier(TableIdentifier tableIdentifier, TableFormat format) { - this.catalog = tableIdentifier.getCatalog(); - this.database = tableIdentifier.getDatabase(); - this.tableName = tableIdentifier.getTableName(); - this.format = format; - } - - private ServerTableIdentifier( - String catalog, String database, String tableName, TableFormat format) { - this.catalog = catalog; - this.database = database; - this.tableName = tableName; - this.format = format; - } - - private ServerTableIdentifier( - Long id, String catalog, String database, String tableName, TableFormat format) { - this.id = id; - this.catalog = catalog; - this.database = database; - this.tableName = tableName; - this.format = format; - } - - public Long getId() { - return id; - } - - public String getCatalog() { - return catalog; - } - - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - - public TableFormat getFormat() { - return this.format; - } - - public void setId(Long id) { - this.id = id; - } - - public void setCatalog(String catalog) { - this.catalog = catalog; - } - - public void setDatabase(String database) { - this.database = database; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public void setFormat(TableFormat format) { - this.format = format; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ServerTableIdentifier that = (ServerTableIdentifier) o; - return Objects.equals(id, that.id) - && Objects.equals(catalog, that.catalog) - && Objects.equals(database, that.database) - && Objects.equals(tableName, that.tableName); - } - - @Override - public int hashCode() { - return Objects.hash(id, catalog, database, tableName); - } - - @Override - public String toString() { - return String.format("%s.%s.%s(tableId=%d)", catalog, database, tableName, id); - } - - public static ServerTableIdentifier of(TableIdentifier tableIdentifier, TableFormat format) { - return new ServerTableIdentifier(tableIdentifier, format); - } - - public static ServerTableIdentifier of( - String catalog, String database, String tableName, TableFormat format) { - return new ServerTableIdentifier(catalog, database, tableName, format); - } - - public static ServerTableIdentifier of( - Long id, String catalog, String database, String tableName, TableFormat format) { - return new ServerTableIdentifier(id, catalog, database, tableName, format); - } - - public TableIdentifier getIdentifier() { - return new TableIdentifier(catalog, database, tableName); - } -} diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableConfiguration.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableConfiguration.java deleted file mode 100644 index fe14d637eb..0000000000 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableConfiguration.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.table; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.google.common.base.Objects; -import com.netease.arctic.server.optimizing.OptimizingConfig; -import com.netease.arctic.table.TableProperties; -import com.netease.arctic.utils.CompatiblePropertyUtil; - -import java.util.Map; -import java.util.Optional; - -/** TODO Use TableConfiguration class in API module, this class shall be removed after 0.7.0 */ -@Deprecated -@JsonIgnoreProperties(ignoreUnknown = true) -public class TableConfiguration { - private boolean expireSnapshotEnabled; - private long snapshotTTLMinutes; - private long changeDataTTLMinutes; - private boolean cleanOrphanEnabled; - private long orphanExistingMinutes; - private boolean deleteDanglingDeleteFilesEnabled; - private OptimizingConfig optimizingConfig; - private DataExpirationConfig expiringDataConfig; - private TagConfiguration tagConfiguration; - - public TableConfiguration() {} - - public boolean isExpireSnapshotEnabled() { - return expireSnapshotEnabled; - } - - public long getSnapshotTTLMinutes() { - return snapshotTTLMinutes; - } - - public long getChangeDataTTLMinutes() { - return changeDataTTLMinutes; - } - - public boolean isCleanOrphanEnabled() { - return cleanOrphanEnabled; - } - - public long getOrphanExistingMinutes() { - return orphanExistingMinutes; - } - - public OptimizingConfig getOptimizingConfig() { - return optimizingConfig; - } - - public TableConfiguration setOptimizingConfig(OptimizingConfig optimizingConfig) { - this.optimizingConfig = optimizingConfig; - return this; - } - - public TableConfiguration setExpireSnapshotEnabled(boolean expireSnapshotEnabled) { - this.expireSnapshotEnabled = expireSnapshotEnabled; - return this; - } - - public TableConfiguration setSnapshotTTLMinutes(long snapshotTTLMinutes) { - this.snapshotTTLMinutes = snapshotTTLMinutes; - return this; - } - - public TableConfiguration setChangeDataTTLMinutes(long changeDataTTLMinutes) { - this.changeDataTTLMinutes = changeDataTTLMinutes; - return this; - } - - public TableConfiguration setCleanOrphanEnabled(boolean cleanOrphanEnabled) { - this.cleanOrphanEnabled = cleanOrphanEnabled; - return this; - } - - public TableConfiguration setOrphanExistingMinutes(long orphanExistingMinutes) { - this.orphanExistingMinutes = orphanExistingMinutes; - return this; - } - - public boolean isDeleteDanglingDeleteFilesEnabled() { - return deleteDanglingDeleteFilesEnabled; - } - - public TableConfiguration setDeleteDanglingDeleteFilesEnabled( - boolean deleteDanglingDeleteFilesEnabled) { - this.deleteDanglingDeleteFilesEnabled = deleteDanglingDeleteFilesEnabled; - return this; - } - - public DataExpirationConfig getExpiringDataConfig() { - return Optional.ofNullable(expiringDataConfig).orElse(new DataExpirationConfig()); - } - - public TableConfiguration setExpiringDataConfig(DataExpirationConfig expiringDataConfig) { - this.expiringDataConfig = expiringDataConfig; - return this; - } - - public TagConfiguration getTagConfiguration() { - return Optional.ofNullable(tagConfiguration).orElse(new TagConfiguration()); - } - - public TableConfiguration setTagConfiguration(TagConfiguration tagConfiguration) { - this.tagConfiguration = tagConfiguration; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TableConfiguration that = (TableConfiguration) o; - return expireSnapshotEnabled == that.expireSnapshotEnabled - && snapshotTTLMinutes == that.snapshotTTLMinutes - && changeDataTTLMinutes == that.changeDataTTLMinutes - && cleanOrphanEnabled == that.cleanOrphanEnabled - && orphanExistingMinutes == that.orphanExistingMinutes - && deleteDanglingDeleteFilesEnabled == that.deleteDanglingDeleteFilesEnabled - && Objects.equal(optimizingConfig, that.optimizingConfig) - && Objects.equal(expiringDataConfig, that.expiringDataConfig) - && Objects.equal(tagConfiguration, that.tagConfiguration); - } - - @Override - public int hashCode() { - return Objects.hashCode( - expireSnapshotEnabled, - snapshotTTLMinutes, - changeDataTTLMinutes, - cleanOrphanEnabled, - orphanExistingMinutes, - deleteDanglingDeleteFilesEnabled, - optimizingConfig, - expiringDataConfig, - tagConfiguration); - } - - public static TableConfiguration parseConfig(Map properties) { - boolean gcEnabled = - CompatiblePropertyUtil.propertyAsBoolean( - properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); - return new TableConfiguration() - .setExpireSnapshotEnabled( - gcEnabled - && CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_TABLE_EXPIRE, - TableProperties.ENABLE_TABLE_EXPIRE_DEFAULT)) - .setSnapshotTTLMinutes( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.BASE_SNAPSHOT_KEEP_MINUTES, - TableProperties.BASE_SNAPSHOT_KEEP_MINUTES_DEFAULT)) - .setChangeDataTTLMinutes( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.CHANGE_DATA_TTL, - TableProperties.CHANGE_DATA_TTL_DEFAULT)) - .setCleanOrphanEnabled( - gcEnabled - && CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_ORPHAN_CLEAN, - TableProperties.ENABLE_ORPHAN_CLEAN_DEFAULT)) - .setOrphanExistingMinutes( - CompatiblePropertyUtil.propertyAsLong( - properties, - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME, - TableProperties.MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT)) - .setDeleteDanglingDeleteFilesEnabled( - gcEnabled - && CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_DANGLING_DELETE_FILES_CLEAN, - TableProperties.ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT)) - .setOptimizingConfig(OptimizingConfig.parseOptimizingConfig(properties)) - .setExpiringDataConfig(DataExpirationConfig.parse(properties)) - .setTagConfiguration(TagConfiguration.parse(properties)); - } -} diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableManager.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableManager.java index a110957ff2..954e502b04 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableManager.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableManager.java @@ -19,6 +19,7 @@ package com.netease.arctic.server.table; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.ServerTableIdentifier; public interface TableManager extends TableRuntimeHandler { diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableMetadata.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableMetadata.java index 2dce05be52..908110d891 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableMetadata.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableMetadata.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableMeta; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.properties.MetaTableProperties; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableMetrics.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableMetrics.java index 29369fa4bd..361cd0d019 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableMetrics.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableMetrics.java @@ -20,6 +20,7 @@ import static com.netease.arctic.api.metrics.MetricDefine.defineGauge; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.metrics.Gauge; import com.netease.arctic.api.metrics.Metric; import com.netease.arctic.api.metrics.MetricDefine; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntime.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntime.java index 40ae6f10e0..72ff91a0c4 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntime.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntime.java @@ -21,11 +21,14 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; import com.netease.arctic.api.BlockableOperation; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.StateField; +import com.netease.arctic.api.config.OptimizingConfig; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.ArcticServiceConstants; import com.netease.arctic.server.exception.BlockerConflictException; import com.netease.arctic.server.exception.ObjectNotExistsException; import com.netease.arctic.server.metrics.MetricRegistry; -import com.netease.arctic.server.optimizing.OptimizingConfig; import com.netease.arctic.server.optimizing.OptimizingProcess; import com.netease.arctic.server.optimizing.OptimizingStatus; import com.netease.arctic.server.optimizing.OptimizingType; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeHandler.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeHandler.java index ddc48923d6..16a4cc8b2d 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeHandler.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeHandler.java @@ -18,6 +18,7 @@ package com.netease.arctic.server.table; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.OptimizingStatus; public interface TableRuntimeHandler { diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeMeta.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeMeta.java index b396f3329a..c1f525f2cf 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeMeta.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableRuntimeMeta.java @@ -19,6 +19,7 @@ package com.netease.arctic.server.table; import com.netease.arctic.TableFormat; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.OptimizingProcess; import com.netease.arctic.server.optimizing.OptimizingStatus; import com.netease.arctic.server.optimizing.OptimizingType; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TableService.java b/ams/server/src/main/java/com/netease/arctic/server/table/TableService.java index 2d26c66784..332dcfacdc 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TableService.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TableService.java @@ -21,6 +21,7 @@ import com.netease.arctic.TableIDWithFormat; import com.netease.arctic.api.BlockableOperation; import com.netease.arctic.api.Blocker; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableIdentifier; import com.netease.arctic.server.catalog.CatalogService; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java b/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java deleted file mode 100644 index 24b2633f7c..0000000000 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.table; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.google.common.base.Objects; -import com.netease.arctic.table.TableProperties; -import com.netease.arctic.utils.CompatiblePropertyUtil; -import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.Locale; -import java.util.Map; - -/** TODO Use TagConfiguration class in API module, this class shall be removed after 0.7.0 */ -@Deprecated -/** Configuration for auto creating tags. */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class TagConfiguration { - // tag.auto-create.enabled - private boolean autoCreateTag = false; - // tag.auto-create.tag-format - private String tagFormat; - // tag.auto-create.trigger.period - private Period triggerPeriod; - // tag.auto-create.trigger.offset.minutes - private int triggerOffsetMinutes; - // tag.auto-create.trigger.max-delay.minutes - private int maxDelayMinutes; - // tag.auto-create.max-age-ms - private long tagMaxAgeMs; - - /** The interval for periodically triggering creating tags */ - public enum Period { - DAILY("daily") { - @Override - protected Duration periodDuration() { - return Duration.ofDays(1); - } - - @Override - public LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes) { - return checkTime.minusMinutes(triggerOffsetMinutes).truncatedTo(ChronoUnit.DAYS); - } - }, - - HOURLY("hourly") { - @Override - protected Duration periodDuration() { - return Duration.ofHours(1); - } - - @Override - public LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes) { - return checkTime.minusMinutes(triggerOffsetMinutes).truncatedTo(ChronoUnit.HOURS); - } - }; - - private final String propertyName; - - Period(String propertyName) { - this.propertyName = propertyName; - } - - public String propertyName() { - return propertyName; - } - - protected abstract Duration periodDuration(); - - /** - * Obtain the tag time for creating a tag, which is the ideal time of the last tag before the - * check time. - * - *

For example, when creating a daily tag, the check time is 2022-08-08 11:00:00 and the - * offset is set to be 5 min, the idea tag time is 2022-08-08 00:00:00. - * - *

For example, when creating a daily tag, the offset is set to be 30 min, if the check time - * is 2022-08-08 02:00:00, the ideal tag time is 2022-08-08 00:00:00; if the check time is - * 2022-08-09 00:20:00 (before 00:30 of the next day), the ideal tag time is still 2022-08-08 - * 00:00:00. - */ - public abstract LocalDateTime getTagTime(LocalDateTime checkTime, int triggerOffsetMinutes); - - public String generateTagName(LocalDateTime tagTime, String tagFormat) { - return tagTime.minus(periodDuration()).format(DateTimeFormatter.ofPattern(tagFormat)); - } - } - - public static TagConfiguration parse(Map tableProperties) { - TagConfiguration tagConfig = new TagConfiguration(); - tagConfig.setAutoCreateTag( - CompatiblePropertyUtil.propertyAsBoolean( - tableProperties, - TableProperties.ENABLE_AUTO_CREATE_TAG, - TableProperties.ENABLE_AUTO_CREATE_TAG_DEFAULT)); - tagConfig.setTriggerPeriod( - Period.valueOf( - CompatiblePropertyUtil.propertyAsString( - tableProperties, - TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD, - TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD_DEFAULT) - .toUpperCase(Locale.ROOT))); - - String defaultFormat; - switch (tagConfig.getTriggerPeriod()) { - case DAILY: - defaultFormat = TableProperties.AUTO_CREATE_TAG_FORMAT_DAILY_DEFAULT; - break; - case HOURLY: - defaultFormat = TableProperties.AUTO_CREATE_TAG_FORMAT_HOURLY_DEFAULT; - break; - default: - throw new IllegalArgumentException( - "Unsupported trigger period: " + tagConfig.getTriggerPeriod()); - } - tagConfig.setTagFormat( - CompatiblePropertyUtil.propertyAsString( - tableProperties, TableProperties.AUTO_CREATE_TAG_FORMAT, defaultFormat)); - tagConfig.setTriggerOffsetMinutes( - CompatiblePropertyUtil.propertyAsInt( - tableProperties, - TableProperties.AUTO_CREATE_TAG_TRIGGER_OFFSET_MINUTES, - TableProperties.AUTO_CREATE_TAG_TRIGGER_OFFSET_MINUTES_DEFAULT)); - tagConfig.setMaxDelayMinutes( - CompatiblePropertyUtil.propertyAsInt( - tableProperties, - TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES, - TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT)); - tagConfig.setTagMaxAgeMs( - CompatiblePropertyUtil.propertyAsLong( - tableProperties, - TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS, - TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT)); - return tagConfig; - } - - public boolean isAutoCreateTag() { - return autoCreateTag; - } - - public void setAutoCreateTag(boolean autoCreateTag) { - this.autoCreateTag = autoCreateTag; - } - - public String getTagFormat() { - return tagFormat; - } - - public void setTagFormat(String tagFormat) { - this.tagFormat = tagFormat; - } - - public Period getTriggerPeriod() { - return triggerPeriod; - } - - public void setTriggerPeriod(Period triggerPeriod) { - this.triggerPeriod = triggerPeriod; - } - - public int getTriggerOffsetMinutes() { - return triggerOffsetMinutes; - } - - public void setTriggerOffsetMinutes(int triggerOffsetMinutes) { - this.triggerOffsetMinutes = triggerOffsetMinutes; - } - - public int getMaxDelayMinutes() { - return maxDelayMinutes; - } - - public void setMaxDelayMinutes(int maxDelayMinutes) { - this.maxDelayMinutes = maxDelayMinutes; - } - - public long getTagMaxAgeMs() { - return tagMaxAgeMs; - } - - public void setTagMaxAgeMs(long tagMaxAgeMs) { - this.tagMaxAgeMs = tagMaxAgeMs; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TagConfiguration that = (TagConfiguration) o; - return autoCreateTag == that.autoCreateTag - && triggerOffsetMinutes == that.triggerOffsetMinutes - && maxDelayMinutes == that.maxDelayMinutes - && Objects.equal(tagFormat, that.tagFormat) - && triggerPeriod == that.triggerPeriod - && tagMaxAgeMs == that.tagMaxAgeMs; - } - - @Override - public int hashCode() { - return Objects.hashCode( - autoCreateTag, - tagFormat, - triggerPeriod, - triggerOffsetMinutes, - maxDelayMinutes, - tagMaxAgeMs); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("autoCreateTag", autoCreateTag) - .add("tagFormat", tagFormat) - .add("triggerPeriod", triggerPeriod) - .add("triggerOffsetMinutes", triggerOffsetMinutes) - .add("maxDelayMinutes", maxDelayMinutes) - .add("tagMaxAgeMs", tagMaxAgeMs) - .toString(); - } -} diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/blocker/TableBlocker.java b/ams/server/src/main/java/com/netease/arctic/server/table/blocker/TableBlocker.java index 3baaea0e03..43117185d9 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/blocker/TableBlocker.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/blocker/TableBlocker.java @@ -20,7 +20,7 @@ import com.netease.arctic.api.BlockableOperation; import com.netease.arctic.api.Blocker; -import com.netease.arctic.server.table.ServerTableIdentifier; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.table.blocker.RenewableBlocker; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.collect.Maps; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/AsyncTableExecutors.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/AsyncTableExecutors.java index c927a8c420..d5cd9bac9a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/AsyncTableExecutors.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/AsyncTableExecutors.java @@ -18,9 +18,9 @@ package com.netease.arctic.server.table.executor; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.table.TableManager; -import com.netease.arctic.server.utils.Configurations; public class AsyncTableExecutors { diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/BaseTableExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/BaseTableExecutor.java index 81bd9408b7..f37421565a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/BaseTableExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/BaseTableExecutor.java @@ -19,10 +19,10 @@ package com.netease.arctic.server.table.executor; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.OptimizingStatus; import com.netease.arctic.server.table.RuntimeHandlerChain; -import com.netease.arctic.server.table.ServerTableIdentifier; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableRuntimeMeta; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/DanglingDeleteFilesCleaningExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/DanglingDeleteFilesCleaningExecutor.java index 30bcce52f1..fb32995a3b 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/DanglingDeleteFilesCleaningExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/DanglingDeleteFilesCleaningExecutor.java @@ -21,8 +21,8 @@ import static com.netease.arctic.server.optimizing.maintainer.TableMaintainer.ofTable; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.maintainer.TableMaintainer; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/DataExpiringExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/DataExpiringExecutor.java index f5c0abca1f..9ec0905b03 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/DataExpiringExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/DataExpiringExecutor.java @@ -19,8 +19,8 @@ package com.netease.arctic.server.table.executor; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.maintainer.TableMaintainer; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/HiveCommitSyncExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/HiveCommitSyncExecutor.java index 06cd7bbb3a..f3aefd3327 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/HiveCommitSyncExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/HiveCommitSyncExecutor.java @@ -18,10 +18,10 @@ package com.netease.arctic.server.table.executor; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.hive.utils.HiveMetaSynchronizer; import com.netease.arctic.hive.utils.TableTypeUtil; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.table.ArcticTable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/OrphanFilesCleaningExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/OrphanFilesCleaningExecutor.java index 548fdd6245..50d3f7e95a 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/OrphanFilesCleaningExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/OrphanFilesCleaningExecutor.java @@ -21,8 +21,8 @@ import static com.netease.arctic.server.optimizing.maintainer.TableMaintainer.ofTable; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.maintainer.TableMaintainer; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/SnapshotsExpiringExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/SnapshotsExpiringExecutor.java index bfd5dd35d8..c3a62d18e9 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/SnapshotsExpiringExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/SnapshotsExpiringExecutor.java @@ -19,8 +19,8 @@ package com.netease.arctic.server.table.executor; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.maintainer.TableMaintainer; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/TableRuntimeRefreshExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/TableRuntimeRefreshExecutor.java index d7b60e891d..2a7b0c07ba 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/TableRuntimeRefreshExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/TableRuntimeRefreshExecutor.java @@ -19,9 +19,9 @@ package com.netease.arctic.server.table.executor; import com.netease.arctic.AmoroTable; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.OptimizingProcess; import com.netease.arctic.server.optimizing.plan.OptimizingEvaluator; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.table.ArcticTable; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/executor/TagsAutoCreatingExecutor.java b/ams/server/src/main/java/com/netease/arctic/server/table/executor/TagsAutoCreatingExecutor.java index 852bdf53bc..dfb2705902 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/executor/TagsAutoCreatingExecutor.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/executor/TagsAutoCreatingExecutor.java @@ -22,8 +22,8 @@ import com.netease.arctic.AmoroTable; import com.netease.arctic.TableFormat; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.server.optimizing.maintainer.TableMaintainer; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import org.slf4j.Logger; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/internal/IcebergInternalTableOperations.java b/ams/server/src/main/java/com/netease/arctic/server/table/internal/IcebergInternalTableOperations.java index 7a250f080f..475ce4104f 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/internal/IcebergInternalTableOperations.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/internal/IcebergInternalTableOperations.java @@ -21,9 +21,9 @@ import static com.netease.arctic.server.table.internal.InternalTableConstants.PROPERTIES_METADATA_LOCATION; import static com.netease.arctic.server.table.internal.InternalTableConstants.PROPERTIES_PREV_METADATA_LOCATION; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.persistence.PersistentBase; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.utils.InternalTableUtil; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.LocationProviders; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/internal/InternalIcebergCreator.java b/ams/server/src/main/java/com/netease/arctic/server/table/internal/InternalIcebergCreator.java index bf7e03fb46..a5e7ba4f23 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/internal/InternalIcebergCreator.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/internal/InternalIcebergCreator.java @@ -23,11 +23,11 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableMeta; import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.properties.MetaTableProperties; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import com.netease.arctic.server.utils.InternalTableUtil; import org.apache.commons.lang3.StringUtils; diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/internal/MixedIcebergInternalTableStoreOperations.java b/ams/server/src/main/java/com/netease/arctic/server/table/internal/MixedIcebergInternalTableStoreOperations.java index 708be72113..fbb733880d 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/internal/MixedIcebergInternalTableStoreOperations.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/internal/MixedIcebergInternalTableStoreOperations.java @@ -22,7 +22,7 @@ import static com.netease.arctic.server.table.internal.InternalTableConstants.PROPERTIES_METADATA_LOCATION; import static com.netease.arctic.server.table.internal.InternalTableConstants.PROPERTIES_PREV_METADATA_LOCATION; -import com.netease.arctic.server.table.ServerTableIdentifier; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.io.FileIO; diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/SparkContextUtil.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/SparkContextUtil.java index d4e264100f..8f774b5aad 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/SparkContextUtil.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/SparkContextUtil.java @@ -18,8 +18,8 @@ package com.netease.arctic.server.terminal; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.catalog.CatalogType; -import com.netease.arctic.server.utils.Configurations; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import java.util.List; diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalManager.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalManager.java index 3879625002..0fdc60ab94 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalManager.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalManager.java @@ -21,6 +21,8 @@ import com.netease.arctic.Constants; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.config.ConfigOptions; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.catalog.CatalogType; @@ -32,8 +34,6 @@ import com.netease.arctic.server.terminal.TerminalSessionFactory.SessionConfigOptions; import com.netease.arctic.server.terminal.kyuubi.KyuubiTerminalSessionFactory; import com.netease.arctic.server.terminal.local.LocalSessionFactory; -import com.netease.arctic.server.utils.ConfigOptions; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableMetaStore; import com.netease.arctic.utils.ArcticCatalogUtil; import org.apache.iceberg.CatalogProperties; diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionContext.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionContext.java index 0c9044c9be..54a76ef488 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionContext.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionContext.java @@ -18,7 +18,7 @@ package com.netease.arctic.server.terminal; -import com.netease.arctic.server.utils.Configurations; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.table.TableMetaStore; import org.apache.commons.io.Charsets; import org.apache.iceberg.relocated.com.google.common.collect.Lists; diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionFactory.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionFactory.java index ddd24a40cb..102ae7ded2 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionFactory.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/TerminalSessionFactory.java @@ -19,9 +19,9 @@ package com.netease.arctic.server.terminal; import com.google.common.collect.Maps; -import com.netease.arctic.server.utils.ConfigOption; -import com.netease.arctic.server.utils.ConfigOptions; -import com.netease.arctic.server.utils.Configurations; +import com.netease.arctic.api.config.ConfigOption; +import com.netease.arctic.api.config.ConfigOptions; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.table.TableMetaStore; import java.util.List; diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/kyuubi/KyuubiTerminalSessionFactory.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/kyuubi/KyuubiTerminalSessionFactory.java index 81e23cd924..7439fae808 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/kyuubi/KyuubiTerminalSessionFactory.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/kyuubi/KyuubiTerminalSessionFactory.java @@ -18,17 +18,12 @@ package com.netease.arctic.server.terminal.kyuubi; -import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE; -import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT; -import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.AUTH_PRINCIPAL; -import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.AUTH_USER; - +import com.netease.arctic.api.config.ConfigOption; +import com.netease.arctic.api.config.ConfigOptions; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.terminal.SparkContextUtil; import com.netease.arctic.server.terminal.TerminalSession; import com.netease.arctic.server.terminal.TerminalSessionFactory; -import com.netease.arctic.server.utils.ConfigOption; -import com.netease.arctic.server.utils.ConfigOptions; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableMetaStore; import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -129,8 +124,8 @@ public TerminalSession create(TableMetaStore metaStore, Configurations configura Properties properties = new Properties(); if (!metaStore.isKerberosAuthMethod()) { - properties.put(AUTH_USER, metaStore.getHadoopUsername()); - sessionConf.put(AUTH_USER, metaStore.getHadoopUsername()); + properties.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername()); + sessionConf.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername()); } Connection connection = metaStore.doAs(() -> driver.connect(kyuubiJdbcUrl, properties)); @@ -169,13 +164,17 @@ private void logMessage(List logs, String message) { private void checkAndFillKerberosInfo( JdbcConnectionParams connectionParams, TableMetaStore metaStore) { - if (connectionParams.getSessionVars().containsKey(AUTH_PRINCIPAL)) { + if (connectionParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_PRINCIPAL)) { throw new RuntimeException( "jdbc url should not contain principal when kyuubi kerberos enable"); } connectionParams .getSessionVars() - .put(AUTH_KERBEROS_AUTH_TYPE, AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT); - connectionParams.getSessionVars().put(AUTH_PRINCIPAL, metaStore.getKrbPrincipal()); + .put( + JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE, + JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT); + connectionParams + .getSessionVars() + .put(JdbcConnectionParams.AUTH_PRINCIPAL, metaStore.getKrbPrincipal()); } } diff --git a/ams/server/src/main/java/com/netease/arctic/server/terminal/local/LocalSessionFactory.java b/ams/server/src/main/java/com/netease/arctic/server/terminal/local/LocalSessionFactory.java index de18d77336..349bb6db8f 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/terminal/local/LocalSessionFactory.java +++ b/ams/server/src/main/java/com/netease/arctic/server/terminal/local/LocalSessionFactory.java @@ -18,12 +18,12 @@ package com.netease.arctic.server.terminal.local; +import com.netease.arctic.api.config.ConfigOption; +import com.netease.arctic.api.config.ConfigOptions; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.terminal.SparkContextUtil; import com.netease.arctic.server.terminal.TerminalSession; import com.netease.arctic.server.terminal.TerminalSessionFactory; -import com.netease.arctic.server.utils.ConfigOption; -import com.netease.arctic.server.utils.ConfigOptions; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableMetaStore; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; diff --git a/ams/server/src/test/java/com/netease/arctic/server/AmsEnvironment.java b/ams/server/src/test/java/com/netease/arctic/server/AmsEnvironment.java index 908282ef2c..3b8f14cd95 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/AmsEnvironment.java +++ b/ams/server/src/test/java/com/netease/arctic/server/AmsEnvironment.java @@ -21,6 +21,7 @@ import com.netease.arctic.SingletonResourceUtil; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.catalog.ArcticCatalog; import com.netease.arctic.catalog.CatalogLoader; @@ -31,7 +32,6 @@ import com.netease.arctic.server.resource.OptimizerManager; import com.netease.arctic.server.resource.ResourceContainers; import com.netease.arctic.server.table.DefaultTableService; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableIdentifier; import org.apache.commons.io.FileUtils; import org.apache.curator.shaded.com.google.common.io.MoreFiles; diff --git a/ams/server/src/test/java/com/netease/arctic/server/RestCatalogServiceTestBase.java b/ams/server/src/test/java/com/netease/arctic/server/RestCatalogServiceTestBase.java index 1a03fb207a..f5c5f01bb7 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/RestCatalogServiceTestBase.java +++ b/ams/server/src/test/java/com/netease/arctic/server/RestCatalogServiceTestBase.java @@ -21,9 +21,9 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.properties.CatalogMetaProperties; import com.netease.arctic.server.catalog.InternalCatalog; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableMetadata; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableService; diff --git a/ams/server/src/test/java/com/netease/arctic/server/dashboard/TestServerTableDescriptor.java b/ams/server/src/test/java/com/netease/arctic/server/dashboard/TestServerTableDescriptor.java index bf2323b728..1c799e3aaa 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/dashboard/TestServerTableDescriptor.java +++ b/ams/server/src/test/java/com/netease/arctic/server/dashboard/TestServerTableDescriptor.java @@ -18,10 +18,10 @@ package com.netease.arctic.server.dashboard; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.formats.AmoroCatalogTestHelper; import com.netease.arctic.server.catalog.TableCatalogTestBase; import com.netease.arctic.server.dashboard.model.DDLInfo; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.TableIdentifier; import org.junit.Assert; import org.junit.Before; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/TestOptimizingQueue.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/TestOptimizingQueue.java index d64b936698..9655beca72 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/TestOptimizingQueue.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/TestOptimizingQueue.java @@ -35,6 +35,7 @@ import com.netease.arctic.api.OptimizerRegisterInfo; import com.netease.arctic.api.OptimizingTaskId; import com.netease.arctic.api.OptimizingTaskResult; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.api.metrics.Gauge; import com.netease.arctic.api.metrics.MetricKey; import com.netease.arctic.api.resource.ResourceGroup; @@ -49,7 +50,6 @@ import com.netease.arctic.server.resource.OptimizerThread; import com.netease.arctic.server.resource.QuotaProvider; import com.netease.arctic.server.table.AMSTableTestBase; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableRuntimeMeta; import com.netease.arctic.table.ArcticTable; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/flow/CompleteOptimizingFlow.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/flow/CompleteOptimizingFlow.java index 81185dfc9f..ce96f73973 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/flow/CompleteOptimizingFlow.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/flow/CompleteOptimizingFlow.java @@ -25,6 +25,9 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.api.OptimizerProperties; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.OptimizingConfig; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.hive.optimizing.MixFormatRewriteExecutor; import com.netease.arctic.optimizing.IcebergRewriteExecutor; import com.netease.arctic.optimizing.OptimizingExecutor; @@ -32,13 +35,10 @@ import com.netease.arctic.optimizing.RewriteFilesOutput; import com.netease.arctic.server.ArcticServiceConstants; import com.netease.arctic.server.optimizing.KeyedTableCommit; -import com.netease.arctic.server.optimizing.OptimizingConfig; import com.netease.arctic.server.optimizing.TaskRuntime; import com.netease.arctic.server.optimizing.UnKeyedTableCommit; import com.netease.arctic.server.optimizing.plan.OptimizingPlanner; import com.netease.arctic.server.optimizing.plan.TaskDescriptor; -import com.netease.arctic.server.table.ServerTableIdentifier; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.utils.IcebergTableUtil; import com.netease.arctic.table.ArcticTable; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java index 483c9677d4..2f1e67a42e 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java @@ -20,9 +20,9 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; +import com.netease.arctic.api.config.TagConfiguration; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.TableTestBase; -import com.netease.arctic.server.table.TagConfiguration; import com.netease.arctic.table.TableProperties; import org.apache.iceberg.ExpireSnapshots; import org.apache.iceberg.Snapshot; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestDataExpire.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestDataExpire.java index 3e21364f80..45bee2645a 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestDataExpire.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestDataExpire.java @@ -25,6 +25,7 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.config.DataExpirationConfig; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.data.ChangeAction; @@ -35,7 +36,6 @@ import com.netease.arctic.server.optimizing.scan.KeyedTableFileScanHelper; import com.netease.arctic.server.optimizing.scan.TableFileScanHelper; import com.netease.arctic.server.optimizing.scan.UnkeyedTableFileScanHelper; -import com.netease.arctic.server.table.DataExpirationConfig; import com.netease.arctic.server.table.KeyedTableSnapshot; import com.netease.arctic.server.table.executor.ExecutorTestBase; import com.netease.arctic.server.utils.IcebergTableUtil; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestOrphanFileClean.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestOrphanFileClean.java index 119a5dbd89..822f0b96df 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestOrphanFileClean.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestOrphanFileClean.java @@ -24,12 +24,12 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.data.ChangeAction; import com.netease.arctic.server.dashboard.utils.AmsUtil; -import com.netease.arctic.server.table.ServerTableIdentifier; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.executor.ExecutorTestBase; import com.netease.arctic.table.KeyedTable; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestSnapshotExpire.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestSnapshotExpire.java index 03be93a076..a1306dbec8 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestSnapshotExpire.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestSnapshotExpire.java @@ -24,14 +24,14 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.data.ChangeAction; import com.netease.arctic.server.dashboard.utils.AmsUtil; import com.netease.arctic.server.optimizing.OptimizingProcess; import com.netease.arctic.server.optimizing.OptimizingStatus; -import com.netease.arctic.server.table.ServerTableIdentifier; -import com.netease.arctic.server.table.TableConfiguration; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.executor.ExecutorTestBase; import com.netease.arctic.table.BaseTable; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/MixedTablePlanTestBase.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/MixedTablePlanTestBase.java index 8d9f5a2ccd..95a98dc3eb 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/MixedTablePlanTestBase.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/MixedTablePlanTestBase.java @@ -20,6 +20,8 @@ import com.google.common.collect.Maps; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.OptimizingConfig; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.catalog.TableTestBase; import com.netease.arctic.data.DataTreeNode; @@ -29,10 +31,8 @@ import com.netease.arctic.optimizing.OptimizingInputProperties; import com.netease.arctic.server.ArcticServiceConstants; import com.netease.arctic.server.dashboard.utils.AmsUtil; -import com.netease.arctic.server.optimizing.OptimizingConfig; import com.netease.arctic.server.optimizing.OptimizingTestHelpers; import com.netease.arctic.server.optimizing.scan.TableFileScanHelper; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.utils.IcebergTableUtil; import com.netease.arctic.table.TableProperties; @@ -567,7 +567,7 @@ protected TableRuntime getTableRuntime() { } private OptimizingConfig getConfig() { - return OptimizingConfig.parseOptimizingConfig(getArcticTable().properties()); + return OptimizingConfig.parse(getArcticTable().properties()); } protected void updateChangeHashBucket(int bucket) { diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveKeyedPartitionPlan.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveKeyedPartitionPlan.java index bb42fe61c2..77d98458f2 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveKeyedPartitionPlan.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveKeyedPartitionPlan.java @@ -23,12 +23,12 @@ import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.data.ChangeAction; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.optimizing.OptimizingInputProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.server.optimizing.OptimizingTestHelpers; import org.apache.iceberg.DataFile; import org.apache.iceberg.StructLike; diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveUnkeyedPartitionPlan.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveUnkeyedPartitionPlan.java index a25235d7d8..9aff083d15 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveUnkeyedPartitionPlan.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/plan/TestHiveUnkeyedPartitionPlan.java @@ -22,12 +22,12 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.optimizing.OptimizingInputProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.server.optimizing.OptimizingTestHelpers; import org.apache.iceberg.DataFile; import org.apache.iceberg.StructLike; diff --git a/ams/server/src/test/java/com/netease/arctic/server/persistence/TestStatedPersistentBase.java b/ams/server/src/test/java/com/netease/arctic/server/persistence/TestStatedPersistentBase.java index 0400481edd..f2ce25f5f5 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/persistence/TestStatedPersistentBase.java +++ b/ams/server/src/test/java/com/netease/arctic/server/persistence/TestStatedPersistentBase.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import com.netease.arctic.api.StateField; import org.junit.Test; import java.util.UUID; @@ -28,15 +29,15 @@ public class TestStatedPersistentBase { private static class ExtendedPersistency extends StatedPersistentBase { - @StatedPersistentBase.StateField private String stringState = ""; - @StatedPersistentBase.StateField private int intState = 0; + @StateField private String stringState = ""; + @StateField private int intState = 0; private boolean booleanField = false; private long longField = 0L; } private static class NormalClass { - @StatedPersistentBase.StateField private String stringState = ""; - @StatedPersistentBase.StateField private int intState = 0; + @StateField private String stringState = ""; + @StateField private int intState = 0; private final boolean booleanField = false; private final long longField = 0L; } diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/AMSTableTestBase.java b/ams/server/src/test/java/com/netease/arctic/server/table/AMSTableTestBase.java index 198ebcb89f..b36f2fe0f0 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/AMSTableTestBase.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/AMSTableTestBase.java @@ -23,6 +23,7 @@ import com.netease.arctic.TableTestHelper; import com.netease.arctic.UnifiedCatalog; import com.netease.arctic.api.CatalogMeta; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.api.TableMeta; import com.netease.arctic.catalog.ArcticCatalog; import com.netease.arctic.catalog.CatalogLoader; diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/DerbyPersistence.java b/ams/server/src/test/java/com/netease/arctic/server/table/DerbyPersistence.java index 36709d2331..9576438696 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/DerbyPersistence.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/DerbyPersistence.java @@ -18,9 +18,9 @@ package com.netease.arctic.server.table; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.persistence.SqlSessionFactoryProvider; -import com.netease.arctic.server.utils.Configurations; import org.apache.ibatis.session.SqlSession; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.junit.rules.ExternalResource; diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/TableServiceTestBase.java b/ams/server/src/test/java/com/netease/arctic/server/table/TableServiceTestBase.java index 40f127dade..e471447299 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/TableServiceTestBase.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/TableServiceTestBase.java @@ -18,12 +18,12 @@ package com.netease.arctic.server.table; +import com.netease.arctic.api.config.Configurations; import com.netease.arctic.api.resource.ResourceGroup; import com.netease.arctic.server.ArcticManagementConf; import com.netease.arctic.server.DefaultOptimizingService; import com.netease.arctic.server.manager.EventsManager; import com.netease.arctic.server.manager.MetricManager; -import com.netease.arctic.server.utils.Configurations; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeHandler.java b/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeHandler.java index e00380647d..d33ebde1d0 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeHandler.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeHandler.java @@ -22,6 +22,9 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.ServerTableIdentifier; +import com.netease.arctic.api.config.Configurations; +import com.netease.arctic.api.config.TableConfiguration; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; @@ -29,7 +32,6 @@ import com.netease.arctic.server.manager.EventsManager; import com.netease.arctic.server.manager.MetricManager; import com.netease.arctic.server.optimizing.OptimizingStatus; -import com.netease.arctic.server.utils.Configurations; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.Lists; diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeManager.java b/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeManager.java index ade9575698..57507ce204 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeManager.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/TestTableRuntimeManager.java @@ -21,6 +21,7 @@ import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.catalog.BasicCatalogTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; diff --git a/ams/server/src/test/java/com/netease/arctic/server/table/executor/TestBlockerExpiringExecutor.java b/ams/server/src/test/java/com/netease/arctic/server/table/executor/TestBlockerExpiringExecutor.java index b7d82f9342..2d253fa39d 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/table/executor/TestBlockerExpiringExecutor.java +++ b/ams/server/src/test/java/com/netease/arctic/server/table/executor/TestBlockerExpiringExecutor.java @@ -20,9 +20,9 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.api.BlockableOperation; +import com.netease.arctic.api.ServerTableIdentifier; import com.netease.arctic.server.persistence.PersistentBase; import com.netease.arctic.server.persistence.mapper.TableBlockerMapper; -import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.server.table.TableManager; import com.netease.arctic.server.table.TableRuntime; import com.netease.arctic.server.table.TableServiceTestBase; diff --git a/ams/server/src/test/java/com/netease/arctic/server/util/TestConfigurationUtil.java b/ams/server/src/test/java/com/netease/arctic/server/util/TestConfigurationUtil.java index 125f424c7e..cf38d1162d 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/util/TestConfigurationUtil.java +++ b/ams/server/src/test/java/com/netease/arctic/server/util/TestConfigurationUtil.java @@ -24,8 +24,8 @@ import static com.netease.arctic.server.ArcticManagementConf.DB_USER_NAME; import static com.netease.arctic.server.ArcticManagementConf.SERVER_EXPOSE_HOST; +import com.netease.arctic.api.config.ConfigHelpers; import com.netease.arctic.server.ArcticManagementConf; -import com.netease.arctic.server.utils.ConfigurationUtil; import org.junit.Assert; import org.junit.Test; @@ -43,7 +43,7 @@ public void testConvertConfigurationKeys() { dummyEnv.put("AMS_SERVER__EXPOSE__HOST", "127.0.0.1"); dummyEnv.put("AMS_ADMIN__USERNAME", "admin"); dummyEnv.put("AMS_ADMIN__PASSWORD", "admin"); - Map result = ConfigurationUtil.convertConfigurationKeys(prefix, dummyEnv); + Map result = ConfigHelpers.convertConfigurationKeys(prefix, dummyEnv); Assert.assertNotNull("AMS_DATABASE_USERNAME Convert Failed", result.get(DB_USER_NAME.key())); Assert.assertNotNull("AMS_DATABASE_PASSWORD Convert Failed", result.get(DB_PASSWORD.key())); Assert.assertNotNull( diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/HiveTableProperties.java b/core/src/main/java/com/netease/arctic/properties/HiveTableProperties.java similarity index 98% rename from mixed-format/hive/src/main/java/com/netease/arctic/hive/HiveTableProperties.java rename to core/src/main/java/com/netease/arctic/properties/HiveTableProperties.java index 99a4b84f26..ddd24f1316 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/HiveTableProperties.java +++ b/core/src/main/java/com/netease/arctic/properties/HiveTableProperties.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.netease.arctic.hive; +package com.netease.arctic.properties; import java.util.HashSet; import java.util.Set; diff --git a/mixed-format/flink/flink-common/src/main/java/com/netease/arctic/flink/write/FlinkTaskWriterBuilder.java b/mixed-format/flink/flink-common/src/main/java/com/netease/arctic/flink/write/FlinkTaskWriterBuilder.java index a1bc16190b..386511a847 100644 --- a/mixed-format/flink/flink-common/src/main/java/com/netease/arctic/flink/write/FlinkTaskWriterBuilder.java +++ b/mixed-format/flink/flink-common/src/main/java/com/netease/arctic/flink/write/FlinkTaskWriterBuilder.java @@ -18,7 +18,6 @@ package com.netease.arctic.flink.write; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.writer.AdaptHiveOperateToTableRelation; import com.netease.arctic.hive.io.writer.AdaptHiveOutputFileFactory; import com.netease.arctic.hive.table.HiveLocationKind; @@ -28,6 +27,7 @@ import com.netease.arctic.io.writer.OutputFileFactory; import com.netease.arctic.io.writer.SortedPosDeleteWriter; import com.netease.arctic.io.writer.TaskWriterBuilder; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.BaseLocationKind; import com.netease.arctic.table.ChangeLocationKind; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/ArcticHiveCatalog.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/ArcticHiveCatalog.java index 8007e41266..2df9ba5b7b 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/ArcticHiveCatalog.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/ArcticHiveCatalog.java @@ -25,8 +25,8 @@ import com.netease.arctic.hive.CachedHiveClientPool; import com.netease.arctic.hive.HMSClient; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.utils.HiveSchemaUtil; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.PrimaryKeySpec; import com.netease.arctic.table.TableBuilder; import com.netease.arctic.table.TableIdentifier; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/MixedHiveTables.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/MixedHiveTables.java index 4223f7c17a..5fcd8a8ed1 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/MixedHiveTables.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/catalog/MixedHiveTables.java @@ -21,7 +21,6 @@ import com.netease.arctic.api.TableMeta; import com.netease.arctic.catalog.MixedTables; import com.netease.arctic.hive.CachedHiveClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.table.KeyedHiveTable; import com.netease.arctic.hive.table.UnkeyedHiveTable; import com.netease.arctic.hive.utils.HiveSchemaUtil; @@ -29,6 +28,7 @@ import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.io.ArcticFileIOs; import com.netease.arctic.io.ArcticHadoopFileIO; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.properties.MetaTableProperties; import com.netease.arctic.table.ChangeTable; import com.netease.arctic.table.KeyedTable; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/io/writer/AdaptHiveGenericTaskWriterBuilder.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/io/writer/AdaptHiveGenericTaskWriterBuilder.java index 2d4e8fe252..4c58c2a6e5 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/io/writer/AdaptHiveGenericTaskWriterBuilder.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/io/writer/AdaptHiveGenericTaskWriterBuilder.java @@ -19,7 +19,6 @@ package com.netease.arctic.hive.io.writer; import com.netease.arctic.data.ChangeAction; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.table.HiveLocationKind; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.hive.utils.TableTypeUtil; @@ -29,6 +28,7 @@ import com.netease.arctic.io.writer.OutputFileFactory; import com.netease.arctic.io.writer.SortedPosDeleteWriter; import com.netease.arctic.io.writer.TaskWriterBuilder; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.BaseLocationKind; import com.netease.arctic.table.ChangeLocationKind; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/ReplaceHivePartitions.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/ReplaceHivePartitions.java index 1d9a385a76..e97d7840dc 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/ReplaceHivePartitions.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/ReplaceHivePartitions.java @@ -19,7 +19,6 @@ package com.netease.arctic.hive.op; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.exceptions.CannotAlterHiveLocationException; import com.netease.arctic.hive.table.UnkeyedHiveTable; import com.netease.arctic.hive.utils.HiveCommitUtil; @@ -27,6 +26,7 @@ import com.netease.arctic.hive.utils.HiveTableUtil; import com.netease.arctic.io.ArcticHadoopFileIO; import com.netease.arctic.op.UpdatePartitionProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.utils.TableFileUtil; import com.netease.arctic.utils.TablePropertyUtil; import org.apache.hadoop.fs.Path; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/UpdateHiveFiles.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/UpdateHiveFiles.java index 422eab8f98..3b18711d84 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/UpdateHiveFiles.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/op/UpdateHiveFiles.java @@ -21,7 +21,6 @@ import static com.netease.arctic.op.OverwriteBaseFiles.PROPERTIES_TRANSACTION_ID; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.exceptions.CannotAlterHiveLocationException; import com.netease.arctic.hive.table.UnkeyedHiveTable; import com.netease.arctic.hive.utils.HiveCommitUtil; @@ -30,6 +29,7 @@ import com.netease.arctic.hive.utils.HiveTableUtil; import com.netease.arctic.io.ArcticHadoopFileIO; import com.netease.arctic.op.UpdatePartitionProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.utils.TableFileUtil; import com.netease.arctic.utils.TablePropertyUtil; import org.apache.commons.collections.CollectionUtils; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/KeyedHiveTable.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/KeyedHiveTable.java index 610a3a617b..777ed77ff5 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/KeyedHiveTable.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/KeyedHiveTable.java @@ -21,11 +21,11 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.api.TableMeta; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.op.BaseSchemaUpdate; import com.netease.arctic.hive.utils.HiveMetaSynchronizer; import com.netease.arctic.io.ArcticFileIO; import com.netease.arctic.io.ArcticHadoopFileIO; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.scan.ChangeTableIncrementalScan; import com.netease.arctic.table.BaseTable; import com.netease.arctic.table.BasicKeyedTable; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/UnkeyedHiveTable.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/UnkeyedHiveTable.java index fa0a9aae23..eb8543958c 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/UnkeyedHiveTable.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/table/UnkeyedHiveTable.java @@ -18,11 +18,10 @@ package com.netease.arctic.hive.table; -import static com.netease.arctic.hive.HiveTableProperties.BASE_HIVE_LOCATION_ROOT; +import static com.netease.arctic.properties.HiveTableProperties.BASE_HIVE_LOCATION_ROOT; import com.netease.arctic.TableFormat; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.op.HiveOperationTransaction; import com.netease.arctic.hive.op.HiveSchemaUpdate; import com.netease.arctic.hive.op.OverwriteHiveFiles; @@ -31,6 +30,7 @@ import com.netease.arctic.hive.utils.HiveMetaSynchronizer; import com.netease.arctic.hive.utils.HiveTableUtil; import com.netease.arctic.io.ArcticHadoopFileIO; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.BaseTable; import com.netease.arctic.table.BasicUnkeyedTable; import com.netease.arctic.table.TableIdentifier; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/CompatibleHivePropertyUtil.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/CompatibleHivePropertyUtil.java index 61aaea1790..e3c67ec38c 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/CompatibleHivePropertyUtil.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/CompatibleHivePropertyUtil.java @@ -18,7 +18,7 @@ package com.netease.arctic.hive.utils; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import org.apache.iceberg.util.PropertyUtil; import java.util.Map; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveMetaSynchronizer.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveMetaSynchronizer.java index b597921366..5e4f737bc3 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveMetaSynchronizer.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveMetaSynchronizer.java @@ -19,10 +19,10 @@ package com.netease.arctic.hive.utils; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.op.OverwriteHiveFiles; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.op.OverwriteBaseFiles; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.table.TableProperties; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveTableUtil.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveTableUtil.java index 737fccce56..a7658e1c2f 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveTableUtil.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/HiveTableUtil.java @@ -19,7 +19,7 @@ package com.netease.arctic.hive.utils; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.utils.IdGenerator; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; diff --git a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/UpgradeHiveTableUtil.java b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/UpgradeHiveTableUtil.java index 298bed51df..d2f7966425 100644 --- a/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/UpgradeHiveTableUtil.java +++ b/mixed-format/hive/src/main/java/com/netease/arctic/hive/utils/UpgradeHiveTableUtil.java @@ -25,12 +25,12 @@ import static com.netease.arctic.table.TableProperties.FILE_FORMAT_PARQUET; import com.netease.arctic.hive.HMSClientPool; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.catalog.ArcticHiveCatalog; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.hive.table.UnkeyedHiveTable; import com.netease.arctic.io.ArcticHadoopFileIO; import com.netease.arctic.op.UpdatePartitionProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.PrimaryKeySpec; import com.netease.arctic.table.TableIdentifier; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/catalog/TestMixedHiveCatalog.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/catalog/TestMixedHiveCatalog.java index 4440e800a3..e070f6e98a 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/catalog/TestMixedHiveCatalog.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/catalog/TestMixedHiveCatalog.java @@ -18,8 +18,8 @@ package com.netease.arctic.hive.catalog; -import static com.netease.arctic.hive.HiveTableProperties.ARCTIC_TABLE_FLAG; -import static com.netease.arctic.hive.HiveTableProperties.ARCTIC_TABLE_ROOT_LOCATION; +import static com.netease.arctic.properties.HiveTableProperties.ARCTIC_TABLE_FLAG; +import static com.netease.arctic.properties.HiveTableProperties.ARCTIC_TABLE_ROOT_LOCATION; import com.netease.arctic.BasicTableTestHelper; import com.netease.arctic.TableFormat; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/io/HiveDataTestHelpers.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/io/HiveDataTestHelpers.java index 2f01315a82..949c844804 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/io/HiveDataTestHelpers.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/io/HiveDataTestHelpers.java @@ -19,13 +19,13 @@ package com.netease.arctic.hive.io; import com.netease.arctic.data.ChangeAction; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.reader.AdaptHiveGenericKeyedDataReader; import com.netease.arctic.hive.io.reader.AdaptHiveGenericUnkeyedDataReader; import com.netease.arctic.hive.io.writer.AdaptHiveGenericTaskWriterBuilder; import com.netease.arctic.hive.table.HiveLocationKind; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.io.MixedDataTestHelpers; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.BaseLocationKind; import com.netease.arctic.table.ChangeLocationKind; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestOverwriteFiles.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestOverwriteFiles.java index c941e1fbe7..54d3b6dace 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestOverwriteFiles.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestOverwriteFiles.java @@ -23,13 +23,13 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.MixedHiveTableTestBase; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.exceptions.CannotAlterHiveLocationException; import com.netease.arctic.hive.io.HiveDataTestHelpers; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.UnkeyedTable; import com.netease.arctic.utils.ArcticTableUtil; import org.apache.hadoop.hive.metastore.api.Table; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewriteFiles.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewriteFiles.java index 9c9a1cc10d..7655a56f0c 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewriteFiles.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewriteFiles.java @@ -23,13 +23,13 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.MixedHiveTableTestBase; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.exceptions.CannotAlterHiveLocationException; import com.netease.arctic.hive.io.HiveDataTestHelpers; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.UnkeyedTable; import com.netease.arctic.utils.ArcticTableUtil; import org.apache.hadoop.hive.metastore.api.Table; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewritePartitions.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewritePartitions.java index bb5dc62746..d4f7c2ce31 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewritePartitions.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestRewritePartitions.java @@ -23,12 +23,12 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.MixedHiveTableTestBase; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.io.HiveDataTestHelpers; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.UnkeyedTable; import com.netease.arctic.utils.ArcticTableUtil; import org.apache.iceberg.DataFile; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestSyncHiveMeta.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestSyncHiveMeta.java index 2e411723b8..427fb9b0ef 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestSyncHiveMeta.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/TestSyncHiveMeta.java @@ -22,13 +22,13 @@ import com.netease.arctic.TableTestHelper; import com.netease.arctic.catalog.CatalogTestHelper; import com.netease.arctic.catalog.TableTestBase; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.io.HiveDataTestHelpers; import com.netease.arctic.hive.utils.HivePartitionUtil; import com.netease.arctic.hive.utils.HiveSchemaUtil; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.UnkeyedTable; import com.netease.arctic.utils.ArcticTableUtil; import com.netease.arctic.utils.TableFileUtil; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/UpdateHiveFilesTestHelpers.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/UpdateHiveFilesTestHelpers.java index b8b4d52e97..1b102bedfa 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/UpdateHiveFilesTestHelpers.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/op/UpdateHiveFilesTestHelpers.java @@ -18,8 +18,8 @@ package com.netease.arctic.hive.op; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.utils.HivePartitionUtil; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.table.UnkeyedTable; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestCompatibleHivePropertyUtil.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestCompatibleHivePropertyUtil.java index 3a54aafebe..43a27a8d3a 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestCompatibleHivePropertyUtil.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestCompatibleHivePropertyUtil.java @@ -18,7 +18,7 @@ package com.netease.arctic.hive.utils; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestHiveMetaSynchronizer.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestHiveMetaSynchronizer.java index 6262af011d..f5b64a508d 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestHiveMetaSynchronizer.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestHiveMetaSynchronizer.java @@ -22,12 +22,12 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.catalog.TableTestBase; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.io.HiveDataTestHelpers; import com.netease.arctic.hive.table.SupportHive; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.UnkeyedTable; import com.netease.arctic.utils.TableFileUtil; diff --git a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestUpgradeHiveTableUtil.java b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestUpgradeHiveTableUtil.java index 46587641a1..bdf2873e4d 100644 --- a/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestUpgradeHiveTableUtil.java +++ b/mixed-format/hive/src/test/java/com/netease/arctic/hive/utils/TestUpgradeHiveTableUtil.java @@ -21,12 +21,12 @@ import com.netease.arctic.TableFormat; import com.netease.arctic.catalog.CatalogTestBase; import com.netease.arctic.catalog.CatalogTestHelper; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.TestHMS; import com.netease.arctic.hive.catalog.ArcticHiveCatalog; import com.netease.arctic.hive.catalog.HiveCatalogTestHelper; import com.netease.arctic.hive.catalog.HiveTableTestHelper; import com.netease.arctic.hive.table.UnkeyedHiveTable; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.utils.TablePropertyUtil; diff --git a/mixed-format/spark/common/src/main/java/com/netease/arctic/spark/mixed/MixedFormatSparkUtil.java b/mixed-format/spark/common/src/main/java/com/netease/arctic/spark/mixed/MixedFormatSparkUtil.java index 9466f117e6..6d77e4385c 100644 --- a/mixed-format/spark/common/src/main/java/com/netease/arctic/spark/mixed/MixedFormatSparkUtil.java +++ b/mixed-format/spark/common/src/main/java/com/netease/arctic/spark/mixed/MixedFormatSparkUtil.java @@ -19,8 +19,8 @@ package com.netease.arctic.spark.mixed; import com.netease.arctic.TableFormat; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.utils.CompatibleHivePropertyUtil; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.TableProperties; import org.apache.spark.sql.connector.catalog.Table; diff --git a/mixed-format/spark/common/src/test/java/com/netease/arctic/spark/test/SparkTestBase.java b/mixed-format/spark/common/src/test/java/com/netease/arctic/spark/test/SparkTestBase.java index f7ab0a83d5..b59ea9b0c1 100644 --- a/mixed-format/spark/common/src/test/java/com/netease/arctic/spark/test/SparkTestBase.java +++ b/mixed-format/spark/common/src/test/java/com/netease/arctic/spark/test/SparkTestBase.java @@ -25,7 +25,7 @@ import com.netease.arctic.UnifiedCatalog; import com.netease.arctic.UnifiedCatalogLoader; import com.netease.arctic.client.ArcticThriftUrl; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.spark.test.utils.TestTableUtil; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; diff --git a/mixed-format/spark/v3.2/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java b/mixed-format/spark/v3.2/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java index ecfaf7d4e3..f7f546f52f 100644 --- a/mixed-format/spark/v3.2/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java +++ b/mixed-format/spark/v3.2/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java @@ -18,12 +18,12 @@ package com.netease.arctic.spark.io; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.writer.AdaptHiveOutputFileFactory; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.io.writer.ChangeTaskWriter; import com.netease.arctic.io.writer.CommonOutputFileFactory; import com.netease.arctic.io.writer.OutputFileFactory; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.KeyedTable; import com.netease.arctic.table.PrimaryKeySpec; diff --git a/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java b/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java index 16f14d9186..f93c100e3a 100644 --- a/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java +++ b/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java @@ -19,7 +19,7 @@ package com.netease.arctic.spark.test.suites.sql; import com.google.common.collect.Maps; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.spark.mixed.SparkSQLProperties; import com.netease.arctic.spark.test.MixedTableTestBase; import com.netease.arctic.spark.test.utils.RecordGenerator; diff --git a/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java b/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java index ab783f2466..f7a0a02111 100644 --- a/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java +++ b/mixed-format/spark/v3.2/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java @@ -18,14 +18,10 @@ package com.netease.arctic.spark.writer; -import static com.netease.arctic.table.TableProperties.BASE_FILE_FORMAT; -import static com.netease.arctic.table.TableProperties.CHANGE_FILE_FORMAT; -import static com.netease.arctic.table.TableProperties.DEFAULT_FILE_FORMAT; - import com.netease.arctic.TableFormat; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.HiveDataTestHelpers; import com.netease.arctic.hive.table.SupportHive; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.spark.io.TaskWriters; import com.netease.arctic.spark.reader.SparkParquetReaders; import com.netease.arctic.spark.test.MixedTableTestBase; @@ -33,6 +29,7 @@ import com.netease.arctic.spark.test.utils.TestTableUtil; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.PrimaryKeySpec; +import com.netease.arctic.table.TableProperties; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -179,9 +176,9 @@ public void testWrite( tableBuilder -> tableBuilder .withPrimaryKeySpec(keySpec) - .withProperty(CHANGE_FILE_FORMAT, fileFormat.name()) - .withProperty(BASE_FILE_FORMAT, fileFormat.name()) - .withProperty(DEFAULT_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.CHANGE_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.BASE_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()) .withPartitionSpec(ptSpec)); Map map = new HashMap<>(); map.put(WriteMode.WRITE_MODE_KEY, writeMode.mode); diff --git a/mixed-format/spark/v3.3/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java b/mixed-format/spark/v3.3/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java index ecfaf7d4e3..f7f546f52f 100644 --- a/mixed-format/spark/v3.3/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java +++ b/mixed-format/spark/v3.3/spark/src/main/java/com/netease/arctic/spark/io/TaskWriters.java @@ -18,12 +18,12 @@ package com.netease.arctic.spark.io; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.writer.AdaptHiveOutputFileFactory; import com.netease.arctic.hive.table.SupportHive; import com.netease.arctic.io.writer.ChangeTaskWriter; import com.netease.arctic.io.writer.CommonOutputFileFactory; import com.netease.arctic.io.writer.OutputFileFactory; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.KeyedTable; import com.netease.arctic.table.PrimaryKeySpec; diff --git a/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java b/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java index 6607850af7..6674253237 100644 --- a/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java +++ b/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/test/suites/sql/TestArcticSessionCatalog.java @@ -19,7 +19,7 @@ package com.netease.arctic.spark.test.suites.sql; import com.google.common.collect.Maps; -import com.netease.arctic.hive.HiveTableProperties; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.spark.mixed.SparkSQLProperties; import com.netease.arctic.spark.test.MixedTableTestBase; import com.netease.arctic.spark.test.utils.RecordGenerator; diff --git a/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java b/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java index ab783f2466..f7a0a02111 100644 --- a/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java +++ b/mixed-format/spark/v3.3/spark/src/test/java/com/netease/arctic/spark/writer/TestMixedWriter.java @@ -18,14 +18,10 @@ package com.netease.arctic.spark.writer; -import static com.netease.arctic.table.TableProperties.BASE_FILE_FORMAT; -import static com.netease.arctic.table.TableProperties.CHANGE_FILE_FORMAT; -import static com.netease.arctic.table.TableProperties.DEFAULT_FILE_FORMAT; - import com.netease.arctic.TableFormat; -import com.netease.arctic.hive.HiveTableProperties; import com.netease.arctic.hive.io.HiveDataTestHelpers; import com.netease.arctic.hive.table.SupportHive; +import com.netease.arctic.properties.HiveTableProperties; import com.netease.arctic.spark.io.TaskWriters; import com.netease.arctic.spark.reader.SparkParquetReaders; import com.netease.arctic.spark.test.MixedTableTestBase; @@ -33,6 +29,7 @@ import com.netease.arctic.spark.test.utils.TestTableUtil; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.PrimaryKeySpec; +import com.netease.arctic.table.TableProperties; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -179,9 +176,9 @@ public void testWrite( tableBuilder -> tableBuilder .withPrimaryKeySpec(keySpec) - .withProperty(CHANGE_FILE_FORMAT, fileFormat.name()) - .withProperty(BASE_FILE_FORMAT, fileFormat.name()) - .withProperty(DEFAULT_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.CHANGE_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.BASE_FILE_FORMAT, fileFormat.name()) + .withProperty(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()) .withPartitionSpec(ptSpec)); Map map = new HashMap<>(); map.put(WriteMode.WRITE_MODE_KEY, writeMode.mode);