Skip to content

Commit

Permalink
[AMORO-2665] Replace deperated classes under ams.server by classes of…
Browse files Browse the repository at this point in the history 
… ams.api (#2666)

* resolve #2665

* fix compiler errors

* fix compiler errors

* spotless apply

* fix compiler error

* follow suggestions

* resolve conflicts with master

* spotless apply

* fix @VisibleForTesting

* remove some illegal imports

* Add @JsonIgnoreProperties(ignoreUnknown = true)

* fix com.google.common reference

---------

Co-authored-by: ZhouJinsong <[email protected]>
  • Loading branch information
@majin1102 @zhoujinsong
majin1102 and zhoujinsong authored Mar 26, 2024
1 parent 48ce96f commit e610bca
Show file tree
Hide file tree
Showing 129 changed files with 804 additions and 1,660 deletions.
6 changes: 3 additions & 3 deletions ...rctic/server/utils/ConfigurationUtil.java → ...ease/arctic/api/config/ConfigHelpers.java
100755 → 100644
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* limitations under the License.
*/

package com.netease.arctic.server.utils;
package com.netease.arctic.api.config;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
Expand All @@ -38,12 +38,12 @@
import java.util.stream.IntStream;

/** Utility class for {@link Configurations} related helper functions. */
public class ConfigurationUtil {
public class ConfigHelpers {

private static final String[] EMPTY = new String[0];

// Make sure that we cannot instantiate this class
private ConfigurationUtil() {}
private ConfigHelpers() {}

/**
* Creates a new {@link Configurations} from the given {@link Properties}.
Expand Down
7 changes: 1 addition & 6 deletions ...ase/arctic/server/utils/ConfigOption.java → ...tease/arctic/api/config/ConfigOption.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,15 @@
* limitations under the License.
*/

package com.netease.arctic.server.utils;
package com.netease.arctic.api.config;

import static com.google.common.base.Preconditions.checkNotNull;

import com.netease.arctic.server.ArcticManagementConf;

/**
* A {@code ConfigOption} describes a configuration parameter. It encapsulates the configuration
* key, deprecated older versions of the key, and an optional default value for the configuration
* parameter.
*
* <p>{@code ConfigOptions} are built via the {@link ArcticManagementConf} class. Once created, a
* config option is immutable.
*
* @param <T> The type of value associated with the configuration option.
*/
public class ConfigOption<T> {
Expand Down
2 changes: 1 addition & 1 deletion ...se/arctic/server/utils/ConfigOptions.java → ...ease/arctic/api/config/ConfigOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* limitations under the License.
*/

package com.netease.arctic.server.utils;
package com.netease.arctic.api.config;

import static com.google.common.base.Preconditions.checkNotNull;

Expand Down
10 changes: 5 additions & 5 deletions ...e/arctic/server/utils/Configurations.java → ...ase/arctic/api/config/Configurations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* limitations under the License.
*/

package com.netease.arctic.server.utils;
package com.netease.arctic.api.config;

import static com.google.common.base.Preconditions.checkNotNull;

Expand Down Expand Up @@ -433,7 +433,7 @@ public <T extends Enum<T>> T getEnum(

Object rawValue = getRawValueFromOption(configOption).orElseGet(configOption::defaultValue);
try {
return ConfigurationUtil.convertToEnum(rawValue, enumClass);
return ConfigHelpers.convertToEnum(rawValue, enumClass);
} catch (IllegalArgumentException ex) {
final String errorMessage =
String.format(
Expand Down Expand Up @@ -538,9 +538,9 @@ public <T> Optional<T> getOptional(ConfigOption<T> option) {

try {
if (option.isList()) {
return rawValue.map(v -> ConfigurationUtil.convertToList(v, clazz));
return rawValue.map(v -> ConfigHelpers.convertToList(v, clazz));
} else {
return rawValue.map(v -> ConfigurationUtil.convertValue(v, clazz));
return rawValue.map(v -> ConfigHelpers.convertValue(v, clazz));
}
} catch (Exception e) {
throw new IllegalArgumentException(
Expand All @@ -562,7 +562,7 @@ public Map<String, String> toMap() {
synchronized (this.confData) {
Map<String, String> ret = new HashMap<>(this.confData.size());
for (Map.Entry<String, Object> entry : confData.entrySet()) {
ret.put(entry.getKey(), ConfigurationUtil.convertToString(entry.getValue()));
ret.put(entry.getKey(), ConfigHelpers.convertToString(entry.getValue()));
}
return ret;
}
Expand Down
139 changes: 123 additions & 16 deletions ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
package com.netease.arctic.api.config;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.netease.arctic.table.ArcticTable;
import com.netease.arctic.table.TableProperties;
import com.netease.arctic.utils.CompatiblePropertyUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Objects;
Expand All @@ -30,6 +34,7 @@
import org.slf4j.LoggerFactory;

import java.util.Locale;
import java.util.Map;
import java.util.Set;

/** Data expiration configuration. */
Expand All @@ -40,15 +45,19 @@ public class DataExpirationConfig {
// data-expire.field
private String expirationField;
// data-expire.level
@JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)
private ExpireLevel expirationLevel;
// data-expire.retention-time
private long retentionTime;
// data-expire.datetime-string-pattern
@JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)
private String dateTimePattern;
// data-expire.datetime-number-format
@JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)
private String numberDateFormat;
// data-expire.since
private Since since;
// data-expire.base-on-rule
@JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT)
private BaseOnRule baseOnRule;

@VisibleForTesting
public enum ExpireLevel {
Expand All @@ -66,14 +75,15 @@ public static ExpireLevel fromString(String level) {
}

@VisibleForTesting
public enum Since {
LATEST_SNAPSHOT,
CURRENT_TIMESTAMP;
public enum BaseOnRule {
LAST_COMMIT_TIME,
CURRENT_TIME;

public static Since fromString(String since) {
Preconditions.checkArgument(null != since, "data-expire.since is invalid: null");
public static BaseOnRule fromString(String since) {
Preconditions.checkArgument(
null != since, TableProperties.DATA_EXPIRATION_BASE_ON_RULE + " is invalid: null");
try {
return Since.valueOf(since.toUpperCase(Locale.ENGLISH));
return BaseOnRule.valueOf(since.toUpperCase(Locale.ENGLISH));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format("Unable to expire data since: %s", since), e);
Expand All @@ -95,14 +105,111 @@ public DataExpirationConfig(
long retentionTime,
String dateTimePattern,
String numberDateFormat,
Since since) {
BaseOnRule baseOnRule) {
this.enabled = enabled;
this.expirationField = expirationField;
this.expirationLevel = expirationLevel;
this.retentionTime = retentionTime;
this.dateTimePattern = dateTimePattern;
this.numberDateFormat = numberDateFormat;
this.since = since;
this.baseOnRule = baseOnRule;
}

public DataExpirationConfig(ArcticTable table) {
Map<String, String> properties = table.properties();
expirationField =
CompatiblePropertyUtil.propertyAsString(
properties, TableProperties.DATA_EXPIRATION_FIELD, null);
Types.NestedField field = table.schema().findField(expirationField);
Preconditions.checkArgument(
StringUtils.isNoneBlank(expirationField) && null != field,
String.format(
"Field(%s) used to determine data expiration is illegal for table(%s)",
expirationField, table.name()));
Type.TypeID typeID = field.type().typeId();
Preconditions.checkArgument(
FIELD_TYPES.contains(typeID),
String.format(
"The type(%s) of filed(%s) is incompatible for table(%s)",
typeID.name(), expirationField, table.name()));

expirationLevel =
ExpireLevel.fromString(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_LEVEL,
TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT));

String retention =
CompatiblePropertyUtil.propertyAsString(
properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null);
if (StringUtils.isNotBlank(retention)) {
retentionTime = ConfigHelpers.TimeUtils.parseDuration(retention).toMillis();
}

dateTimePattern =
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN,
TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT);
numberDateFormat =
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT,
TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT);
baseOnRule =
BaseOnRule.fromString(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_BASE_ON_RULE,
TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT));
}

public static DataExpirationConfig parse(Map<String, String> properties) {
boolean gcEnabled =
CompatiblePropertyUtil.propertyAsBoolean(
properties, org.apache.iceberg.TableProperties.GC_ENABLED, true);
DataExpirationConfig config =
new DataExpirationConfig()
.setEnabled(
gcEnabled
&& CompatiblePropertyUtil.propertyAsBoolean(
properties,
TableProperties.ENABLE_DATA_EXPIRATION,
TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT))
.setExpirationLevel(
ExpireLevel.fromString(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_LEVEL,
TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)))
.setExpirationField(
CompatiblePropertyUtil.propertyAsString(
properties, TableProperties.DATA_EXPIRATION_FIELD, null))
.setDateTimePattern(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN,
TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT))
.setNumberDateFormat(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT,
TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT))
.setBaseOnRule(
BaseOnRule.fromString(
CompatiblePropertyUtil.propertyAsString(
properties,
TableProperties.DATA_EXPIRATION_BASE_ON_RULE,
TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT)));
String retention =
CompatiblePropertyUtil.propertyAsString(
properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null);
if (StringUtils.isNotBlank(retention)) {
config.setRetentionTime(ConfigHelpers.TimeUtils.parseDuration(retention).toMillis());
}

return config;
}

public boolean isEnabled() {
Expand Down Expand Up @@ -159,12 +266,12 @@ public DataExpirationConfig setNumberDateFormat(String numberDateFormat) {
return this;
}

public Since getSince() {
return since;
public BaseOnRule getBaseOnRule() {
return baseOnRule;
}

public DataExpirationConfig setSince(Since since) {
this.since = since;
public DataExpirationConfig setBaseOnRule(BaseOnRule baseOnRule) {
this.baseOnRule = baseOnRule;
return this;
}

Expand All @@ -183,7 +290,7 @@ public boolean equals(Object o) {
&& expirationLevel == config.expirationLevel
&& Objects.equal(dateTimePattern, config.dateTimePattern)
&& Objects.equal(numberDateFormat, config.numberDateFormat)
&& since == config.since;
&& baseOnRule == config.baseOnRule;
}

@Override
Expand All @@ -195,7 +302,7 @@ public int hashCode() {
retentionTime,
dateTimePattern,
numberDateFormat,
since);
baseOnRule);
}

public boolean isValid(Types.NestedField field, String name) {
Expand Down
Loading

0 comments on commit e610bca

Please sign in to comment.