Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AMORO-3118][Improvement]: Move iceberg codes from amoro-core to a new module amoro-iceberg #3108

Merged
merged 3 commits into from
Aug 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions amoro-ams/amoro-ams-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,19 @@
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.CompatiblePropertyUtil;
import org.apache.iceberg.util.PropertyUtil;
import org.apache.amoro.utils.PropertyUtil;

import java.util.Map;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ public int hashCode() {
}

public static TableConfiguration parseConfig(Map<String, String> properties) {
boolean gcEnabled =
CompatiblePropertyUtil.propertyAsBoolean(
properties, org.apache.iceberg.TableProperties.GC_ENABLED, true);
boolean gcEnabled = CompatiblePropertyUtil.propertyAsBoolean(properties, "gc.enabled", true);
return new TableConfiguration()
.setExpireSnapshotEnabled(
gcEnabled
Expand Down
13 changes: 13 additions & 0 deletions amoro-ams/amoro-ams-server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@
<artifactId>amoro-ams-api</artifactId>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-mixed-format-hive</artifactId>
Expand Down Expand Up @@ -351,6 +356,14 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-ams-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.amoro.utils.TablePropertyUtil;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -435,7 +435,7 @@ private InternalCatalog getCatalog(String catalog) {
ServerCatalog internalCatalog = tableService.getServerCatalog(catalog);
Preconditions.checkArgument(
internalCatalog instanceof InternalCatalog, "The catalog is not an iceberg rest catalog");
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(internalCatalog.getMetadata());
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(internalCatalog.getMetadata());
Preconditions.checkArgument(
tableFormats.size() == 1
&& (tableFormats.contains(TableFormat.ICEBERG)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;

import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -68,7 +68,7 @@ private static String getAmsURI(Configurations serviceConfig) {
public static ServerCatalog buildServerCatalog(
CatalogMeta catalogMeta, Configurations serverConfiguration) {
String type = catalogMeta.getCatalogType();
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(catalogMeta);

Preconditions.checkState(
formatSupportedMatrix.containsKey(type), "unsupported catalog type: %s", type);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -46,7 +46,7 @@ public class ExternalCatalog extends ServerCatalog {

protected ExternalCatalog(CatalogMeta metadata) {
super(metadata);
this.tableMetaStore = MixedCatalogUtil.buildMetaStore(metadata);
this.tableMetaStore = CatalogUtil.buildMetaStore(metadata);
this.unifiedCatalog =
this.tableMetaStore.doAs(
() -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap()));
Expand Down Expand Up @@ -76,7 +76,7 @@ public void disposeTable(String database, String tableName) {
@Override
public void updateMetadata(CatalogMeta metadata) {
super.updateMetadata(metadata);
this.tableMetaStore = MixedCatalogUtil.buildMetaStore(metadata);
this.tableMetaStore = CatalogUtil.buildMetaStore(metadata);
this.unifiedCatalog.refresh();
updateDatabaseFilter(metadata);
updateTableFilter(metadata);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.apache.amoro.server.table.internal.InternalTableHandler;
import org.apache.amoro.server.utils.InternalTableUtil;
import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -100,7 +100,7 @@ public AmoroTable<?> loadTable(String database, String tableName) {
return IcebergTable.newIcebergTable(
tableIdentifier,
table,
MixedCatalogUtil.buildMetaStore(getMetadata()),
CatalogUtil.buildMetaStore(getMetadata()),
getMetadata().getCatalogProperties());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.amoro.server.table.TableMetadata;
import org.apache.amoro.server.table.internal.InternalTableCreator;
import org.apache.amoro.server.table.internal.InternalTableHandler;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.iceberg.rest.requests.CreateTableRequest;
import org.apache.thrift.TException;
Expand All @@ -44,7 +44,7 @@ protected MixedHiveCatalogImpl(CatalogMeta catalogMeta) {
super(catalogMeta);
this.tables =
new MixedHiveTables(
catalogMeta.getCatalogProperties(), MixedCatalogUtil.buildMetaStore(catalogMeta));
catalogMeta.getCatalogProperties(), CatalogUtil.buildMetaStore(catalogMeta));
hiveClientPool = ((MixedHiveTables) tables()).getHiveClientPool();
}

Expand All @@ -53,8 +53,7 @@ public void updateMetadata(CatalogMeta metadata) {
super.updateMetadata(metadata);
hiveClientPool = ((MixedHiveTables) tables()).getHiveClientPool();
this.tables =
new MixedHiveTables(
metadata.getCatalogProperties(), MixedCatalogUtil.buildMetaStore(metadata));
new MixedHiveTables(metadata.getCatalogProperties(), CatalogUtil.buildMetaStore(metadata));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.AwsClientProperties;
Expand Down Expand Up @@ -249,12 +249,12 @@ private void fillAuthConfigs2CatalogMeta(
metaAuthConfig.put(
AUTH_CONFIGS_KEY_SECRET_KEY, serverAuthConfig.get(AUTH_CONFIGS_KEY_SECRET_KEY));

MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
serverAuthConfig,
catalogMeta.getCatalogProperties(),
AUTH_CONFIGS_KEY_ACCESS_KEY,
S3FileIOProperties.ACCESS_KEY_ID);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
serverAuthConfig,
catalogMeta.getCatalogProperties(),
AUTH_CONFIGS_KEY_SECRET_KEY,
Expand Down Expand Up @@ -295,12 +295,12 @@ private Map<String, Object> extractAuthConfigsFromCatalogMeta(
catalogName, CONFIG_TYPE_AUTH, AUTH_CONFIGS_KEY_KRB5.replace("\\.", "-"))));
break;
case AUTH_CONFIGS_VALUE_TYPE_AK_SK:
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
serverAuthConfig,
S3FileIOProperties.ACCESS_KEY_ID,
AUTH_CONFIGS_KEY_ACCESS_KEY);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
serverAuthConfig,
S3FileIOProperties.SECRET_ACCESS_KEY,
Expand All @@ -315,7 +315,7 @@ private Map<String, Object> extractStorageConfigsFromCatalogMeta(
String catalogName, CatalogMeta catalogMeta) {
Map<String, Object> storageConfig = new HashMap<>();
Map<String, String> config = catalogMeta.getStorageConfigs();
String storageType = MixedCatalogUtil.getCompatibleStorageType(config);
String storageType = CatalogUtil.getCompatibleStorageType(config);
storageConfig.put(STORAGE_CONFIGS_KEY_TYPE, storageType);
if (STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equals(storageType)) {
storageConfig.put(
Expand Down Expand Up @@ -345,12 +345,12 @@ private Map<String, Object> extractStorageConfigsFromCatalogMeta(
CONFIG_TYPE_STORAGE,
STORAGE_CONFIGS_KEY_HIVE_SITE.replace("\\.", "-"))));
} else if (STORAGE_CONFIGS_VALUE_TYPE_S3.equals(storageType)) {
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
storageConfig,
AwsClientProperties.CLIENT_REGION,
STORAGE_CONFIGS_KEY_REGION);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
storageConfig,
S3FileIOProperties.ENDPOINT,
Expand Down Expand Up @@ -421,12 +421,12 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o
}
}
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_S3)) {
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
info.getStorageConfig(),
catalogMeta.getCatalogProperties(),
STORAGE_CONFIGS_KEY_REGION,
AwsClientProperties.CLIENT_REGION);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
info.getStorageConfig(),
catalogMeta.getCatalogProperties(),
STORAGE_CONFIGS_KEY_ENDPOINT,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
Expand Down Expand Up @@ -171,7 +171,7 @@ public void getHiveTableDetail(Context ctx) {
"catalog.database.tableName can not be empty in any element");
ServerCatalog serverCatalog = tableService.getServerCatalog(catalog);
CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
HMSClientPool hmsClientPool =
new CachedHiveClientPool(tableMetaStore, catalogMeta.getCatalogProperties());

Expand Down Expand Up @@ -212,9 +212,9 @@ public void upgradeHiveTable(Context ctx) {
CatalogMeta catalogMeta = serverCatalog.getMetadata();
String amsUri = AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_TABLE_SERVICE_NAME);
catalogMeta.putToCatalogProperties(CatalogMetaProperties.AMS_URI, amsUri);
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
// check whether catalog support MIXED_HIVE format.
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(catalogMeta);
Preconditions.checkState(
tableFormats.contains(TableFormat.MIXED_HIVE),
"Catalog %s does not support MIXED_HIVE format",
Expand Down Expand Up @@ -527,7 +527,7 @@ public void getTableList(Context ctx) {
String catalogType = serverCatalog.getMetadata().getCatalogType();
if (catalogType.equals(CATALOG_TYPE_HIVE)) {
CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
HMSClientPool hmsClientPool =
new CachedHiveClientPool(tableMetaStore, catalogMeta.getCatalogProperties());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.PrimaryKeySpec;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.amoro.utils.TablePropertyUtil;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -65,7 +65,7 @@ private TableOperations newTableOperations(boolean changeStore) {
if (InternalTableUtil.isLegacyMixedIceberg(tableMetadata())) {
String tableLocation =
changeStore ? tableMetadata().getChangeLocation() : tableMetadata().getBaseLocation();
TableMetaStore metaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore metaStore = CatalogUtil.buildMetaStore(catalogMeta);

MixedHadoopTableOperations ops =
new MixedHadoopTableOperations(new Path(tableLocation), io, metaStore.getConfiguration());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.CatalogProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -251,7 +251,7 @@ public void dispose() {

private String catalogConnectorType(CatalogMeta catalogMeta) {
String catalogType = catalogMeta.getCatalogType();
Set<TableFormat> tableFormatSet = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormatSet = CatalogUtil.tableFormats(catalogMeta);

if (catalogType.equalsIgnoreCase(CatalogType.AMS.name())) {
if (tableFormatSet.contains(TableFormat.MIXED_ICEBERG)) {
Expand Down Expand Up @@ -294,7 +294,7 @@ private TableMetaStore getCatalogTableMetaStore(CatalogMeta catalogMeta) {
if (catalogMeta.getStorageConfigs() != null) {
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
MixedCatalogUtil.getCompatibleStorageType(storageConfigs))) {
CatalogUtil.getCompatibleStorageType(storageConfigs))) {
builder
.withBase64MetaStoreSite(
catalogMeta
Expand Down Expand Up @@ -379,7 +379,7 @@ private TerminalSessionFactory loadTerminalSessionFactory(Configurations conf) {
}

private void applyClientProperties(CatalogMeta catalogMeta) {
Set<TableFormat> formats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> formats = CatalogUtil.tableFormats(catalogMeta);
String catalogType = catalogMeta.getCatalogType();
if (formats.contains(TableFormat.ICEBERG)) {
if (CatalogMetaProperties.CATALOG_TYPE_AMS.equalsIgnoreCase(catalogType)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import org.apache.amoro.io.AuthenticatedFileIOs;
import org.apache.amoro.properties.CatalogMetaProperties;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
Expand Down Expand Up @@ -82,7 +82,7 @@ public static boolean isKeyedMixedTable(
*/
public static AuthenticatedFileIO newIcebergFileIo(CatalogMeta meta) {
Map<String, String> catalogProperties = meta.getCatalogProperties();
TableMetaStore store = MixedCatalogUtil.buildMetaStore(meta);
TableMetaStore store = CatalogUtil.buildMetaStore(meta);
Configuration conf = store.getConfiguration();
String warehouse = meta.getCatalogProperties().get(CatalogMetaProperties.KEY_WAREHOUSE);
String defaultImpl = HADOOP_FILE_IO_IMPL;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@
import org.apache.amoro.table.PrimaryKeySpec;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.iceberg.CatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.rest.RESTCatalog;
Expand Down Expand Up @@ -99,10 +98,10 @@ protected RESTCatalog loadIcebergCatalog(Map<String, String> clientProperties) {
clientProperties.putIfAbsent("warehouse", catalogName());

CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore store = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore store = CatalogUtil.buildMetaStore(catalogMeta);

return (RESTCatalog)
CatalogUtil.loadCatalog(
org.apache.iceberg.CatalogUtil.loadCatalog(
"org.apache.iceberg.rest.RESTCatalog",
"test",
clientProperties,
Expand Down
Loading
Loading