Skip to content

Commit

Permalink
[feature](external) process tbl/db exist when create/drop db/tbl (apa…
Browse files Browse the repository at this point in the history 
…che#33119)

Issue Number: apache#31442
  • Loading branch information
@wsjz
wsjz authored Apr 3, 2024
1 parent 962aef9 commit 10ff3f4
Show file tree
Hide file tree
Showing 8 changed files with 198 additions and 39 deletions.
51 changes: 39 additions & 12 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.jdbc.client.JdbcClient;
Expand All @@ -40,7 +42,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -99,6 +100,14 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
String fullDbName = stmt.getFullDbName();
Map<String, String> properties = stmt.getProperties();
long dbId = Env.getCurrentEnv().getNextId();
if (databaseExist(fullDbName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create database[{}] which already exists", fullDbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, fullDbName);
}
}
try {
HiveDatabaseMetadata catalogDatabase = new HiveDatabaseMetadata();
catalogDatabase.setDbName(fullDbName);
Expand All @@ -119,6 +128,14 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
@Override
public void dropDb(DropDbStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
if (!databaseExist(dbName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop database[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
}
}
try {
client.dropDatabase(dbName);
catalog.onRefresh(true);
Expand All @@ -135,6 +152,14 @@ public void createTable(CreateTableStmt stmt) throws UserException {
if (db == null) {
throw new UserException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
}
if (tableExist(dbName, tblName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create table[{}] which already exists", tblName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tblName);
}
}
try {
Map<String, String> props = stmt.getProperties();
String fileFormat = props.getOrDefault(FILE_FORMAT_KEY, Config.hive_default_file_format);
Expand Down Expand Up @@ -186,24 +211,21 @@ public void createTable(CreateTableStmt stmt) throws UserException {
}
}

private static List<FieldSchema> parsePartitionKeys(Map<String, String> props) {
List<FieldSchema> parsedKeys = new ArrayList<>();
String pkStr = props.getOrDefault("partition_keys", "");
if (pkStr.isEmpty()) {
return parsedKeys;
} else {
// TODO: parse string to partition keys list
return parsedKeys;
}
}

@Override
public void dropTable(DropTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
ExternalDatabase<?> db = catalog.getDbNullable(stmt.getDbName());
if (db == null) {
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
}
if (!tableExist(dbName, stmt.getTableName())) {
if (stmt.isSetIfExists()) {
LOG.info("drop table[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE, stmt.getTableName(), dbName);
}
}
try {
client.dropTable(dbName, stmt.getTableName());
db.setUnInitialized(true);
Expand All @@ -222,6 +244,11 @@ public boolean tableExist(String dbName, String tblName) {
return client.tableExists(dbName, tblName);
}

@Override
public boolean databaseExist(String dbName) {
return listDatabaseNames().contains(dbName);
}

public List<String> listDatabaseNames() {
return client.getAllDatabases();
}
Expand Down
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
Expand Down Expand Up @@ -255,4 +256,15 @@ private static Pair<List<FieldSchema>, List<FieldSchema>> toHiveSchema(List<Colu
}
return Pair.of(hiveCols, hiveParts);
}

public static Database toHiveDatabase(HiveDatabaseMetadata hiveDb) {
Database database = new Database();
database.setName(hiveDb.getDbName());
if (StringUtils.isNotEmpty(hiveDb.getLocationUri())) {
database.setLocationUri(hiveDb.getLocationUri());
}
database.setParameters(hiveDb.getProperties());
database.setDescription(hiveDb.getComment());
return database;
}
}
14 changes: 1 addition & 13 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnList;
Expand Down Expand Up @@ -137,7 +136,7 @@ public void createDatabase(DatabaseMetadata db) {
if (db instanceof HiveDatabaseMetadata) {
HiveDatabaseMetadata hiveDb = (HiveDatabaseMetadata) db;
ugiDoAs(() -> {
client.client.createDatabase(toHiveDatabase(hiveDb));
client.client.createDatabase(HiveUtil.toHiveDatabase(hiveDb));
return null;
});
}
Expand All @@ -150,17 +149,6 @@ public void createDatabase(DatabaseMetadata db) {
}
}

private static Database toHiveDatabase(HiveDatabaseMetadata hiveDb) {
Database database = new Database();
database.setName(hiveDb.getDbName());
if (StringUtils.isNotEmpty(hiveDb.getLocationUri())) {
database.setLocationUri(hiveDb.getLocationUri());
}
database.setParameters(hiveDb.getProperties());
database.setDescription(hiveDb.getComment());
return database;
}

@Override
public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
if (tableExists(tbl.getDbName(), tbl.getTableName())) {
Expand Down
38 changes: 38 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.DorisTypeVisitor;
import org.apache.doris.datasource.ExternalCatalog;
Expand Down Expand Up @@ -67,6 +69,10 @@ public boolean tableExist(String dbName, String tblName) {
return catalog.tableExists(TableIdentifier.of(dbName, tblName));
}

public boolean databaseExist(String dbName) {
return nsCatalog.namespaceExists(Namespace.of(dbName));
}

public List<String> listDatabaseNames() {
return nsCatalog.listNamespaces().stream()
.map(e -> e.toString())
Expand All @@ -84,6 +90,14 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
String dbName = stmt.getFullDbName();
Map<String, String> properties = stmt.getProperties();
if (databaseExist(dbName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create database[{}] which already exists", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
}
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefresh(true);
}
Expand All @@ -97,6 +111,14 @@ public void dropDb(DropDbStmt stmt) throws DdlException {
dorisCatalog.getIdToDb().remove(aLong);
dorisCatalog.getDbNameToId().remove(dbName);
}
if (!databaseExist(dbName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop database[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
}
}
nsCatalog.dropNamespace(Namespace.of(dbName));
dorisCatalog.onRefresh(true);
}
Expand All @@ -109,6 +131,14 @@ public void createTable(CreateTableStmt stmt) throws UserException {
throw new UserException("Failed to get database: '" + dbName + "' in catalog: " + dorisCatalog.getName());
}
String tableName = stmt.getTableName();
if (tableExist(dbName, tableName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create table[{}] which already exists", tableName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
}
}
List<Column> columns = stmt.getColumns();
List<StructField> collect = columns.stream()
.map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull()))
Expand All @@ -132,6 +162,14 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + dorisCatalog.getName());
}
String tableName = stmt.getTableName();
if (!tableExist(dbName, tableName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop table[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE, tableName, dbName);
}
}
catalog.dropTable(TableIdentifier.of(dbName, tableName));
db.setUnInitialized(true);
}
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,6 @@ public interface ExternalMetadataOps {
* @return
*/
boolean tableExist(String dbName, String tblName);

boolean databaseExist(String dbName);
}
Loading

0 comments on commit 10ff3f4

Please sign in to comment.