Skip to content

Commit

Permalink
HIVE-1537. Allow users to specify LOCATION in CREATE DATABASE stateme…
Browse files Browse the repository at this point in the history 
…nt. Contributed by Thiruvel Thirumoolan

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1145053 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
Amareshwari Sri Ramadasu committed Jul 11, 2011
1 parent 1ee2e63 commit ea2fe2e
Show file tree
Hide file tree
Showing 12 changed files with 355 additions and 43 deletions.
8 changes: 7 additions & 1 deletion metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;

/**
* Hive specific implementation of alter
Expand Down Expand Up @@ -115,7 +116,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname,
// that means user is asking metastore to move data to new location
// corresponding to the new name
// get new location
newTblLoc = wh.getDefaultTablePath(newt.getDbName(), newt.getTableName()).toString();
newTblLoc = wh.getTablePath(msdb.getDatabase(newt.getDbName()), newt.getTableName()).toString();
newt.getSd().setLocation(newTblLoc);
oldTblLoc = oldt.getSd().getLocation();
moveData = true;
Expand Down Expand Up @@ -176,6 +177,11 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname,
throw new InvalidOperationException(
"Unable to change partition or table."
+ " Check metastore logs for detailed stack." + e.getMessage());
} catch (NoSuchObjectException e) {
LOG.debug(e);
throw new InvalidOperationException(
"Unable to change partition or table. Database " + dbname + " does not exist"
+ " Check metastore logs for detailed stack." + e.getMessage());
} finally {
if (!success) {
msdb.rollbackTransaction();
Expand Down
42 changes: 32 additions & 10 deletions metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObje
} catch (NoSuchObjectException e) {
ms.createDatabase(
new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT,
wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null));
getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null));
}
HMSHandler.createDefaultDB = true;
}
Expand Down Expand Up @@ -570,31 +570,53 @@ public void shutdown() {
logInfo("Metastore shutdown complete.");
}

private static final String DATABASE_WAREHOUSE_SUFFIX = ".db";

private Path getDefaultDatabasePath(String dbName) throws MetaException {
if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
return wh.getWhRoot();
}
return new Path(wh.getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
}

private void create_database_core(RawStore ms, final Database db)
throws AlreadyExistsException, InvalidObjectException, MetaException,
IOException {
if (!validateName(db.getName())) {
throw new InvalidObjectException(db.getName() + " is not a valid database name");
}
if (null == db.getLocationUri()) {
db.setLocationUri(getDefaultDatabasePath(db.getName()).toString());
} else {
db.setLocationUri(wh.getDnsPath(new Path(db.getLocationUri())).toString());
}
Path dbPath = new Path(db.getLocationUri());
boolean success = false;
boolean madeDir = false;
try {
ms.openTransaction();
if (null == db.getLocationUri()) {
db.setLocationUri(wh.getDefaultDatabasePath(db.getName()).toString());
if (!wh.isDir(dbPath)) {
if (!wh.mkdirs(dbPath)) {
throw new MetaException("Unable to create database path " + dbPath +
", failed to create database " + db.getName());
}
madeDir = true;
}

ms.openTransaction();
ms.createDatabase(db);
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
} else {
wh.mkdirs(new Path(db.getLocationUri()));
if (madeDir) {
wh.deleteDir(dbPath, true);
}
}
for (MetaStoreEventListener listener : listeners) {
listener.onCreateDatabase(new CreateDatabaseEvent(db, success, this));
}
}
}
}

public void create_database(final Database db)
throws AlreadyExistsException, InvalidObjectException, MetaException {
Expand Down Expand Up @@ -923,7 +945,7 @@ public Map<String, Type> get_type_all(String name) throws MetaException {
}

private void create_table_core(final RawStore ms, final Table tbl)
throws AlreadyExistsException, MetaException, InvalidObjectException {
throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException {

if (!MetaStoreUtils.validateName(tbl.getTableName())
|| !MetaStoreUtils.validateColNames(tbl.getSd().getCols())
Expand All @@ -947,8 +969,8 @@ private void create_table_core(final RawStore ms, final Table tbl)
if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
if (tbl.getSd().getLocation() == null
|| tbl.getSd().getLocation().isEmpty()) {
tblPath = wh.getDefaultTablePath(
tbl.getDbName(), tbl.getTableName());
tblPath = wh.getTablePath(
ms.getDatabase(tbl.getDbName()), tbl.getTableName());
} else {
if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) {
LOG.warn("Location: " + tbl.getSd().getLocation()
Expand Down
19 changes: 9 additions & 10 deletions metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.shims.ShimLoader;
Expand All @@ -59,8 +60,6 @@ public class Warehouse {
private final Configuration conf;
private final String whRootString;

private static final String DATABASE_WAREHOUSE_SUFFIX = ".db";

public static final Log LOG = LogFactory.getLog("hive.metastore.warehouse");

private MetaStoreFS fsHandler = null;
Expand Down Expand Up @@ -143,7 +142,7 @@ public Path getDnsPath(Path path) throws MetaException {
* dir (but that should be ok given that this is only called during DDL
* statements for non-external tables).
*/
private Path getWhRoot() throws MetaException {
public Path getWhRoot() throws MetaException {
if (whRoot != null) {
return whRoot;
}
Expand All @@ -156,16 +155,16 @@ public Path getTablePath(String whRootString, String tableName) throws MetaExcep
return new Path(whRoot, tableName.toLowerCase());
}

public Path getDefaultDatabasePath(String dbName) throws MetaException {
if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
public Path getDatabasePath(Database db) throws MetaException {
if (db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
return getWhRoot();
}
return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
return new Path(db.getLocationUri());
}

public Path getDefaultTablePath(String dbName, String tableName)
public Path getTablePath(Database db, String tableName)
throws MetaException {
return new Path(getDefaultDatabasePath(dbName), tableName.toLowerCase());
return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase()));
}

public boolean mkdirs(Path f) throws MetaException {
Expand Down Expand Up @@ -393,9 +392,9 @@ public static void makeSpecFromName(Map<String, String> partSpec, Path currPath)
}
}

public Path getPartitionPath(String dbName, String tableName,
public Path getPartitionPath(Database db, String tableName,
LinkedHashMap<String, String> pm) throws MetaException {
return new Path(getDefaultTablePath(dbName, tableName), makePartPath(pm));
return new Path(getTablePath(db, tableName), makePartPath(pm));
}

public Path getPartitionPath(Path tblPath, LinkedHashMap<String, String> pm)
Expand Down
149 changes: 147 additions & 2 deletions metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
Expand Down Expand Up @@ -473,7 +474,7 @@ public void testDatabase() throws Throwable {
assertEquals("name of returned db is different from that of inserted db",
TEST_DB1_NAME, db.getName());
assertEquals("location of the returned db is different from that of inserted db",
warehouse.getDefaultDatabasePath(TEST_DB1_NAME).toString(), db.getLocationUri());
warehouse.getDatabasePath(db).toString(), db.getLocationUri());

Database db2 = new Database();
db2.setName(TEST_DB2_NAME);
Expand All @@ -484,7 +485,7 @@ public void testDatabase() throws Throwable {
assertEquals("name of returned db is different from that of inserted db",
TEST_DB2_NAME, db2.getName());
assertEquals("location of the returned db is different from that of inserted db",
warehouse.getDefaultDatabasePath(TEST_DB2_NAME).toString(), db2.getLocationUri());
warehouse.getDatabasePath(db2).toString(), db2.getLocationUri());

List<String> dbs = client.getDatabases(".*");

Expand All @@ -502,6 +503,96 @@ public void testDatabase() throws Throwable {
}
}

public void testDatabaseLocation() throws Throwable {
try {
// clear up any existing databases
silentDropDatabase(TEST_DB1_NAME);

Database db = new Database();
db.setName(TEST_DB1_NAME);
String dbLocation =
HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/_testDB_create_";
db.setLocationUri(dbLocation);
client.createDatabase(db);

db = client.getDatabase(TEST_DB1_NAME);

assertEquals("name of returned db is different from that of inserted db",
TEST_DB1_NAME, db.getName());
assertEquals("location of the returned db is different from that of inserted db",
warehouse.getDnsPath(new Path(dbLocation)).toString(), db.getLocationUri());

client.dropDatabase(TEST_DB1_NAME);
silentDropDatabase(TEST_DB1_NAME);

db = new Database();
db.setName(TEST_DB1_NAME);
dbLocation =
HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test/_testDB_create_";
FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), hiveConf);
fs.mkdirs(
new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"),
new FsPermission((short) 0));
db.setLocationUri(dbLocation);

boolean createFailed = false;
try {
client.createDatabase(db);
} catch (MetaException cantCreateDB) {
createFailed = true;
}
assertTrue("Database creation succeeded even with permission problem", createFailed);

boolean objectNotExist = false;
try {
client.getDatabase(TEST_DB1_NAME);
} catch (NoSuchObjectException e) {
objectNotExist = true;
}
assertTrue("Database " + TEST_DB1_NAME + " exists ", objectNotExist);

// Cleanup
fs.setPermission(
new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"),
new FsPermission((short) 755));
fs.delete(new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"), true);


db = new Database();
db.setName(TEST_DB1_NAME);
dbLocation =
HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/_testDB_file_";
fs = FileSystem.get(new Path(dbLocation).toUri(), hiveConf);
fs.createNewFile(new Path(dbLocation));
fs.deleteOnExit(new Path(dbLocation));
db.setLocationUri(dbLocation);

createFailed = false;
try {
client.createDatabase(db);
} catch (MetaException cantCreateDB) {
System.err.println(cantCreateDB.getMessage());
createFailed = true;
}
assertTrue("Database creation succeeded even location exists and is a file", createFailed);

objectNotExist = false;
try {
client.getDatabase(TEST_DB1_NAME);
} catch (NoSuchObjectException e) {
objectNotExist = true;
}
assertTrue("Database " + TEST_DB1_NAME + " exists when location is specified and is a file",
objectNotExist);

} catch (Throwable e) {
System.err.println(StringUtils.stringifyException(e));
System.err.println("testDatabaseLocation() failed.");
throw e;
}
}


public void testSimpleTypeApi() throws Exception {
try {
client.dropType(Constants.INT_TYPE_NAME);
Expand Down Expand Up @@ -996,6 +1087,60 @@ public void testComplexTable() throws Exception {
}
}

public void testTableDatabase() throws Exception {
String dbName = "testDb";
String tblName_1 = "testTbl_1";
String tblName_2 = "testTbl_2";

try {
silentDropDatabase(dbName);

Database db = new Database();
db.setName(dbName);
String dbLocation =
HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "_testDB_table_create_";
db.setLocationUri(dbLocation);
client.createDatabase(db);
db = client.getDatabase(dbName);

Table tbl = new Table();
tbl.setDbName(dbName);
tbl.setTableName(tblName_1);

ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
cols.add(new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
cols.add(new FieldSchema("income", Constants.INT_TYPE_NAME, ""));

StorageDescriptor sd = new StorageDescriptor();
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(tbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(
org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "9");
sd.getSerdeInfo().setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());

tbl.setSd(sd);
tbl.getSd().setCols(cols);
client.createTable(tbl);
tbl = client.getTable(dbName, tblName_1);

Path path = new Path(tbl.getSd().getLocation());
System.err.println("Table's location " + path + ", Database's location " + db.getLocationUri());
assertEquals("Table location is not a subset of the database location",
path.getParent().toString(), db.getLocationUri());

} catch (Exception e) {
System.err.println(StringUtils.stringifyException(e));
System.err.println("testTableDatabase() failed.");
throw e;
} finally {
silentDropDatabase(dbName);
}
}


public void testGetConfigValue() {

String val = "value";
Expand Down
2 changes: 1 addition & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ private int aggregateStats() {
if (!tableStatsExist && atomic) {
return 0;
}
Path tablePath = wh.getDefaultTablePath(table.getDbName(), table.getTableName());
Path tablePath = wh.getTablePath(db.getDatabase(table.getDbName()), table.getTableName());
fileSys = tablePath.getFileSystem(conf);
fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys);

Expand Down
Loading

0 comments on commit ea2fe2e

Please sign in to comment.