Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move hive.metastore-timeout to ThriftHiveMetastoreConfig #1346

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ public class HiveConfig

private long perTransactionMetastoreCacheMaximumSize = 1000;
private HostAndPort metastoreSocksProxy;
private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS);

private Duration ipcPingInterval = new Duration(10, TimeUnit.SECONDS);
private Duration dfsTimeout = new Duration(60, TimeUnit.SECONDS);
Expand Down Expand Up @@ -407,19 +406,6 @@ public HiveConfig setMetastoreSocksProxy(HostAndPort metastoreSocksProxy)
return this;
}

@NotNull
public Duration getMetastoreTimeout()
{
return metastoreTimeout;
}

@Config("hive.metastore-timeout")
public HiveConfig setMetastoreTimeout(Duration metastoreTimeout)
{
this.metastoreTimeout = metastoreTimeout;
return this;
}

@Min(1)
public int getMinPartitionBatchSize()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,31 @@
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;

import java.util.concurrent.TimeUnit;

public class ThriftHiveMetastoreConfig
{
private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS);
private int maxRetries = RetryDriver.DEFAULT_MAX_ATTEMPTS - 1;
private double backoffScaleFactor = RetryDriver.DEFAULT_SCALE_FACTOR;
private Duration minBackoffDelay = RetryDriver.DEFAULT_SLEEP_TIME;
private Duration maxBackoffDelay = RetryDriver.DEFAULT_SLEEP_TIME;
private Duration maxRetryTime = RetryDriver.DEFAULT_MAX_RETRY_TIME;


@NotNull
public Duration getMetastoreTimeout()
{
return metastoreTimeout;
}

@Config("hive.metastore-timeout")
public ThriftHiveMetastoreConfig setMetastoreTimeout(Duration metastoreTimeout)
{
this.metastoreTimeout = metastoreTimeout;
return this;
}

@Min(0)
public int getMaxRetries()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ public ThriftMetastoreClientFactory(
}

@Inject
public ThriftMetastoreClientFactory(HiveConfig config, HiveMetastoreAuthentication metastoreAuthentication)
public ThriftMetastoreClientFactory(HiveConfig config, ThriftHiveMetastoreConfig thriftConfig, HiveMetastoreAuthentication metastoreAuthentication)
{
this(Optional.empty(), Optional.ofNullable(config.getMetastoreSocksProxy()), config.getMetastoreTimeout(), metastoreAuthentication);
this(Optional.empty(), Optional.ofNullable(config.getMetastoreSocksProxy()), thriftConfig.getMetastoreTimeout(), metastoreAuthentication);
}

public ThriftMetastoreClient create(HostAndPort address)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ public void testDefaults()
.setAllowCorruptWritesForTesting(false)
.setPerTransactionMetastoreCacheMaximumSize(1000)
.setMetastoreSocksProxy(null)
.setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS))
.setMinPartitionBatchSize(10)
.setMaxPartitionBatchSize(100)
.setMaxInitialSplits(200)
Expand Down Expand Up @@ -137,7 +136,6 @@ public void testExplicitPropertyMappings()
.put("hive.allow-corrupt-writes-for-testing", "true")
.put("hive.per-transaction-metastore-cache-maximum-size", "500")
.put("hive.metastore.thrift.client.socks-proxy", "localhost:1080")
.put("hive.metastore-timeout", "20s")
.put("hive.metastore.partition-batch-size.min", "1")
.put("hive.metastore.partition-batch-size.max", "1000")
.put("hive.dfs.ipc-ping-interval", "34s")
Expand Down Expand Up @@ -222,7 +220,6 @@ public void testExplicitPropertyMappings()
.setAllowCorruptWritesForTesting(true)
.setPerTransactionMetastoreCacheMaximumSize(500)
.setMetastoreSocksProxy(HostAndPort.fromParts("localhost", 1080))
.setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS))
.setMinPartitionBatchSize(1)
.setMaxPartitionBatchSize(1000)
.setMaxInitialSplits(10)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.testng.annotations.Test;

import java.util.Map;
import java.util.concurrent.TimeUnit;

import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
Expand All @@ -30,6 +31,7 @@ public class TestThriftHiveMetastoreConfig
public void testDefaults()
{
assertRecordedDefaults(recordDefaults(ThriftHiveMetastoreConfig.class)
.setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS))
.setMaxRetries(9)
.setBackoffScaleFactor(2.0)
.setMinBackoffDelay(new Duration(1, SECONDS))
Expand All @@ -41,6 +43,7 @@ public void testDefaults()
public void testExplicitPropertyMappings()
{
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("hive.metastore-timeout", "20s")
.put("hive.metastore.thrift.client.max-retries", "15")
.put("hive.metastore.thrift.client.backoff-scale-factor", "3.0")
.put("hive.metastore.thrift.client.min-backoff-delay", "2s")
Expand All @@ -49,6 +52,7 @@ public void testExplicitPropertyMappings()
.build();

ThriftHiveMetastoreConfig expected = new ThriftHiveMetastoreConfig()
.setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS))
.setMaxRetries(15)
.setBackoffScaleFactor(3.0)
.setMinBackoffDelay(new Duration(2, SECONDS))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ public TestingMetastoreLocator(HiveConfig config, String host, int port)
public ThriftMetastoreClient createMetastoreClient()
throws TException
{
return new ThriftMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(address);
return new ThriftMetastoreClientFactory(config, new ThriftHiveMetastoreConfig(), new NoHiveMetastoreAuthentication()).create(address);
}
}