From 2e8c960eefc67f7e7c2e9ea8f8aa4631f32f97e3 Mon Sep 17 00:00:00 2001 From: Tom Boam Date: Wed, 8 Nov 2017 10:32:16 +0000 Subject: [PATCH 01/67] Fix SweepBatchConfig values to properly decrease to 1 with each failure and increase with each success (#2630) * Fix SweepBatchConfig values to properly decrease to 1 with each failure and increase with each success * add logging when we stop reducing the batch size multiplier * further improve the tests * Allow sweep to recover faster after backing off. Before we would increase by 1% for each successive success, if we had reduced a value to 1 it would be 70 iterations before we got 2 and 700 iterations before we got back to 1000. Now we always 25 iterations with the lower batch size and then try increasing the rate by doubling each time. This means that when sweep has to back off it should speed up again quickly. * Use an AtomicInteger to handle concurrent updates --- .../AdjustableSweepBatchConfigSource.java | 38 +++- .../AdjustableSweepBatchConfigSourceTest.java | 188 ++++++++++++++++++ docs/source/release_notes/release-notes.rst | 6 + 3 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSourceTest.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSource.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSource.java index 75c94da35a2..0bb5e615413 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSource.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSource.java @@ -16,6 +16,9 @@ package com.palantir.atlasdb.sweep; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +32,7 @@ public final class AdjustableSweepBatchConfigSource { private final Supplier rawSweepBatchConfig; private static volatile double batchSizeMultiplier = 1.0; + private final AtomicInteger successiveIncreases = new AtomicInteger(0); private AdjustableSweepBatchConfigSource(Supplier rawSweepBatchConfig) { this.rawSweepBatchConfig = rawSweepBatchConfig; @@ -67,14 +71,21 @@ private static int adjust(int parameterValue, double multiplier) { } public void increaseMultiplier() { - batchSizeMultiplier = Math.min(1.0, batchSizeMultiplier * 1.01); + if (batchSizeMultiplier == 1.0) { + return; + } + + if (successiveIncreases.incrementAndGet() > 25) { + batchSizeMultiplier = Math.min(1.0, batchSizeMultiplier * 2); + } } public void decreaseMultiplier() { + successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); - // Cut batch size in half, always sweep at least one row (we round down). - batchSizeMultiplier = Math.max(batchSizeMultiplier / 2, 1.5 / lastBatchConfig.candidateBatchSize()); + // Cut batch size in half, always sweep at least one row. + reduceBatchSizeMultiplier(); log.warn("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," @@ -85,4 +96,25 @@ public void decreaseMultiplier() { SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); } + + private void reduceBatchSizeMultiplier() { + SweepBatchConfig config = getRawSweepConfig(); + double smallestSensibleBatchSizeMultiplier = + 1.0 / NumberUtils.max( + config.maxCellTsPairsToExamine(), config.candidateBatchSize(), config.deleteBatchSize()); + + if (batchSizeMultiplier == smallestSensibleBatchSizeMultiplier) { + return; + } + + double newBatchSizeMultiplier = batchSizeMultiplier / 2; + if (newBatchSizeMultiplier < smallestSensibleBatchSizeMultiplier) { + log.info("batchSizeMultiplier reached the smallest sensible value for the current sweep config ({}), " + + "will not reduce further.", + SafeArg.of("batchSizeMultiplier", smallestSensibleBatchSizeMultiplier)); + batchSizeMultiplier = smallestSensibleBatchSizeMultiplier; + } else { + batchSizeMultiplier = newBatchSizeMultiplier; + } + } } diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSourceTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSourceTest.java new file mode 100644 index 00000000000..671a469669a --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/AdjustableSweepBatchConfigSourceTest.java @@ -0,0 +1,188 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.sweep; + +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.Assert.assertThat; + +import java.util.function.Function; + +import org.junit.Test; + +/** + * Created by tboam on 03/11/2017. + */ +public class AdjustableSweepBatchConfigSourceTest { + + private AdjustableSweepBatchConfigSource adjustableConfig; + private SweepBatchConfig previousConfig; + private double previousMultiplier; + + @Test + public void batchSizeMultiplierDecreasesOnFailure() { + // Given + configWithValues(1000, 1000, 1000); + + // When + adjustableConfig.decreaseMultiplier(); + + // Then + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThan(previousMultiplier))); + } + + @Test + public void canDecreaseAndIncreaseConfigWithAllSmallValues() { + //Given + configWithValues(1, 1, 1); + + whenDecreasingTheMultiplier_thenAdjustedConfigValuesDecrease(); + + whenIncreasingTheMultiplier_thenAdjustedConfigValuesIncrease(); + + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(1.0)); + } + + @Test + public void canDecreaseAndIncreaseConfigWithAllLargeValues() { + //Given + configWithValues(1000, 1000, 1000); + + whenDecreasingTheMultiplier_thenAdjustedConfigValuesDecrease(); + + whenIncreasingTheMultiplier_thenAdjustedConfigValuesIncrease(); + + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(1.0)); + } + + @Test + public void canDecreaseAndIncreaseConfigWithMixOfValues() { + //Given + configWithValues(1000, 1, 100); + + whenDecreasingTheMultiplier_thenAdjustedConfigValuesDecrease(); + + whenIncreasingTheMultiplier_thenAdjustedConfigValuesIncrease(); + + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(1.0)); + } + + private void whenDecreasingTheMultiplier_thenAdjustedConfigValuesDecrease() { + for (int i = 0; i < 10_000; i++) { + // When + adjustableConfig.decreaseMultiplier(); + + // Then + batchSizeMultiplierDecreases(); + maxCellTsPairsToExamineDecreasesToAMinimumOfOne(); + candidateBatchSizeDecreasesToAMinimumOfOne(); + deleteBatchSizeDecreasesToAMinimumOfOne(); + + updatePreviousValues(); + } + } + + private void whenIncreasingTheMultiplier_thenAdjustedConfigValuesIncrease() { + for (int i = 0; i < 1_000; i++) { + // When + adjustableConfig.increaseMultiplier(); + + // Then + batchSizeMultiplierIncreases(); + batchSizeMultiplierDoesNotExceedOne(); + + maxCellTsPairsToExamineIncreasesBackUpToBaseConfig(); + candidateBatchSizeIncreasesBackUpToBaseConfig(); + deleteBatchSizeIncreasesBackUpToBaseConfig(); + + updatePreviousValues(); + } + } + + private void configWithValues(int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { + adjustableConfig = AdjustableSweepBatchConfigSource.create(() -> + ImmutableSweepBatchConfig.builder() + .maxCellTsPairsToExamine(maxCellTsPairsToExamine) + .candidateBatchSize(candidateBatchSize) + .deleteBatchSize(deleteBatchSize) + .build() + ); + + updatePreviousValues(); + } + + private void batchSizeMultiplierDecreases() { + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThanOrEqualTo(previousMultiplier))); + } + + private void decreasesToOne(Function getValue) { + int newValue = getValue.apply(adjustableConfig.getAdjustedSweepConfig()); + int previousValue = getValue.apply(previousConfig); + + assertThat(newValue, is(anyOf(equalTo(1), lessThan(previousValue)))); + } + + private void maxCellTsPairsToExamineDecreasesToAMinimumOfOne() { + decreasesToOne(SweepBatchConfig::maxCellTsPairsToExamine); + } + + private void candidateBatchSizeDecreasesToAMinimumOfOne() { + decreasesToOne(SweepBatchConfig::candidateBatchSize); + } + + private void deleteBatchSizeDecreasesToAMinimumOfOne() { + decreasesToOne(SweepBatchConfig::deleteBatchSize); + } + + private void batchSizeMultiplierIncreases() { + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(greaterThanOrEqualTo(previousMultiplier))); + } + + private void batchSizeMultiplierDoesNotExceedOne() { + assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThanOrEqualTo(1.0))); + } + + private void maxCellTsPairsToExamineIncreasesBackUpToBaseConfig() { + increasesBackUpToBaseConfig(SweepBatchConfig::maxCellTsPairsToExamine); + } + + private void candidateBatchSizeIncreasesBackUpToBaseConfig() { + increasesBackUpToBaseConfig(SweepBatchConfig::candidateBatchSize); + } + + private void deleteBatchSizeIncreasesBackUpToBaseConfig() { + increasesBackUpToBaseConfig(SweepBatchConfig::deleteBatchSize); + } + + private void increasesBackUpToBaseConfig(Function getValue) { + assertThat(getValue.apply(adjustableConfig.getAdjustedSweepConfig()), + is(anyOf( + greaterThan(getValue.apply(previousConfig)), + lessThanOrEqualTo(getValue.apply(adjustableConfig.getRawSweepConfig())) + ))); + } + + private void updatePreviousValues() { + previousMultiplier = adjustableConfig.getBatchSizeMultiplier(); + previousConfig = adjustableConfig.getAdjustedSweepConfig(); + } +} diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index a0c48fe9dac..1094fd3e27b 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -53,6 +53,12 @@ develop Also, all Cassandra KVS exceptions like ``KeyAlreadyExists`` or ``TTransportException`` as well as ``NotInitializedException`` will get wrapped into ``AtlasDbDependencyException`` in the interest of consistent exceptions. (`Pull Request `__) + * - |fixed| + - ``SweepBatchConfig`` values are now decayed correctly when there's an error. + ``SweepBatchConfig`` should be decreased until sweep succeeds, however the config actually oscillated between values, these were normally small but could be larger than the original config. This was caused by us fixing one of the values at 1. + ``SweepBatchConfig`` values will now be halved with each failure until they reach 1 (previously they only went to about 30% due to another bug). This ensures we fully backoff and gives us the best possible chance of success. Values will slowly increase with each successful run until they are back to their default level. + (`Pull Request `__) + * - |fixed| - Reverted the Cassandra KVS executor PR (`Pull Request `__) that caused a performance regression. (`Pull Request `__) From f116645cca5dd4026c86570c93323581e780a842 Mon Sep 17 00:00:00 2001 From: Tom Boam Date: Wed, 8 Nov 2017 10:33:23 +0000 Subject: [PATCH 02/67] SweeperService logging improvements (#2618) * SweeperServiceImpl now logs when it starts sweeping make it clear if it is running full sweep or not * Added sweep parameters to the log lines * no longer default the service parameter in the interface, this way we can see when the parameter isn't provided and we are defaulting to true. Behaviour is unchanged but we can log a message when defaulting. --- .../atlasdb/sweep/SweeperService.java | 3 +- .../atlasdb/sweep/SweeperServiceImpl.java | 49 ++++++++++++++----- docs/source/release_notes/release-notes.rst | 6 +++ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperService.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperService.java index b9c9e69b27f..1a68449d9c4 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperService.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperService.java @@ -18,7 +18,6 @@ import java.util.Optional; import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -60,7 +59,7 @@ default SweepTableResponse sweepTableFrom(String tableName, String startRow) { SweepTableResponse sweepTable( @QueryParam("tablename") String tableName, @QueryParam("startRow") Optional startRow, - @Safe @QueryParam("fullSweep") @DefaultValue("true") Optional fullSweep, + @Safe @QueryParam("fullSweep") Optional fullSweep, @Safe @QueryParam("maxCellTsPairsToExamine") Optional maxCellTsPairsToExamine, @Safe @QueryParam("candidateBatchSize") Optional candidateBatchSize, @Safe @QueryParam("deleteBatchSize") Optional deleteBatchSize); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperServiceImpl.java index dd65bcf05c6..99db582e62f 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperServiceImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SweeperServiceImpl.java @@ -17,13 +17,21 @@ import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.base.Preconditions; import com.palantir.atlasdb.encoding.PtBytes; import com.palantir.atlasdb.keyvalue.api.SweepResults; import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.UnsafeArg; import com.palantir.remoting3.servers.jersey.WebPreconditions; public final class SweeperServiceImpl implements SweeperService { + private static final Logger log = LoggerFactory.getLogger(SweeperServiceImpl.class); + private final SpecificTableSweeper specificTableSweeper; private final AdjustableSweepBatchConfigSource sweepBatchConfigSource; @@ -48,17 +56,36 @@ public SweepTableResponse sweepTable( SweepBatchConfig config = buildConfigWithOverrides(maxCellTsPairsToExamine, candidateBatchSize, deleteBatchSize); - SweepResults sweepResults = fullSweep.orElse(true) - ? runFullSweepWithoutSavingResults( - tableRef, - decodedStartRow, - config) - : runOneBatchWithoutSavingResults( - tableRef, - decodedStartRow, - config); - - return SweepTableResponse.from(sweepResults); + return SweepTableResponse.from(runSweep(fullSweep, tableRef, decodedStartRow, config)); + } + + private SweepResults runSweep(Optional fullSweep, TableReference tableRef, byte[] decodedStartRow, + SweepBatchConfig config) { + if (!fullSweep.isPresent()) { + log.warn("fullSweep parameter was not specified, defaulting to true"); + } + + if (fullSweep.orElse(true)) { + log.info("Running sweep of full table {}, " + + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + + "starting from row {}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), + SafeArg.of("candidateBatchSize", config.candidateBatchSize()), + SafeArg.of("deleteBatchSize", config.deleteBatchSize()), + UnsafeArg.of("decodedStartRow", decodedStartRow)); + return runFullSweepWithoutSavingResults(tableRef, decodedStartRow, config); + } else { + log.info("Running sweep of a single batch on table {}, " + + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + + "starting from row {}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), + SafeArg.of("candidateBatchSize", config.candidateBatchSize()), + SafeArg.of("deleteBatchSize", config.deleteBatchSize()), + UnsafeArg.of("decodedStartRow", decodedStartRow)); + return runOneBatchWithoutSavingResults(tableRef, decodedStartRow, config); + } } private SweepBatchConfig buildConfigWithOverrides( diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index 1094fd3e27b..58cd8fd2b0d 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -77,6 +77,12 @@ develop - AtlasDB can now tag RC releases. (`Pull Request `__) + * - |improved| |logs| + - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not + + * - + - + .. <<<<------------------------------------------------------------------------------------------------------------->>>> ======= From c1e21ee2b44499877eb5c2bcdeee8f81834b1fef Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Wed, 8 Nov 2017 11:07:24 +0000 Subject: [PATCH 03/67] Refactor TracingKVS (#2643) * Wrap next() and hasNext() in traces * Use span names as safe * Remove iterator wrappings * checkstyle * refactor methods and remove misleading traces * Fix unit tests * release notes * Final nits * fix java arrays usage --- .../keyvalue/impl/TracingKeyValueService.java | 92 +++++++++---------- .../palantir/atlasdb/logging/LoggingArgs.java | 21 +++++ .../impl/TracingKeyValueServiceTest.java | 91 +++++------------- .../atlasdb/logging/LoggingArgsTest.java | 21 +++++ docs/source/release_notes/release-notes.rst | 4 + 5 files changed, 113 insertions(+), 116 deletions(-) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueService.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueService.java index c18f01a612b..785df79280b 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueService.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueService.java @@ -41,12 +41,13 @@ import com.palantir.atlasdb.keyvalue.api.RowResult; import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.api.Value; +import com.palantir.atlasdb.logging.LoggingArgs; import com.palantir.atlasdb.tracing.CloseableTrace; import com.palantir.common.base.ClosableIterator; import com.palantir.util.paging.TokenBackedBasicResultsPage; /** - * Wraps a {@link KeyValueService}'s methods with {@link com.palantir.remoting2.tracing.Tracer} + * Wraps a {@link KeyValueService}'s methods with {@link com.palantir.remoting3.tracing.Tracer} * instrumentation. */ public final class TracingKeyValueService extends ForwardingObject implements KeyValueService { @@ -76,7 +77,7 @@ protected KeyValueService delegate() { public void addGarbageCollectionSentinelValues(TableReference tableRef, Iterable cells) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("addGarbageCollectionSentinelValues({}, {} cells)", - tableRef, Iterables.size(cells))) { + LoggingArgs.safeTableOrPlaceholder(tableRef), Iterables.size(cells))) { delegate().addGarbageCollectionSentinelValues(tableRef, cells); } } @@ -84,8 +85,8 @@ public void addGarbageCollectionSentinelValues(TableReference tableRef, Iterable @Override public void checkAndSet(CheckAndSetRequest checkAndSetRequest) throws CheckAndSetException { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("checkAndSet({}, {})", - checkAndSetRequest.table(), checkAndSetRequest.cell())) { + try (CloseableTrace trace = startLocalTrace("checkAndSet({})", + LoggingArgs.safeTableOrPlaceholder(checkAndSetRequest.table()))) { delegate().checkAndSet(checkAndSetRequest); } } @@ -101,7 +102,8 @@ public void close() { @Override public void compactInternally(TableReference tableRef) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("compactInternally({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("compactInternally({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { delegate().compactInternally(tableRef); } } @@ -122,7 +124,8 @@ public boolean isInitialized() { @Override public void createTable(TableReference tableRef, byte[] tableMetadata) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("createTable({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("createTable({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { delegate().createTable(tableRef, tableMetadata); } } @@ -131,7 +134,7 @@ public void createTable(TableReference tableRef, byte[] tableMetadata) { public void createTables(Map tableNamesToTableMetadata) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("createTables({})", - tableNamesToTableMetadata.keySet())) { + LoggingArgs.safeTablesOrPlaceholder(tableNamesToTableMetadata.keySet()))) { delegate().createTables(tableNamesToTableMetadata); } } @@ -139,7 +142,8 @@ public void createTables(Map tableNamesToTableMetadata) @Override public void delete(TableReference tableRef, Multimap keys) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("delete({}, {} keys)", tableRef, keys.size())) { + try (CloseableTrace trace = startLocalTrace("delete({}, {} keys)", + LoggingArgs.safeTableOrPlaceholder(tableRef), keys.size())) { delegate().delete(tableRef, keys); } } @@ -147,7 +151,8 @@ public void delete(TableReference tableRef, Multimap keys) { @Override public void deleteRange(TableReference tableRef, RangeRequest range) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("deleteRange({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("deleteRange({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { delegate().deleteRange(tableRef, range); } } @@ -155,7 +160,8 @@ public void deleteRange(TableReference tableRef, RangeRequest range) { @Override public void dropTable(TableReference tableRef) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("dropTable({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("dropTable({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { delegate().dropTable(tableRef); } } @@ -163,7 +169,8 @@ public void dropTable(TableReference tableRef) { @Override public void dropTables(Set tableRefs) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("dropTables({})", tableRefs)) { + try (CloseableTrace trace = startLocalTrace("dropTables({})", + LoggingArgs.safeTablesOrPlaceholder(tableRefs))) { delegate().dropTables(tableRefs); } } @@ -172,7 +179,7 @@ public void dropTables(Set tableRefs) { public Map get(TableReference tableRef, Map timestampByCell) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("get({}, {} cells)", - tableRef, timestampByCell.size())) { + LoggingArgs.safeTableOrPlaceholder(tableRef), timestampByCell.size())) { return delegate().get(tableRef, timestampByCell); } } @@ -189,7 +196,7 @@ public Set getAllTableNames() { public Multimap getAllTimestamps(TableReference tableRef, Set keys, long timestamp) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("getAllTimestamps({}, {} keys, ts {})", - tableRef, keys.size(), timestamp)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), keys.size(), timestamp)) { return delegate().getAllTimestamps(tableRef, keys, timestamp); } } @@ -209,7 +216,7 @@ public Map, byte[]>> long timestamp) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("getFirstBatchForRanges({}, {} ranges, ts {})", - tableRef, Iterables.size(rangeRequests), timestamp)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), Iterables.size(rangeRequests), timestamp)) { return delegate().getFirstBatchForRanges(tableRef, rangeRequests, timestamp); } } @@ -219,7 +226,7 @@ public Map getLatestTimestamps(TableReference tableRef, Map timestampByCell) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("getLatestTimestamps({}, {} cells)", - tableRef, timestampByCell.size())) { + LoggingArgs.safeTableOrPlaceholder(tableRef), timestampByCell.size())) { return delegate().getLatestTimestamps(tableRef, timestampByCell); } } @@ -227,7 +234,8 @@ public Map getLatestTimestamps(TableReference tableRef, @Override public byte[] getMetadataForTable(TableReference tableRef) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("getMetadataForTable({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("getMetadataForTable({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { return delegate().getMetadataForTable(tableRef); } } @@ -244,32 +252,23 @@ public Map getMetadataForTables() { public ClosableIterator> getRange(TableReference tableRef, RangeRequest rangeRequest, long timestamp) { - //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("getRange({}, ts {})", - tableRef, timestamp)) { - return delegate().getRange(tableRef, rangeRequest, timestamp); - } + // No tracing, as we just return a lazy iterator and don't perform any calls to the backing KVS. + return delegate().getRange(tableRef, rangeRequest, timestamp); } @Override public ClosableIterator>> getRangeOfTimestamps(TableReference tableRef, RangeRequest rangeRequest, long timestamp) { - //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("getRangeOfTimestamps({}, ts {})", - tableRef, timestamp)) { - return delegate().getRangeOfTimestamps(tableRef, rangeRequest, timestamp); - } + // No tracing, as we just return a lazy iterator and don't perform any calls to the backing KVS. + return delegate().getRangeOfTimestamps(tableRef, rangeRequest, timestamp); } @Override public ClosableIterator> getCandidateCellsForSweeping(TableReference tableRef, CandidateCellForSweepingRequest request) { - //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("getCandidateCellsForSweeping({}, ts {})", - tableRef, request.maxTimestampExclusive())) { - return delegate().getCandidateCellsForSweeping(tableRef, request); - } + // No tracing, as we just return a lazy iterator and don't perform any calls to the backing KVS. + return delegate().getCandidateCellsForSweeping(tableRef, request); } @Override @@ -279,7 +278,7 @@ public Map getRows(TableReference tableRef, long timestamp) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("getRows({}, {} rows, ts {})", - tableRef, Iterables.size(rows), timestamp)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), Iterables.size(rows), timestamp)) { return delegate().getRows(tableRef, rows, columnSelection, timestamp); } } @@ -291,7 +290,7 @@ public Map getRowsColumnRange(TableReference tab long timestamp) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("getRowsColumnRange({}, {} rows, ts {})", - tableRef, Iterables.size(rows), timestamp)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), Iterables.size(rows), timestamp)) { return delegate().getRowsColumnRange(tableRef, rows, columnRangeSelection, timestamp); } } @@ -302,11 +301,8 @@ public RowColumnRangeIterator getRowsColumnRange(TableReference tableRef, ColumnRangeSelection columnRangeSelection, int cellBatchHint, long timestamp) { - //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("getRowsColumnRange({}, {} rows, {} hint, ts {})", - tableRef, Iterables.size(rows), cellBatchHint, timestamp)) { - return delegate().getRowsColumnRange(tableRef, rows, columnRangeSelection, cellBatchHint, timestamp); - } + // No tracing, as we just return a lazy iterator and don't perform any calls to the backing KVS. + return delegate().getRowsColumnRange(tableRef, rows, columnRangeSelection, cellBatchHint, timestamp); } @Override @@ -323,7 +319,7 @@ public void multiPut(Map> valuesByTa public void put(TableReference tableRef, Map values, long timestamp) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("put({}, {} values, ts {})", - tableRef, values.size(), timestamp)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), values.size(), timestamp)) { delegate().put(tableRef, values, timestamp); } } @@ -332,7 +328,7 @@ public void put(TableReference tableRef, Map values, long timestam public void putMetadataForTable(TableReference tableRef, byte[] metadata) { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("putMetadataForTable({}, {} bytes)", - tableRef, (metadata == null) ? 0 : metadata.length)) { + LoggingArgs.safeTableOrPlaceholder(tableRef), (metadata == null) ? 0 : metadata.length)) { delegate().putMetadataForTable(tableRef, metadata); } } @@ -340,7 +336,8 @@ public void putMetadataForTable(TableReference tableRef, byte[] metadata) { @Override public void putMetadataForTables(Map tableRefToMetadata) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("putMetadataForTables({})", tableRefToMetadata.keySet())) { + try (CloseableTrace trace = startLocalTrace("putMetadataForTables({})", + LoggingArgs.safeTablesOrPlaceholder(tableRefToMetadata.keySet()))) { delegate().putMetadataForTables(tableRefToMetadata); } } @@ -350,7 +347,7 @@ public void putUnlessExists(TableReference tableRef, Map values) throws KeyAlreadyExistsException { //noinspection unused - try-with-resources closes trace try (CloseableTrace trace = startLocalTrace("putUnlessExists({}, {} values)", - tableRef, values.size())) { + LoggingArgs.safeTableOrPlaceholder(tableRef), values.size())) { delegate().putUnlessExists(tableRef, values); } } @@ -363,8 +360,8 @@ public boolean supportsCheckAndSet() { @Override public void putWithTimestamps(TableReference tableRef, Multimap values) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace( - "putWithTimestamps({}, {} values)", tableRef, values.size())) { + try (CloseableTrace trace = startLocalTrace("putWithTimestamps({}, {} values)", + LoggingArgs.safeTableOrPlaceholder(tableRef), values.size())) { delegate().putWithTimestamps(tableRef, values); } } @@ -372,7 +369,8 @@ public void putWithTimestamps(TableReference tableRef, Multimap val @Override public void truncateTable(TableReference tableRef) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("truncateTable({})", tableRef)) { + try (CloseableTrace trace = startLocalTrace("truncateTable({})", + LoggingArgs.safeTableOrPlaceholder(tableRef))) { delegate().truncateTable(tableRef); } } @@ -380,10 +378,10 @@ public void truncateTable(TableReference tableRef) { @Override public void truncateTables(Set tableRefs) { //noinspection unused - try-with-resources closes trace - try (CloseableTrace trace = startLocalTrace("truncateTables({})", tableRefs)) { + try (CloseableTrace trace = startLocalTrace("truncateTables({})", + LoggingArgs.safeTablesOrPlaceholder(tableRefs))) { delegate().truncateTables(tableRefs); } } - } diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java index 3a593e54b3a..bd8ed1d9dcc 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java @@ -43,6 +43,11 @@ * Always returns unsafe, until hydrated. */ public final class LoggingArgs { + + @VisibleForTesting + static final TableReference PLACEHOLDER_TABLE_REFERENCE = + TableReference.createWithEmptyNamespace("{table}"); + @Value.Immutable public interface SafeAndUnsafeTableReferences { SafeArg> safeTableRefs(); @@ -82,6 +87,11 @@ public static SafeAndUnsafeTableReferences tableRefs(Collection .build(); } + public static Iterable safeTablesOrPlaceholder(Collection tables) { + //noinspection StaticPseudoFunctionalStyleMethod - Use lazy iterator. + return Iterables.transform(tables, LoggingArgs::safeTableOrPlaceholder); + } + /** * Returns a safe or unsafe arg corresponding to the supplied table reference, with name "tableRef". */ @@ -89,6 +99,17 @@ public static Arg tableRef(TableReference tableReference) { return tableRef("tableRef", tableReference); } + /** + * If table is safe, returns the table. If unsafe, returns a placeholder. + */ + public static TableReference safeTableOrPlaceholder(TableReference tableReference) { + if (logArbitrator.isTableReferenceSafe(tableReference)) { + return tableReference; + } else { + return PLACEHOLDER_TABLE_REFERENCE; + } + } + public static Arg tableRef(String argName, TableReference tableReference) { return getArg(argName, tableReference.toString(), logArbitrator.isTableReferenceSafe(tableReference)); } diff --git a/atlasdb-client/src/test/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueServiceTest.java b/atlasdb-client/src/test/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueServiceTest.java index aba8879d187..aeb1b3fa5c3 100644 --- a/atlasdb-client/src/test/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueServiceTest.java +++ b/atlasdb-client/src/test/java/com/palantir/atlasdb/keyvalue/impl/TracingKeyValueServiceTest.java @@ -48,7 +48,6 @@ import com.palantir.atlasdb.keyvalue.api.BatchColumnRangeSelection; import com.palantir.atlasdb.keyvalue.api.Cell; import com.palantir.atlasdb.keyvalue.api.CheckAndSetRequest; -import com.palantir.atlasdb.keyvalue.api.ColumnRangeSelection; import com.palantir.atlasdb.keyvalue.api.ColumnSelection; import com.palantir.atlasdb.keyvalue.api.KeyValueService; import com.palantir.atlasdb.keyvalue.api.Namespace; @@ -58,8 +57,6 @@ import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.api.Value; import com.palantir.atlasdb.tracing.TestSpanObserver; -import com.palantir.common.base.ClosableIterator; -import com.palantir.common.base.ClosableIterators; import com.palantir.remoting.api.tracing.SpanType; import com.palantir.remoting3.tracing.Tracer; import com.palantir.util.paging.TokenBackedBasicResultsPage; @@ -120,7 +117,7 @@ public void addGarbageCollectionSentinelValues() throws Exception { ImmutableSet cells = ImmutableSet.of(CELL); kvs.addGarbageCollectionSentinelValues(TABLE_REF, cells); - checkSpan("atlasdb-kvs.addGarbageCollectionSentinelValues(test.testTable, 1 cells)"); + checkSpan("atlasdb-kvs.addGarbageCollectionSentinelValues({table}, 1 cells)"); verify(delegate).addGarbageCollectionSentinelValues(TABLE_REF, cells); verifyNoMoreInteractions(delegate); } @@ -130,7 +127,7 @@ public void checkAndSet() throws Exception { CheckAndSetRequest request = CheckAndSetRequest.singleCell(TABLE_REF, CELL, ROW_NAME, ROW_NAME); kvs.checkAndSet(request); - checkSpan("atlasdb-kvs.checkAndSet(test.testTable, Cell{rowName=726f77, columnName=636f6c, no TTL})"); + checkSpan("atlasdb-kvs.checkAndSet({table})"); verify(delegate).checkAndSet(request); verifyNoMoreInteractions(delegate); } @@ -148,7 +145,7 @@ public void close() throws Exception { public void compactInternally() throws Exception { kvs.compactInternally(TABLE_REF); - checkSpan("atlasdb-kvs.compactInternally(test.testTable)"); + checkSpan("atlasdb-kvs.compactInternally({table})"); verify(delegate).compactInternally(TABLE_REF); verifyNoMoreInteractions(delegate); } @@ -158,7 +155,7 @@ public void createTable() throws Exception { byte[] metadata = new byte[0]; kvs.createTable(TABLE_REF, metadata); - checkSpan("atlasdb-kvs.createTable(test.testTable)"); + checkSpan("atlasdb-kvs.createTable({table})"); verify(delegate).createTable(TABLE_REF, metadata); verifyNoMoreInteractions(delegate); } @@ -168,7 +165,7 @@ public void createTables() throws Exception { byte[] metadata = new byte[0]; kvs.createTables(ImmutableMap.of(TABLE_REF, metadata)); - checkSpan("atlasdb-kvs.createTables([test.testTable])"); + checkSpan("atlasdb-kvs.createTables([{table}])"); verify(delegate).createTables(ImmutableMap.of(TABLE_REF, metadata)); verifyNoMoreInteractions(delegate); } @@ -178,7 +175,7 @@ public void delete() throws Exception { Multimap cells = ImmutableMultimap.of(CELL, 1L); kvs.delete(TABLE_REF, cells); - checkSpan("atlasdb-kvs.delete(test.testTable, 1 keys)"); + checkSpan("atlasdb-kvs.delete({table}, 1 keys)"); verify(delegate).delete(TABLE_REF, cells); verifyNoMoreInteractions(delegate); } @@ -187,7 +184,7 @@ public void delete() throws Exception { public void dropTable() throws Exception { kvs.dropTable(TABLE_REF); - checkSpan("atlasdb-kvs.dropTable(test.testTable)"); + checkSpan("atlasdb-kvs.dropTable({table})"); verify(delegate).dropTable(TABLE_REF); verifyNoMoreInteractions(delegate); } @@ -196,7 +193,7 @@ public void dropTable() throws Exception { public void dropTables() throws Exception { kvs.dropTables(ImmutableSet.of(TABLE_REF)); - checkSpan("atlasdb-kvs.dropTables([test.testTable])"); + checkSpan("atlasdb-kvs.dropTables([{table}])"); verify(delegate).dropTables(ImmutableSet.of(TABLE_REF)); verifyNoMoreInteractions(delegate); } @@ -210,7 +207,7 @@ public void get() throws Exception { Map result = kvs.get(TABLE_REF, cells); assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.get(test.testTable, 1 cells)"); + checkSpan("atlasdb-kvs.get({table}, 1 cells)"); verify(delegate).get(TABLE_REF, cells); verifyNoMoreInteractions(delegate); } @@ -229,7 +226,7 @@ public void getAllTimestamps() throws Exception { Set cells = ImmutableSet.of(CELL); kvs.getAllTimestamps(TABLE_REF, cells, 1L); - checkSpan("atlasdb-kvs.getAllTimestamps(test.testTable, 1 keys, ts 1)"); + checkSpan("atlasdb-kvs.getAllTimestamps({table}, 1 keys, ts 1)"); verify(delegate).getAllTimestamps(TABLE_REF, cells, 1L); verifyNoMoreInteractions(delegate); } @@ -252,7 +249,7 @@ public void getFirstBatchForRanges() throws Exception { TABLE_REF, RANGE_REQUESTS, TIMESTAMP); assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getFirstBatchForRanges(test.testTable, 1 ranges, ts 1)"); + checkSpan("atlasdb-kvs.getFirstBatchForRanges({table}, 1 ranges, ts 1)"); verify(delegate).getFirstBatchForRanges(TABLE_REF, RANGE_REQUESTS, TIMESTAMP); verifyNoMoreInteractions(delegate); } @@ -265,7 +262,7 @@ public void getLatestTimestamps() throws Exception { Map result = kvs.getLatestTimestamps(TABLE_REF, cells); assertThat(result.entrySet(), hasSize(1)); - checkSpan("atlasdb-kvs.getLatestTimestamps(test.testTable, 1 cells)"); + checkSpan("atlasdb-kvs.getLatestTimestamps({table}, 1 cells)"); verify(delegate).getLatestTimestamps(TABLE_REF, cells); verifyNoMoreInteractions(delegate); } @@ -277,7 +274,7 @@ public void getMetadataForTable() throws Exception { byte[] result = kvs.getMetadataForTable(TABLE_REF); assertThat(result, equalTo(METADATA_BYTES)); - checkSpan("atlasdb-kvs.getMetadataForTable(test.testTable)"); + checkSpan("atlasdb-kvs.getMetadataForTable({table})"); verify(delegate).getMetadataForTable(TABLE_REF); verifyNoMoreInteractions(delegate); } @@ -295,34 +292,6 @@ public void getMetadataForTables() throws Exception { verifyNoMoreInteractions(delegate); } - @Test - public void getRange() throws Exception { - ClosableIterator> expectedResult = ClosableIterators.wrap(ImmutableList.of( - RowResult.of(CELL, VALUE)).iterator()); - when(delegate.getRange(TABLE_REF, RANGE_REQUEST, TIMESTAMP)).thenReturn(expectedResult); - try (ClosableIterator> result = kvs.getRange(TABLE_REF, RANGE_REQUEST, TIMESTAMP)) { - assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getRange(test.testTable, ts 1)"); - verify(delegate).getRange(TABLE_REF, RANGE_REQUEST, TIMESTAMP); - verifyNoMoreInteractions(delegate); - } - } - - @Test - public void getRangeOfTimestamps() throws Exception { - Set longs = ImmutableSet.of(TIMESTAMP); - ClosableIterator>> expectedResult = ClosableIterators.wrap(ImmutableList.of( - RowResult.of(CELL, longs)).iterator()); - when(delegate.getRangeOfTimestamps(TABLE_REF, RANGE_REQUEST, TIMESTAMP)).thenReturn(expectedResult); - try (ClosableIterator>> result = kvs.getRangeOfTimestamps( - TABLE_REF, RANGE_REQUEST, TIMESTAMP)) { - assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getRangeOfTimestamps(test.testTable, ts 1)"); - verify(delegate).getRangeOfTimestamps(TABLE_REF, RANGE_REQUEST, TIMESTAMP); - verifyNoMoreInteractions(delegate); - } - } - @Test public void getRows() throws Exception { ImmutableList rows = ImmutableList.of(ROW_NAME); @@ -332,7 +301,7 @@ public void getRows() throws Exception { Map result = kvs.getRows(TABLE_REF, rows, ColumnSelection.all(), TIMESTAMP); assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getRows(test.testTable, 1 rows, ts 1)"); + checkSpan("atlasdb-kvs.getRows({table}, 1 rows, ts 1)"); verify(delegate).getRows(TABLE_REF, rows, ColumnSelection.all(), TIMESTAMP); verifyNoMoreInteractions(delegate); } @@ -348,27 +317,11 @@ public void getRowsColumnRangeBatch() throws Exception { Map result = kvs.getRowsColumnRange(TABLE_REF, rows, range, TIMESTAMP); assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getRowsColumnRange(test.testTable, 1 rows, ts 1)"); + checkSpan("atlasdb-kvs.getRowsColumnRange({table}, 1 rows, ts 1)"); verify(delegate).getRowsColumnRange(TABLE_REF, rows, range, TIMESTAMP); verifyNoMoreInteractions(delegate); } - @Test - public void getRowsColumnRange() throws Exception { - RowColumnRangeIterator expectedResult = mock(RowColumnRangeIterator.class); - List rows = ImmutableList.of(ROW_NAME); - ColumnRangeSelection range = new ColumnRangeSelection(COL_NAME, COL_NAME); - int cellBatchHint = 2; - when(delegate.getRowsColumnRange(TABLE_REF, rows, range, cellBatchHint, TIMESTAMP)).thenReturn(expectedResult); - - RowColumnRangeIterator result = kvs.getRowsColumnRange(TABLE_REF, rows, range, cellBatchHint, TIMESTAMP); - - assertThat(result, equalTo(expectedResult)); - checkSpan("atlasdb-kvs.getRowsColumnRange(test.testTable, 1 rows, 2 hint, ts 1)"); - verify(delegate).getRowsColumnRange(TABLE_REF, rows, range, cellBatchHint, TIMESTAMP); - verifyNoMoreInteractions(delegate); - } - @Test public void multiPut() throws Exception { Map> values = ImmutableMap.of(TABLE_REF, ImmutableMap.of(CELL, VALUE_BYTES)); @@ -384,7 +337,7 @@ public void put() throws Exception { Map values = ImmutableMap.of(CELL, VALUE_BYTES); kvs.put(TABLE_REF, values, TIMESTAMP); - checkSpan("atlasdb-kvs.put(test.testTable, 1 values, ts 1)"); + checkSpan("atlasdb-kvs.put({table}, 1 values, ts 1)"); verify(delegate).put(TABLE_REF, values, TIMESTAMP); verifyNoMoreInteractions(delegate); } @@ -393,7 +346,7 @@ public void put() throws Exception { public void putMetadataForTable() throws Exception { kvs.putMetadataForTable(TABLE_REF, METADATA_BYTES); - checkSpan("atlasdb-kvs.putMetadataForTable(test.testTable, 8 bytes)"); + checkSpan("atlasdb-kvs.putMetadataForTable({table}, 8 bytes)"); verify(delegate).putMetadataForTable(TABLE_REF, METADATA_BYTES); verifyNoMoreInteractions(delegate); } @@ -402,7 +355,7 @@ public void putMetadataForTable() throws Exception { public void putMetadataForTables() throws Exception { kvs.putMetadataForTables(ImmutableMap.of(TABLE_REF, METADATA_BYTES)); - checkSpan("atlasdb-kvs.putMetadataForTables([test.testTable])"); + checkSpan("atlasdb-kvs.putMetadataForTables([{table}])"); verify(delegate).putMetadataForTables(ImmutableMap.of(TABLE_REF, METADATA_BYTES)); verifyNoMoreInteractions(delegate); } @@ -412,7 +365,7 @@ public void putUnlessExists() throws Exception { Map values = ImmutableMap.of(CELL, VALUE_BYTES); kvs.putUnlessExists(TABLE_REF, values); - checkSpan("atlasdb-kvs.putUnlessExists(test.testTable, 1 values)"); + checkSpan("atlasdb-kvs.putUnlessExists({table}, 1 values)"); verify(delegate).putUnlessExists(TABLE_REF, values); verifyNoMoreInteractions(delegate); } @@ -422,7 +375,7 @@ public void putWithTimestamps() throws Exception { Multimap values = ImmutableMultimap.of(CELL, VALUE); kvs.putWithTimestamps(TABLE_REF, values); - checkSpan("atlasdb-kvs.putWithTimestamps(test.testTable, 1 values)"); + checkSpan("atlasdb-kvs.putWithTimestamps({table}, 1 values)"); verify(delegate).putWithTimestamps(TABLE_REF, values); verifyNoMoreInteractions(delegate); } @@ -431,7 +384,7 @@ public void putWithTimestamps() throws Exception { public void truncateTable() throws Exception { kvs.truncateTable(TABLE_REF); - checkSpan("atlasdb-kvs.truncateTable(test.testTable)"); + checkSpan("atlasdb-kvs.truncateTable({table})"); verify(delegate).truncateTable(TABLE_REF); verifyNoMoreInteractions(delegate); } @@ -440,7 +393,7 @@ public void truncateTable() throws Exception { public void truncateTables() throws Exception { kvs.truncateTables(ImmutableSet.of(TABLE_REF)); - checkSpan("atlasdb-kvs.truncateTables([test.testTable])"); + checkSpan("atlasdb-kvs.truncateTables([{table}])"); verify(delegate).truncateTables(ImmutableSet.of(TABLE_REF)); verifyNoMoreInteractions(delegate); } diff --git a/atlasdb-client/src/test/java/com/palantir/atlasdb/logging/LoggingArgsTest.java b/atlasdb-client/src/test/java/com/palantir/atlasdb/logging/LoggingArgsTest.java index 465787822f8..ac81b57a79f 100644 --- a/atlasdb-client/src/test/java/com/palantir/atlasdb/logging/LoggingArgsTest.java +++ b/atlasdb-client/src/test/java/com/palantir/atlasdb/logging/LoggingArgsTest.java @@ -234,4 +234,25 @@ public void returnsUnsafeBatchBatchColumnRangeEvenWhenContainsSafeColumns() { assertThat(LoggingArgs.batchColumnRangeSelection(SAFE_TABLE_REFERENCE, MIXED_BATCH_COLUMN_RANGE)) .isInstanceOf(UnsafeArg.class); } + + @Test + public void returnsSafeTableWhenTableIsSafe() { + assertThat(LoggingArgs.safeTableOrPlaceholder(SAFE_TABLE_REFERENCE)).isEqualTo(SAFE_TABLE_REFERENCE); + } + + @Test + public void returnsPlaceholderWhenTableIsUnsafe() { + assertThat(LoggingArgs.safeTableOrPlaceholder(UNSAFE_TABLE_REFERENCE)) + .isEqualTo(LoggingArgs.PLACEHOLDER_TABLE_REFERENCE); + } + + @Test + public void returnsTablesAndPlaceholderWhenTablesAreSafeAndUnsafe() { + List tables = ImmutableList.of(SAFE_TABLE_REFERENCE, UNSAFE_TABLE_REFERENCE); + List returnedList = Lists.newArrayList(LoggingArgs.safeTablesOrPlaceholder(tables)); + List expectedList = Lists.newArrayList(SAFE_TABLE_REFERENCE, + LoggingArgs.PLACEHOLDER_TABLE_REFERENCE); + + assertThat(returnedList).containsOnly(expectedList.toArray(new TableReference[expectedList.size()])); + } } diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index 58cd8fd2b0d..4be3f47d3e8 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -124,6 +124,10 @@ v0.65.1 - AtlasDB tables will now be logged as ``ns.tablename`` instead of ``map[namespace:map[name:ns] tablename:tablename]``. (`Pull Request `__) + * - |fixed| + - TracingKVS now has spans with safe names. + (`Pull Request `__) + .. <<<<------------------------------------------------------------------------------------------------------------->>>> ======= From 53afd9106e50431c61bf1d8200bfd39bd657e0f0 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 9 Nov 2017 14:50:21 +0000 Subject: [PATCH 04/67] Delete docs (#2657) --- .../miscellaneous/dropwizard-metrics.rst | 104 +----------------- 1 file changed, 3 insertions(+), 101 deletions(-) diff --git a/docs/source/miscellaneous/dropwizard-metrics.rst b/docs/source/miscellaneous/dropwizard-metrics.rst index eaa73f45965..be382d42c7e 100644 --- a/docs/source/miscellaneous/dropwizard-metrics.rst +++ b/docs/source/miscellaneous/dropwizard-metrics.rst @@ -1,107 +1,9 @@ .. _dropwizard-metrics: -================== -Dropwizard Metrics -================== +=============== +AtlasDB Metrics +=============== AtlasDB makes use of the Dropwizard `Metrics library `__ to expose a global ``MetricRegistry`` called ``AtlasDbRegistry``. Users of AtlasDB should use ``AtlasDbMetrics.setMetricRegistry`` to inject their own ``MetricRegistry`` for their application prior to initializing the AtlasDB transaction manager. - -Each AtlasDB client will expose their own KeyValueService metrics, as well as CassandraClientPool metrics -for every Cassandra host. -We expose sweep metrics specific to every table that has been swept, as well as aggregate metrics. - -For more details on what information each type of metric provides, we recommend reading -the Metrics `Getting Started Guide `__. - -A reasonably comprehensive list of metrics exposed by AtlasDB can be found below. - -**Gauges** - - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.numBlacklistedHosts`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.requestConnectionExceptionProportion`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.requestFailureProportion`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.approximatePoolSize`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.meanActiveTimeMillis`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.meanBorrowWaitTimeMillis`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.meanIdleTimeMillis`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.numIdle`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.numActive`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.proportionDestroyedByBorrower`` - - ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.proportionDestroyedByEvictor`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.estimated.size`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.eviction.count`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.hit.count`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.hit.ratio`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.load.average.millis`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.load.failure.count`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.load.success.count`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.miss.count`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.miss.ratio`` - - ``com.palantir.atlasdb.cache.TimestampCache.startToCommitTimestamp.cache.request.count`` - -**Histograms** - -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.byteSizeTx`` -- ``com.palantir.atlasdb.sweep.SweepMetrics.cellTimestampPairsExamined`` -- ``com.palantir.atlasdb.sweep.SweepMetrics.staleValuesDeleted`` - -**Meters** - -- ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.requestConnectionExceptions`` -- ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.requestExceptions`` -- ``com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.requests`` -- ``com.palantir.atlasdb.transaction.api.LockAwareTransactionManager.runTaskReadOnly.failures`` -- ``com.palantir.atlasdb.transaction.api.LockAwareTransactionManager.runTaskWithRetry.failures`` -- ``com.palantir.atlasdb.transaction.api.LockAwareTransactionManager.runTaskWithRetry.failures.`` - -Additional failure counts will be dynamically generated based on the returned exceptions, so you may see metrics like -the following: - -- ``com.palantir.atlasdb.transaction.api.LockAwareTransactionManager.runTaskWithRetry.failures.java.lang.IllegalStateException`` - -**Timers** - -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.addGarbageCollectionSentinelValues`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.checkAndSet`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.close`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.compactInternally`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.createTable`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.createTables`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.delete`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.deleteRange`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.dropTable`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.dropTables`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.get`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getAllTableNames`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getAllTimestamps`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getDelegates`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getFirstBatchForRanges`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getLatestTimestamps`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getMetadataForTable`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getMetadataForTables`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getRange`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getRangeOfTimestamps`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getRows`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.getRowsColumnRange`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.multiPut`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.put`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.putMetadataForTable`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.putMetadataForTables`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.putUnlessExists`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.putWithTimestamps`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.supportsCheckAndSet`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.truncateTable`` -- ``com.palantir.atlasdb.keyvalue.api.KeyValueService.truncateTables`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.commitAcquireLocks`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.commitCheckingForConflicts`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.commitPutCommitTs`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.commitTotalTimeSinceTxCreation`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.commitWrite`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.get`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.getRows`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.millisForPunch`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.processedRangeMillis`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.transactionMillis`` -- ``com.palantir.atlasdb.transaction.impl.SnapshotTransaction.waitForCommitTsMillis`` From e25d80602645bb72cd1886cd594b6a0f0bf0fe27 Mon Sep 17 00:00:00 2001 From: Tom Boam Date: Thu, 9 Nov 2017 15:17:35 +0000 Subject: [PATCH 05/67] [20 minute tasks] Add test for when a batch is full (#2655) * [no release notes] Drive-by add test for when a batch is full --- .../atlasdb/sweep/ExaminedCellLimitTest.java | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/ExaminedCellLimitTest.java diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/ExaminedCellLimitTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/ExaminedCellLimitTest.java new file mode 100644 index 00000000000..20238af565c --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/sweep/ExaminedCellLimitTest.java @@ -0,0 +1,174 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.sweep; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Test; + +import com.palantir.atlasdb.keyvalue.api.Cell; + +public class ExaminedCellLimitTest { + private String startingRow; + private int maxCellsToExamine; + private int cellsExamined; + private String currentRow; + + @Test + public void whenSameRowAndLessThanLimit_thenExaminedEnoughIsFalse() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(1); + on("row1"); + + //Then + examinedEnoughCellsIs(false); + } + + @Test + public void whenSameRowAndAtLimit_thenExaminedEnoughIsFalse() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(10); + on("row1"); + + //Then + examinedEnoughCellsIs(false); + } + + @Test + public void whenSameRowAndExceededLimit_thenExaminedEnoughIsFalse() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(11); + on("row1"); + + //Then + examinedEnoughCellsIs(false); + } + + @Test + public void whenSameRowAndGreatlyExceededLimit_thenExaminedEnoughIsFalse() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(1000); + on("row1"); + + //Then + examinedEnoughCellsIs(false); + } + + @Test + public void whenNextRowAndLessThanLimit_thenExaminedEnoughIsFalse() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(1); + on("row2"); + + //Then + examinedEnoughCellsIs(false); + } + + @Test + public void whenNextRowAndAtLimit_thenExaminedEnoughIsTrue() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(10); + on("row2"); + + //Then + examinedEnoughCellsIs(true); + } + + @Test + public void whenNextRowAndGreaterThanLimit_thenExaminedEnoughIsTrue() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(11); + on("row2"); + + //Then + examinedEnoughCellsIs(true); + } + + @Test + public void whenNextRowAndGreatlyExceededLimit_thenExaminedEnoughIsTrue() { + //Given + startingOn("row1"); + maxCellsToExamineIs(10); + + //When + examined(1000); + on("row2"); + + //Then + examinedEnoughCellsIs(true); + } + + //Given + private void startingOn(String rowName) { + startingRow = rowName; + } + + private void maxCellsToExamineIs(int maxCells) { + maxCellsToExamine = maxCells; + } + + //When + private void examined(int examined) { + cellsExamined = examined; + } + + private void on(String curRow) { + currentRow = curRow; + } + + //Then + + private void examinedEnoughCellsIs(boolean expected) { + CellsToSweepPartitioningIterator.ExaminedCellLimit limit + = new CellsToSweepPartitioningIterator.ExaminedCellLimit(startingRow.getBytes(), maxCellsToExamine); + + assertThat(limit.examinedEnoughCells(cellsExamined, cell()), is(expected)); + } + + private Cell cell() { + return Cell.create(currentRow.getBytes(), "COLUMN".getBytes()); + } +} From 88e3ffe913388320fcb19ba63f22f76fb2a61d7a Mon Sep 17 00:00:00 2001 From: Jeremy Kong Date: Thu, 9 Nov 2017 19:34:07 +0000 Subject: [PATCH 06/67] MetricRegistry log level downgrade + multiple timestamp tracker tests (#2636) * change metrics manager to warn plus log the metric name * more timestamp tracker tests * release notes --- .../palantir/atlasdb/util/MetricsManager.java | 10 ++++- .../monitoring/TimestampTrackerTest.java | 44 +++++++++++++++++++ docs/source/release_notes/release-notes.rst | 8 ++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java index ae884890c2d..044b39c1bbf 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java @@ -27,6 +27,7 @@ import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; +import com.palantir.logsafe.SafeArg; public class MetricsManager { @@ -71,7 +72,14 @@ private synchronized void registerMetricWithFqn(String fullyQualifiedMetricName, registeredMetrics.add(fullyQualifiedMetricName); } catch (Exception e) { // Primarily to handle integration tests that instantiate this class multiple times in a row - log.error("Unable to register metric {}", fullyQualifiedMetricName, e); + log.warn("Unable to register metric {}." + + " This may occur if you are running integration tests that don't clean up completely after " + + " themselves, or if you are trying to use multiple TransactionManagers concurrently in the same" + + " JVM (e.g. in a KVS migration). If this is not the case, this is likely to be a product and/or" + + " an AtlasDB bug. This is no cause for immediate alarm, but it does mean that your telemetry for" + + " the aforementioned metric may be reported incorrectly.", + SafeArg.of("metricName", fullyQualifiedMetricName), + e); } } diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/monitoring/TimestampTrackerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/monitoring/TimestampTrackerTest.java index fcde559fdd5..a4d443786e8 100644 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/monitoring/TimestampTrackerTest.java +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/monitoring/TimestampTrackerTest.java @@ -196,6 +196,50 @@ public Long get() { } } + @Test + public void canCreateMultipleDistinctTimestampTrackers() { + when(mockClock.getTick()).thenReturn(0L, CACHE_INTERVAL_NANOS + 1); + + try (TimestampTrackerImpl firstTracker = createTrackerWithClock(mockClock)) { + firstTracker.registerTimestampForTracking(FAKE_METRIC, () -> 1L); + assertThat(getGauge(FAKE_METRIC).getValue()).isEqualTo(1L); + } + + try (TimestampTrackerImpl secondTracker = createTrackerWithClock(mockClock)) { + secondTracker.registerTimestampForTracking(FAKE_METRIC, () -> 2L); + assertThat(getGauge(FAKE_METRIC).getValue()).isEqualTo(2L); + } + } + + @Test + public void cachesNotSharedAcrossDistinctTimestampTrackers() { + when(mockClock.getTick()).thenReturn(0L, CACHE_INTERVAL_NANOS - 1); + + try (TimestampTrackerImpl firstTracker = createTrackerWithClock(mockClock)) { + firstTracker.registerTimestampForTracking(FAKE_METRIC, () -> 1L); + assertThat(getGauge(FAKE_METRIC).getValue()).isEqualTo(1L); + } + + try (TimestampTrackerImpl secondTracker = createTrackerWithClock(mockClock)) { + secondTracker.registerTimestampForTracking(FAKE_METRIC, () -> 2L); + + // If caches were shared, this would return 1L because we haven't had a cache interval yet. + assertThat(getGauge(FAKE_METRIC).getValue()).isEqualTo(2L); + } + } + + @Test + public void doesNotThrowIfMetricsAreAccidentallyRegisteredMultipleTimes() { + try (TimestampTrackerImpl firstTracker = createTrackerWithClock(mockClock); + TimestampTrackerImpl secondTracker = createTrackerWithClock(mockClock)) { + firstTracker.registerTimestampForTracking(FAKE_METRIC, () -> 1L); + secondTracker.registerTimestampForTracking(FAKE_METRIC, () -> 2L); // OK + + // No guarantees on the value, other than that it's one of them + assertThat(getGauge(FAKE_METRIC).getValue()).isIn(1L, 2L); + } + } + private TimestampTrackerImpl createTrackerWithClock(Clock clock) { return new TimestampTrackerImpl(clock, timelockService, cleaner); } diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index 4be3f47d3e8..c1a1abefab8 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -48,6 +48,11 @@ develop * - Type - Change + * - |fixed| |metrics| + - ``MetricsManager`` now logs failures to register metrics at ``WARN`` instead of ``ERROR``, as failure to do so is not necessarily a systemic failure. + Also, we now log the name of the metric as a Safe argument (previously it was logged as Unsafe). + (`Pull Request `__) + * - |improved| |devbreak| - AtlasDB will now consistently throw a ``InsufficientConsistencyException`` if Cassandra reports an ``UnavailableException``. Also, all Cassandra KVS exceptions like ``KeyAlreadyExists`` or ``TTransportException`` as well as ``NotInitializedException`` will get wrapped into ``AtlasDbDependencyException`` in the interest of consistent exceptions. @@ -80,9 +85,6 @@ develop * - |improved| |logs| - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not - * - - - - .. <<<<------------------------------------------------------------------------------------------------------------->>>> ======= From 8b678558dbe281bdc9ff00b3c5017f9ba7530818 Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Fri, 10 Nov 2017 10:31:08 +0000 Subject: [PATCH 07/67] Extract interface for Cassandra client (#2660) * Create a CassandraClient * Propagate CassandraClient to all classes but CKVS * Use CassandraClient on CKVS * Propagate CassandraClient to remaining Impl classes * Use CassandraClient in tests * [no release notes] --- .../CassandraClientPoolIntegrationTest.java | 20 ++-- ...ssandraKeyValueServiceIntegrationTest.java | 4 +- .../SchemaMutationLockTestTools.java | 3 +- .../keyvalue/cassandra/CassandraClient.java | 68 +++++++++++++ .../cassandra/CassandraClientFactory.java | 22 +++-- .../cassandra/CassandraClientImpl.java | 96 +++++++++++++++++++ .../cassandra/CassandraClientPool.java | 12 +-- .../cassandra/CassandraClientPoolImpl.java | 28 +++--- .../CassandraClientPoolingContainer.java | 26 ++--- .../CassandraKeyValueServiceImpl.java | 79 +++++++-------- .../cassandra/CassandraKeyValueServices.java | 2 +- .../keyvalue/cassandra/CassandraTables.java | 13 ++- .../CassandraTimestampBackupRunner.java | 7 +- .../CassandraTimestampBoundStore.java | 13 +-- .../keyvalue/cassandra/CassandraVerifier.java | 20 ++-- .../keyvalue/cassandra/CqlExecutor.java | 7 +- .../atlasdb/keyvalue/cassandra/Heartbeat.java | 6 +- .../cassandra/SchemaMutationLock.java | 7 +- .../cassandra/SchemaMutationLockTables.java | 13 ++- .../cassandra/TracingQueryRunner.java | 9 +- .../keyvalue/cassandra/paging/RowGetter.java | 6 +- .../cassandra/CassandraClientPoolTest.java | 11 +-- 22 files changed, 317 insertions(+), 155 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java diff --git a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolIntegrationTest.java b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolIntegrationTest.java index d600bb24431..3fca3851820 100644 --- a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolIntegrationTest.java +++ b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolIntegrationTest.java @@ -29,7 +29,6 @@ import java.util.Map.Entry; import java.util.NavigableMap; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.KsDef; import org.apache.cassandra.thrift.TokenRange; import org.apache.thrift.TException; @@ -99,7 +98,8 @@ public void testTokenMapping() { public void testSanitiseReplicationFactorPassesForTheKeyspace() { clientPool.run(client -> { try { - CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client, CassandraContainer.KVS_CONFIG); + CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client.rawClient(), + CassandraContainer.KVS_CONFIG); } catch (TException e) { fail("currentRf On Keyspace does not Match DesiredRf"); } @@ -111,7 +111,7 @@ public void testSanitiseReplicationFactorPassesForTheKeyspace() { public void testSanitiseReplicationFactorFailsAfterManipulatingReplicationFactorInConfig() { clientPool.run(client -> { try { - CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client, + CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client.rawClient(), ImmutableCassandraKeyValueServiceConfig.copyOf( CassandraContainer.KVS_CONFIG).withReplicationFactor( MODIFIED_REPLICATION_FACTOR)); @@ -128,7 +128,8 @@ public void testSanitiseReplicationFactorFailsAfterManipulatingReplicationFactor changeReplicationFactor(MODIFIED_REPLICATION_FACTOR); clientPool.run(client -> { try { - CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client, CassandraContainer.KVS_CONFIG); + CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client.rawClient(), + CassandraContainer.KVS_CONFIG); fail("currentRf On Keyspace Matches DesiredRf after manipulating the cassandra keyspace"); } catch (Exception e) { assertReplicationFactorMismatchError(e); @@ -145,13 +146,14 @@ private void assertReplicationFactorMismatchError(Exception ex) { } private void changeReplicationFactor(int replicationFactor) throws TException { - clientPool.run((FunctionCheckedException) client -> { - KsDef originalKsDef = client.describe_keyspace(CassandraContainer.KVS_CONFIG.getKeyspaceOrThrow()); + clientPool.run((FunctionCheckedException) client -> { + KsDef originalKsDef = client.rawClient().describe_keyspace( + CassandraContainer.KVS_CONFIG.getKeyspaceOrThrow()); KsDef modifiedKsDef = originalKsDef.deepCopy(); modifiedKsDef.setStrategy_class(CassandraConstants.NETWORK_STRATEGY); modifiedKsDef.setStrategy_options(ImmutableMap.of("dc1", Integer.toString(replicationFactor))); modifiedKsDef.setCf_defs(ImmutableList.of()); - client.system_update_keyspace(modifiedKsDef); + client.rawClient().system_update_keyspace(modifiedKsDef); return null; }); } @@ -169,8 +171,8 @@ public void testPoolGivenNoOptionTalksToBlacklistedHosts() { clientPool.blacklistedHosts.clear(); } - private FunctionCheckedException, Exception> describeRing = - client -> client.describe_ring("atlasdb"); + private FunctionCheckedException, Exception> describeRing = + client -> client.rawClient().describe_ring("atlasdb"); @Test public void testWeightedHostsWithUniformActivity() { diff --git a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceIntegrationTest.java b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceIntegrationTest.java index 66371901a89..115314b03c6 100644 --- a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceIntegrationTest.java +++ b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceIntegrationTest.java @@ -161,7 +161,7 @@ public void testGcGraceSecondsUpgradeIsApplied() throws TException { private void assertThatGcGraceSecondsIs(CassandraKeyValueService kvs, int gcGraceSeconds) throws TException { List knownCfs = kvs.getClientPool().runWithRetry(client -> - client.describe_keyspace("atlasdb").getCf_defs()); + client.rawClient().describe_keyspace("atlasdb").getCf_defs()); CfDef clusterSideCf = Iterables.getOnlyElement(knownCfs.stream() .filter(cf -> cf.getName().equals(getInternalTestTableName())) .collect(Collectors.toList())); @@ -185,7 +185,7 @@ public void testCfEqualityChecker() throws TException { kvs.createTable(testTable, tableMetadata); List knownCfs = kvs.getClientPool().runWithRetry(client -> - client.describe_keyspace("atlasdb").getCf_defs()); + client.rawClient().describe_keyspace("atlasdb").getCf_defs()); CfDef clusterSideCf = Iterables.getOnlyElement(knownCfs.stream() .filter(cf -> cf.getName().equals(getInternalTestTableName())) .collect(Collectors.toList())); diff --git a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java index 577a7e0e581..de51535bc7e 100644 --- a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java +++ b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java @@ -19,7 +19,6 @@ import java.nio.charset.StandardCharsets; import java.util.List; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.Compression; import org.apache.cassandra.thrift.ConsistencyLevel; @@ -106,7 +105,7 @@ private static String getHexEncodedBytes(String str) { return CassandraKeyValueServices.encodeAsHex(str.getBytes(StandardCharsets.UTF_8)); } - private static CqlResult runCqlQuery(String query, Cassandra.Client client, ConsistencyLevel consistency) + private static CqlResult runCqlQuery(String query, CassandraClient client, ConsistencyLevel consistency) throws TException { ByteBuffer queryBuffer = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); return client.execute_cql3_query(queryBuffer, Compression.NONE, consistency); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java new file mode 100644 index 00000000000..8a9c40e1957 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.thrift.CASResult; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.ColumnParent; +import org.apache.cassandra.thrift.ColumnPath; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.InvalidRequestException; +import org.apache.cassandra.thrift.KeyRange; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.NotFoundException; +import org.apache.cassandra.thrift.SchemaDisagreementException; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.TimedOutException; +import org.apache.cassandra.thrift.UnavailableException; + +@SuppressWarnings({"all"}) // thrift variable names. +public interface CassandraClient { + Cassandra.Client rawClient(); + + Map> multiget_slice(List keys, ColumnParent column_parent, + SlicePredicate predicate, ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; + + List get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; + + void batch_mutate(Map>> mutation_map, ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; + + public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) + throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, + org.apache.thrift.TException; + + CASResult cas(ByteBuffer key, String column_family, List expected, List updates, + ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; + + public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) + throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, + org.apache.thrift.TException; +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index c43960dfc18..74ecb10e19c 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -51,7 +51,7 @@ import com.palantir.logsafe.UnsafeArg; import com.palantir.remoting3.config.ssl.SslSocketFactories; -public class CassandraClientFactory extends BasePooledObjectFactory { +public class CassandraClientFactory extends BasePooledObjectFactory { private static final Logger log = LoggerFactory.getLogger(CassandraClientFactory.class); private static final LoadingCache sslSocketFactories = @@ -75,9 +75,9 @@ public CassandraClientFactory(InetSocketAddress addr, CassandraKeyValueServiceCo } @Override - public Client create() throws Exception { + public CassandraClient create() throws Exception { try { - return getClient(addr, config); + return instrumentClient(getClient(addr, config)); } catch (Exception e) { String message = String.format("Failed to construct client for %s/%s", addr, config.getKeyspaceOrThrow()); if (config.usingSsl()) { @@ -87,6 +87,10 @@ public Client create() throws Exception { } } + private CassandraClient instrumentClient(Client client) { + return new CassandraClientImpl(client); + } + private static Cassandra.Client getClient(InetSocketAddress addr, CassandraKeyValueServiceConfig config) throws Exception { Client ret = getClientInternal(addr, config); @@ -167,18 +171,18 @@ private static void login(Client client, CassandraCredentialsConfig config) thro } @Override - public boolean validateObject(PooledObject client) { - return client.getObject().getOutputProtocol().getTransport().isOpen(); + public boolean validateObject(PooledObject client) { + return client.getObject().rawClient().getOutputProtocol().getTransport().isOpen(); } @Override - public PooledObject wrap(Client client) { - return new DefaultPooledObject(client); + public PooledObject wrap(CassandraClient client) { + return new DefaultPooledObject<>(client); } @Override - public void destroyObject(PooledObject client) { - client.getObject().getOutputProtocol().getTransport().close(); + public void destroyObject(PooledObject client) { + client.getObject().rawClient().getOutputProtocol().getTransport().close(); log.debug("Closed transport for client {} of host {}", UnsafeArg.of("client", client), SafeArg.of("cassandraClient", CassandraLogHelper.host(addr))); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java new file mode 100644 index 00000000000..7efc9087bf9 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java @@ -0,0 +1,96 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.thrift.CASResult; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.ColumnParent; +import org.apache.cassandra.thrift.ColumnPath; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.InvalidRequestException; +import org.apache.cassandra.thrift.KeyRange; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.NotFoundException; +import org.apache.cassandra.thrift.SchemaDisagreementException; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.TimedOutException; +import org.apache.cassandra.thrift.UnavailableException; +import org.apache.thrift.TException; + +@SuppressWarnings({"all"}) // thrift variable names. +public class CassandraClientImpl implements CassandraClient { + private Cassandra.Client client; + + public CassandraClientImpl(Cassandra.Client client) { + this.client = client; + } + + @Override + public Cassandra.Client rawClient() { + return client; + } + + @Override + public Map> multiget_slice(List keys, ColumnParent column_parent, + SlicePredicate predicate, ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + return client.multiget_slice(keys, column_parent, predicate, consistency_level); + } + + @Override + public List get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + return client.get_range_slices(column_parent, predicate, range, consistency_level); + } + + @Override + public void batch_mutate(Map>> mutation_map, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + client.batch_mutate(mutation_map, consistency_level); + } + + @Override + public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) + throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { + return client.get(key, column_path, consistency_level); + } + + @Override + public CASResult cas(ByteBuffer key, String column_family, List expected, List updates, + ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + return client.cas(key, column_family, expected, updates, serial_consistency_level, commit_consistency_level); + } + + @Override + public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) + throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, + TException { + return client.execute_cql3_query(query, compression, consistency); + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPool.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPool.java index 3fdbd654275..9da25d5b211 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPool.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPool.java @@ -20,19 +20,17 @@ import java.net.UnknownHostException; import java.util.Map; -import org.apache.cassandra.thrift.Cassandra; - import com.palantir.common.base.FunctionCheckedException; public interface CassandraClientPool { - FunctionCheckedException getValidatePartitioner(); + FunctionCheckedException getValidatePartitioner(); V runOnHost(InetSocketAddress specifiedHost, - FunctionCheckedException fn) throws K; - V run(FunctionCheckedException fn) throws K; + FunctionCheckedException fn) throws K; + V run(FunctionCheckedException fn) throws K; V runWithRetryOnHost( InetSocketAddress specifiedHost, - FunctionCheckedException fn) throws K; - V runWithRetry(FunctionCheckedException fn) throws K; + FunctionCheckedException fn) throws K; + V runWithRetry(FunctionCheckedException fn) throws K; InetSocketAddress getAddressForHost(String host) throws UnknownHostException; InetSocketAddress getRandomHostForKey(byte[] key); Map getCurrentPools(); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java index 4fcd7e4d85e..8efeb1a8259 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java @@ -561,23 +561,23 @@ private void refreshTokenRanges() { } } - private FunctionCheckedException, Exception> describeRing = - new FunctionCheckedException, Exception>() { + private FunctionCheckedException, Exception> describeRing = + new FunctionCheckedException, Exception>() { @Override - public List apply(Cassandra.Client client) throws Exception { - return client.describe_ring(config.getKeyspaceOrThrow()); + public List apply(CassandraClient client) throws Exception { + return client.rawClient().describe_ring(config.getKeyspaceOrThrow()); } }; @Override - public V runWithRetry(FunctionCheckedException fn) throws K { + public V runWithRetry(FunctionCheckedException fn) throws K { return runWithRetryOnHost(getRandomGoodHost().getHost(), fn); } @Override public V runWithRetryOnHost( InetSocketAddress specifiedHost, - FunctionCheckedException fn) throws K { + FunctionCheckedException fn) throws K { int numTries = 0; boolean shouldRetryOnDifferentHost = false; Set triedHosts = Sets.newHashSet(); @@ -630,20 +630,20 @@ public V runWithRetryOnHost( } @Override - public V run(FunctionCheckedException fn) throws K { + public V run(FunctionCheckedException fn) throws K { return runOnHost(getRandomGoodHost().getHost(), fn); } @Override public V runOnHost(InetSocketAddress specifiedHost, - FunctionCheckedException fn) throws K { + FunctionCheckedException fn) throws K { CassandraClientPoolingContainer hostPool = currentPools.get(specifiedHost); return runWithPooledResourceRecordingMetrics(hostPool, fn); } private V runWithPooledResourceRecordingMetrics( CassandraClientPoolingContainer hostPool, - FunctionCheckedException fn) throws K { + FunctionCheckedException fn) throws K { recordRequestOnHost(hostPool); try { @@ -823,17 +823,17 @@ static boolean isFastFailoverException(Throwable ex) { || isFastFailoverException(ex.getCause())); } - private final FunctionCheckedException validatePartitioner = - new FunctionCheckedException() { + private final FunctionCheckedException validatePartitioner = + new FunctionCheckedException() { @Override - public Void apply(Cassandra.Client client) throws Exception { - CassandraVerifier.validatePartitioner(client.describe_partitioner(), config); + public Void apply(CassandraClient client) throws Exception { + CassandraVerifier.validatePartitioner(client.rawClient().describe_partitioner(), config); return null; } }; @Override - public FunctionCheckedException getValidatePartitioner() { + public FunctionCheckedException getValidatePartitioner() { return validatePartitioner; } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolingContainer.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolingContainer.java index dce05fa63f9..9b771bb0b49 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolingContainer.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolingContainer.java @@ -25,7 +25,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import org.apache.cassandra.thrift.Cassandra.Client; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.thrift.protocol.TProtocolException; @@ -46,7 +45,7 @@ import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.UnsafeArg; -public class CassandraClientPoolingContainer implements PoolingContainer { +public class CassandraClientPoolingContainer implements PoolingContainer { private static final Logger log = LoggerFactory.getLogger(CassandraClientPoolingContainer.class); private final InetSocketAddress host; @@ -54,7 +53,7 @@ public class CassandraClientPoolingContainer implements PoolingContainer private final MetricsManager metricsManager = new MetricsManager(); private final AtomicLong count = new AtomicLong(); private final AtomicInteger openRequests = new AtomicInteger(); - private final GenericObjectPool clientPool; + private final GenericObjectPool clientPool; public CassandraClientPoolingContainer( InetSocketAddress host, @@ -90,7 +89,7 @@ protected int getPoolSize() { } @Override - public V runWithPooledResource(FunctionCheckedException fn) + public V runWithPooledResource(FunctionCheckedException fn) throws K { final String origName = Thread.currentThread().getName(); Thread.currentThread().setName(origName @@ -111,16 +110,16 @@ public V runWithPooledResource(FunctionCheckedException } @Override - public V runWithPooledResource(Function fn) { + public V runWithPooledResource(Function fn) { throw new UnsupportedOperationException("you should use FunctionCheckedException " + "to ensure the TTransportException type is propagated correctly."); } @SuppressWarnings("unchecked") - private V runWithGoodResource(FunctionCheckedException fn) + private V runWithGoodResource(FunctionCheckedException fn) throws K { boolean shouldReuse = true; - Client resource = null; + CassandraClient resource = null; try { resource = clientPool.borrowObject(); return fn.apply(resource); @@ -153,10 +152,11 @@ private V runWithGoodResource(FunctionCheckedException< } } - private static void eagerlyCleanupReadBuffersFromIdleConnection(Client idleClient, InetSocketAddress host) { + private static void eagerlyCleanupReadBuffersFromIdleConnection(CassandraClient idleClient, + InetSocketAddress host) { // eagerly cleanup idle-connection read buffer to keep a smaller memory footprint try { - TTransport transport = idleClient.getInputProtocol().getTransport(); + TTransport transport = idleClient.rawClient().getInputProtocol().getTransport(); if (transport instanceof TFramedTransport) { Field readBuffer = ((TFramedTransport) transport).getClass().getDeclaredField("readBuffer_"); readBuffer.setAccessible(true); @@ -181,7 +181,7 @@ private static boolean isInvalidClientConnection(Exception ex) { || ex instanceof NoSuchElementException; } - private void invalidateQuietly(Client resource) { + private void invalidateQuietly(CassandraClient resource) { try { log.debug("Discarding {} of host {}", UnsafeArg.of("pool", resource), @@ -234,7 +234,7 @@ public String toString() { * * @param poolNumber number of the pool for metric registration. */ - private GenericObjectPool createClientPool(int poolNumber) { + private GenericObjectPool createClientPool(int poolNumber) { CassandraClientFactory cassandraClientFactory = new CassandraClientFactory(host, config); GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); @@ -261,12 +261,12 @@ private GenericObjectPool createClientPool(int poolNumber) { poolConfig.setTestWhileIdle(true); poolConfig.setJmxNamePrefix(CassandraLogHelper.host(host)); - GenericObjectPool pool = new GenericObjectPool<>(cassandraClientFactory, poolConfig); + GenericObjectPool pool = new GenericObjectPool<>(cassandraClientFactory, poolConfig); registerMetrics(pool, poolNumber); return pool; } - private void registerMetrics(GenericObjectPool pool, int poolNumber) { + private void registerMetrics(GenericObjectPool pool, int poolNumber) { registerMetric(getMetricName("meanActiveTimeMillis", poolNumber), pool::getMeanActiveTimeMillis); registerMetric(getMetricName("meanIdleTimeMillis", poolNumber), pool::getMeanIdleTimeMillis); registerMetric(getMetricName("meanBorrowWaitTimeMillis", poolNumber), pool::getMeanBorrowWaitTimeMillis); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java index cfdf0b7b871..99068879d07 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java @@ -38,7 +38,6 @@ import java.util.stream.Stream; import org.apache.cassandra.thrift.CASResult; -import org.apache.cassandra.thrift.Cassandra.Client; import org.apache.cassandra.thrift.CfDef; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; @@ -316,7 +315,7 @@ private void upgradeFromOlderInternalSchema() { final Collection updatedCfs = Lists.newArrayListWithExpectedSize(metadataForTables.size()); List knownCfs = clientPool.runWithRetry(client -> - client.describe_keyspace(configManager.getConfig().getKeyspaceOrThrow()).getCf_defs()); + client.rawClient().describe_keyspace(configManager.getConfig().getKeyspaceOrThrow()).getCf_defs()); for (CfDef clusterSideCf : knownCfs) { TableReference tableRef = tableReferenceFromCfDef(clusterSideCf); @@ -363,10 +362,10 @@ private void lowerConsistencyWhenSafe() { try { dcs = clientPool.runWithRetry(client -> CassandraVerifier.sanityCheckDatacenters( - client, + client.rawClient(), config)); KsDef ksDef = clientPool.runWithRetry(client -> - client.describe_keyspace(config.getKeyspaceOrThrow())); + client.rawClient().describe_keyspace(config.getKeyspaceOrThrow())); strategyOptions = Maps.newHashMap(ksDef.getStrategy_options()); if (dcs.size() == 1) { @@ -441,9 +440,9 @@ private Map getRowsForSingleHost(final InetSocketAddress host, for (final List batch : Lists.partition(rows, fetchBatchCount)) { rowCount += batch.size(); result.putAll(clientPool.runWithRetryOnHost(host, - new FunctionCheckedException, Exception>() { + new FunctionCheckedException, Exception>() { @Override - public Map apply(Client client) throws Exception { + public Map apply(CassandraClient client) throws Exception { // We want to get all the columns in the row so set start and end to empty. SlicePredicate pred = SlicePredicates.create(Range.ALL, Limit.NO_LIMIT); @@ -627,9 +626,9 @@ private List> getLoadWithTsTasksForSingleHost(final InetSocketAdd } for (final List partition : Lists.partition(ImmutableList.copyOf(columnCells), fetchBatchCount)) { Callable multiGetCallable = () -> clientPool.runWithRetryOnHost(host, - new FunctionCheckedException() { + new FunctionCheckedException() { @Override - public Void apply(Client client) throws Exception { + public Void apply(CassandraClient client) throws Exception { Range range = Range.singleColumn(col, startTs); Limit limit = loadAllTs ? Limit.NO_LIMIT : Limit.ONE; SlicePredicate predicate = SlicePredicates.create(range, limit); @@ -782,9 +781,11 @@ private RowColumnRangeExtractor.RowColumnRangeResult getRowsColumnRangeForSingle long startTs) { try { return clientPool.runWithRetryOnHost(host, - new FunctionCheckedException() { + new FunctionCheckedException() { @Override - public RowColumnRangeExtractor.RowColumnRangeResult apply(Client client) throws Exception { + public RowColumnRangeExtractor.RowColumnRangeResult apply(CassandraClient client) + throws Exception { Range range = createColumnRange( batchColumnRangeSelection.getStartCol(), batchColumnRangeSelection.getEndCol(), startTs); @@ -836,11 +837,11 @@ protected TokenBackedBasicResultsPage, byte[]> getNextPage( TokenBackedBasicResultsPage, byte[]> page(final byte[] startCol) throws Exception { return clientPool.runWithRetryOnHost(host, new FunctionCheckedException< - Client, + CassandraClient, TokenBackedBasicResultsPage, byte[]>, Exception>() { @Override - public TokenBackedBasicResultsPage, byte[]> apply(Client client) + public TokenBackedBasicResultsPage, byte[]> apply(CassandraClient client) throws Exception { Range range = createColumnRange(startCol, batchColumnRangeSelection.getEndCol(), startTs); Limit limit = Limit.of(batchColumnRangeSelection.getBatchHint()); @@ -1005,9 +1006,9 @@ private void putForSingleHostInternal(final InetSocketAddress host, final TableReference tableRef, final Iterable> values, final int ttl) throws Exception { - clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { + clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { @Override - public Void apply(Client client) throws Exception { + public Void apply(CassandraClient client) throws Exception { final CassandraKeyValueServiceConfig config = configManager.getConfig(); int mutationBatchCount = config.mutationBatchCount(); int mutationBatchSizeBytes = config.mutationBatchSizeBytes(); @@ -1110,9 +1111,9 @@ private Void multiPutForSingleHostInternal(final InetSocketAddress host, final List batch, long timestamp) throws Exception { final Map>> map = convertToMutations(batch, timestamp); - return clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { + return clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { @Override - public Void apply(Client client) throws Exception { + public Void apply(CassandraClient client) throws Exception { return batchMutateInternal(client, tableRefs, map, writeConsistency); } @@ -1172,14 +1173,14 @@ private Column createColumn(Cell cell, Value value, final int ttl) { return col; } - private void batchMutateInternal(Client client, + private void batchMutateInternal(CassandraClient client, TableReference tableRef, Map>> map, ConsistencyLevel consistency) throws TException { batchMutateInternal(client, ImmutableSet.of(tableRef), map, consistency); } - private Void batchMutateInternal(Client client, + private Void batchMutateInternal(CassandraClient client, Set tableRefs, Map>> map, ConsistencyLevel consistency) throws TException { @@ -1196,7 +1197,7 @@ private Void batchMutateInternal(Client client, } private Map> multigetInternal( - Client client, + CassandraClient client, TableReference tableRef, List rowNames, ColumnParent colFam, @@ -1255,9 +1256,9 @@ public void truncateTables(final Set tablesToTruncate) { } private void runTruncateInternal(final Set tablesToTruncate) throws TException { - clientPool.run(new FunctionCheckedException() { + clientPool.run(new FunctionCheckedException() { @Override - public Void apply(Client client) throws TException { + public Void apply(CassandraClient client) throws TException { for (TableReference tableRef : tablesToTruncate) { truncateInternal(client, tableRef); } @@ -1271,12 +1272,12 @@ public String toString() { }); } - private void truncateInternal(Client client, TableReference tableRef) throws TException { + private void truncateInternal(CassandraClient client, TableReference tableRef) throws TException { for (int tries = 1; tries <= CassandraConstants.MAX_TRUNCATION_ATTEMPTS; tries++) { boolean successful = true; try { queryRunner.run(client, tableRef, () -> { - client.truncate(internalTableName(tableRef)); + client.rawClient().truncate(internalTableName(tableRef)); return true; }); } catch (TException e) { @@ -1324,11 +1325,11 @@ private void deleteOnSingleHost(final InetSocketAddress host, final TableReference tableRef, final Map> cellVersionsMap) { try { - clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { + clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { private int numVersions = 0; @Override - public Void apply(Client client) throws Exception { + public Void apply(CassandraClient client) throws Exception { // Delete must delete in the order of timestamp and we don't trust batch_mutate to do it // atomically so we have to potentially do many deletes if there are many timestamps for the // same key. @@ -1564,8 +1565,8 @@ public void dropTables(final Set tablesToDrop) { private void dropTablesInternal(final Set tablesToDrop) { try { - clientPool.runWithRetry((FunctionCheckedException) client -> { - KsDef ks = client.describe_keyspace(configManager.getConfig().getKeyspaceOrThrow()); + clientPool.runWithRetry((FunctionCheckedException) client -> { + KsDef ks = client.rawClient().describe_keyspace(configManager.getConfig().getKeyspaceOrThrow()); Set existingTables = Sets.newHashSet(); existingTables.addAll(ks.getCf_defs().stream() @@ -1575,7 +1576,7 @@ private void dropTablesInternal(final Set tablesToDrop) { for (TableReference table : tablesToDrop) { CassandraVerifier.sanityCheckTableName(table); if (existingTables.contains(table)) { - client.system_drop_column_family(internalTableName(table)); + client.rawClient().system_drop_column_family(internalTableName(table)); putMetadataWithoutChangingSettings(table, PtBytes.EMPTY_BYTE_ARRAY); } else { log.warn("Ignored call to drop a table ({}) that did not exist.", @@ -1584,7 +1585,7 @@ private void dropTablesInternal(final Set tablesToDrop) { } CassandraKeyValueServices.waitForSchemaVersions( configManager.getConfig(), - client, + client.rawClient(), "(all tables in a call to dropTables)"); return null; }); @@ -1728,7 +1729,7 @@ private void createTablesInternal(final Map tableNamesTo clientPool.runWithRetry(client -> { for (Entry tableEntry : tableNamesToTableMetadata.entrySet()) { try { - client.system_add_column_family(ColumnFamilyDefinitions.getCfDef( + client.rawClient().system_add_column_family(ColumnFamilyDefinitions.getCfDef( configManager.getConfig().getKeyspaceOrThrow(), tableEntry.getKey(), configManager.getConfig().gcGraceSeconds(), @@ -1746,7 +1747,7 @@ private void createTablesInternal(final Map tableNamesTo CassandraKeyValueServices.waitForSchemaVersions( configManager.getConfig(), - client, + client.rawClient(), "(a call to createTables, filtered down to create: " + tableNamesToTableMetadata.keySet() + ")", true); return null; @@ -1968,12 +1969,12 @@ private void putMetadataAndMaybeAlterTables( clientPool.runWithRetry(client -> { if (possiblyNeedToPerformSettingsChanges) { for (CfDef cf : updatedCfs) { - client.system_update_column_family(cf); + client.rawClient().system_update_column_family(cf); } CassandraKeyValueServices.waitForSchemaVersions( configManager.getConfig(), - client, + client.rawClient(), "(all tables in a call to putMetadataForTables)"); } // Done with actual schema mutation, push the metadata @@ -2142,7 +2143,7 @@ public void checkAndSet(final CheckAndSetRequest request) throws CheckAndSetExce } } - private CASResult executeCheckAndSet(Client client, CheckAndSetRequest request) + private CASResult executeCheckAndSet(CassandraClient client, CheckAndSetRequest request) throws TException { try { TableReference table = request.table(); @@ -2242,7 +2243,7 @@ private boolean isClusterQuorumAvaialble(ClusterAvailabilityStatus clusterStatus private boolean doesConfigReplicationFactorMatchWithCluster() { return clientPool.run(client -> { try { - CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client, configManager.getConfig()); + CassandraVerifier.currentRfOnKeyspaceMatchesDesiredRf(client.rawClient(), configManager.getConfig()); return true; } catch (Exception e) { log.warn("The config and Cassandra cluster do not agree on the replication factor.", e); @@ -2314,8 +2315,8 @@ private void alterGcAndTombstoneInternal( int gcGraceSeconds, float tombstoneThresholdRatio) { try { - clientPool.runWithRetry((FunctionCheckedException) client -> { - KsDef ks = client.describe_keyspace(keyspace); + clientPool.runWithRetry((FunctionCheckedException) client -> { + KsDef ks = client.rawClient().describe_keyspace(keyspace); List cfs = ks.getCf_defs(); for (CfDef cf : cfs) { if (cf.getName().equalsIgnoreCase(internalTableName(tableRef))) { @@ -2323,10 +2324,10 @@ private void alterGcAndTombstoneInternal( cf.setCompaction_strategy_options(ImmutableMap.of( "tombstone_threshold", String.valueOf(tombstoneThresholdRatio))); - client.system_update_column_family(cf); + client.rawClient().system_update_column_family(cf); CassandraKeyValueServices.waitForSchemaVersions( configManager.getConfig(), - client, + client.rawClient(), tableRef.getQualifiedName()); log.trace("gc_grace_seconds is set to {} for {}.{}", SafeArg.of("gcGraceSeconds", gcGraceSeconds), UnsafeArg.of("keyspace", keyspace), diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServices.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServices.java index f7f8d9d4a53..b2946fa60a4 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServices.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServices.java @@ -161,7 +161,7 @@ static void warnUserInInitializationIfClusterAlreadyInInconsistentState( clientPool.run(client -> { waitForSchemaVersions( config, - client, + client.rawClient(), "(none, just an initialization check)", true); return null; diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTables.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTables.java index 3c1edc3d906..64bb9571663 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTables.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTables.java @@ -19,7 +19,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.CfDef; import org.apache.cassandra.thrift.KsDef; import org.apache.thrift.TException; @@ -41,9 +40,9 @@ Set getExisting() { String keyspace = configManager.getConfig().getKeyspaceOrThrow(); try { - return clientPool.runWithRetry(new FunctionCheckedException, Exception>() { + return clientPool.runWithRetry(new FunctionCheckedException, Exception>() { @Override - public Set apply(Cassandra.Client client) throws Exception { + public Set apply(CassandraClient client) throws Exception { return getExisting(client, keyspace); } @@ -57,7 +56,7 @@ public String toString() { } } - private Set getExisting(Cassandra.Client client, String keyspace) throws TException { + private Set getExisting(CassandraClient client, String keyspace) throws TException { return getTableNames(client, keyspace, CfDef::getName); } @@ -69,13 +68,13 @@ private Set getExistingLowerCased(String keyspace) throws TException { return clientPool.runWithRetry((client) -> getExistingLowerCased(client, keyspace)); } - private Set getExistingLowerCased(Cassandra.Client client, String keyspace) throws TException { + private Set getExistingLowerCased(CassandraClient client, String keyspace) throws TException { return getTableNames(client, keyspace, cf -> cf.getName().toLowerCase()); } - private Set getTableNames(Cassandra.Client client, String keyspace, + private Set getTableNames(CassandraClient client, String keyspace, Function nameGetter) throws TException { - KsDef ks = client.describe_keyspace(keyspace); + KsDef ks = client.rawClient().describe_keyspace(keyspace); return ks.getCf_defs().stream() .map(nameGetter) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java index 028f9e5d278..18ee96a9de3 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java @@ -20,7 +20,6 @@ import javax.annotation.Nullable; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.Compression; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.CqlResult; @@ -151,7 +150,7 @@ private BoundReadability getReadability(BoundData boundData) { return backupBoundReadable ? BoundReadability.BACKUP : BoundReadability.NEITHER; } - private BoundData getCurrentBoundData(Cassandra.Client client) { + private BoundData getCurrentBoundData(CassandraClient client) { checkTimestampTableExists(); ByteBuffer selectQuery = CassandraTimestampUtils.constructSelectFromTimestampTableQuery(); @@ -171,14 +170,14 @@ private void checkTimestampTableExists() { "[BACKUP/RESTORE] Tried to get timestamp bound data when the timestamp table didn't exist!"); } - private void executeAndVerifyCas(Cassandra.Client client, Map> casMap) { + private void executeAndVerifyCas(CassandraClient client, Map> casMap) { ByteBuffer casQueryBuffer = CassandraTimestampUtils.constructCheckAndSetMultipleQuery(casMap); CqlResult casResult = executeQueryUnchecked(client, casQueryBuffer); CassandraTimestampUtils.verifyCompatible(casResult, casMap); } - private CqlResult executeQueryUnchecked(Cassandra.Client client, ByteBuffer query) { + private CqlResult executeQueryUnchecked(CassandraClient client, ByteBuffer query) { try { return queryRunner().run(client, AtlasDbConstants.TIMESTAMP_TABLE, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java index a13fe76860a..cc220f536a6 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java @@ -21,7 +21,6 @@ import javax.annotation.concurrent.GuardedBy; import org.apache.cassandra.thrift.CASResult; -import org.apache.cassandra.thrift.Cassandra.Client; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; import org.apache.cassandra.thrift.ColumnPath; @@ -124,10 +123,12 @@ private void tryInitialize() { @Override public synchronized long getUpperLimit() { DebugLogger.logger.debug("[GET] Getting upper limit"); - Long upperLimit = clientPool.runWithRetry(new FunctionCheckedException() { + Long upperLimit = clientPool.runWithRetry(new FunctionCheckedException() { + @GuardedBy("CassandraTimestampBoundStore.this") @Override - public Long apply(Client client) { + public Long apply(CassandraClient client) { ByteBuffer rowName = getRowName(); ColumnPath columnPath = new ColumnPath(AtlasDbConstants.TIMESTAMP_TABLE.getQualifiedName()); columnPath.setColumn(getColumnName()); @@ -173,10 +174,10 @@ private long extractUpperLimit(ColumnOrSuperColumn result) { public synchronized void storeUpperLimit(final long limit) { DebugLogger.logger.debug("[PUT] Storing upper limit of {}.", limit); - clientPool.runWithRetry(new FunctionCheckedException() { + clientPool.runWithRetry(new FunctionCheckedException() { @GuardedBy("CassandraTimestampBoundStore.this") @Override - public Void apply(Client client) { + public Void apply(CassandraClient client) { cas(client, currentLimit, limit); return null; } @@ -184,7 +185,7 @@ public Void apply(Client client) { } @GuardedBy("this") - private void cas(Client client, Long oldVal, long newVal) { + private void cas(CassandraClient client, Long oldVal, long newVal) { final CASResult result; try { DebugLogger.logger.info("[CAS] Trying to set upper limit from {} to {}.", oldVal, newVal); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraVerifier.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraVerifier.java index 8e050e9c7b8..11a9ab58b54 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraVerifier.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraVerifier.java @@ -56,8 +56,8 @@ private CassandraVerifier() { // Utility class } - static final FunctionCheckedException healthCheck = client -> { - client.describe_version(); + static final FunctionCheckedException healthCheck = client -> { + client.rawClient().describe_version(); return null; }; @@ -243,13 +243,13 @@ private static boolean attemptedToCreateKeyspaceTwice(InvalidRequestException ex private static void updateExistingKeyspace(CassandraClientPool clientPool, CassandraKeyValueServiceConfig config) throws TException { - clientPool.runWithRetry((FunctionCheckedException) client -> { - KsDef originalKsDef = client.describe_keyspace(config.getKeyspaceOrThrow()); + clientPool.runWithRetry((FunctionCheckedException) client -> { + KsDef originalKsDef = client.rawClient().describe_keyspace(config.getKeyspaceOrThrow()); // there was an existing keyspace // check and make sure it's definition is up to date with our config KsDef modifiedKsDef = originalKsDef.deepCopy(); checkAndSetReplicationFactor( - client, + client.rawClient(), modifiedKsDef, false, config); @@ -257,10 +257,10 @@ private static void updateExistingKeyspace(CassandraClientPool clientPool, Cassa if (!modifiedKsDef.equals(originalKsDef)) { // Can't call system_update_keyspace to update replication factor if CfDefs are set modifiedKsDef.setCf_defs(ImmutableList.of()); - client.system_update_keyspace(modifiedKsDef); + client.rawClient().system_update_keyspace(modifiedKsDef); CassandraKeyValueServices.waitForSchemaVersions( config, - client, + client.rawClient(), "(updating the existing keyspace)"); } @@ -310,7 +310,7 @@ private static void checkAndSetReplicationFactor( sanityCheckReplicationFactor(ks, config, dcs); } - static void currentRfOnKeyspaceMatchesDesiredRf(Client client, CassandraKeyValueServiceConfig config) + static void currentRfOnKeyspaceMatchesDesiredRf(Cassandra.Client client, CassandraKeyValueServiceConfig config) throws TException { KsDef ks = client.describe_keyspace(config.getKeyspaceOrThrow()); Set dcs = sanityCheckDatacenters(client, config); @@ -336,10 +336,10 @@ private static void sanityCheckReplicationFactor(KsDef ks, CassandraKeyValueServ } } - static final FunctionCheckedException + static final FunctionCheckedException underlyingCassandraClusterSupportsCASOperations = client -> { try { - CassandraApiVersion serverVersion = new CassandraApiVersion(client.describe_version()); + CassandraApiVersion serverVersion = new CassandraApiVersion(client.rawClient().describe_version()); log.debug("Connected cassandra thrift version is: {}", SafeArg.of("cassandraVersion", serverVersion)); return serverVersion.supportsCheckAndSet(); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java index 2ac44f7ec4f..587a957d47b 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java @@ -24,7 +24,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.Compression; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.CqlResult; @@ -226,12 +225,12 @@ private RuntimeException wrapIfConsistencyAll(UnavailableException ex) { } } - private FunctionCheckedException createCqlFunction(String query) { + private FunctionCheckedException createCqlFunction(String query) { ByteBuffer queryBytes = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); - return new FunctionCheckedException() { + return new FunctionCheckedException() { @Override - public CqlResult apply(Cassandra.Client client) throws TException { + public CqlResult apply(CassandraClient client) throws TException { return client.execute_cql3_query(queryBytes, Compression.NONE, consistency); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java index 73efdbc2b8b..340a57b517f 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java @@ -18,7 +18,6 @@ import java.util.List; import org.apache.cassandra.thrift.CASResult; -import org.apache.cassandra.thrift.Cassandra.Client; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.UnavailableException; @@ -68,7 +67,7 @@ public void run() { } } - private Void beat(Client client) throws TException { + private Void beat(CassandraClient client) throws TException { Column ourUpdate = SchemaMutationLock.lockColumnFromIdAndHeartbeat(lockId, heartbeatCount + 1); List expected = ImmutableList.of( @@ -90,7 +89,8 @@ private Void beat(Client client) throws TException { return null; } - private CASResult writeDdlLockWithCas(Client client, Column ourUpdate, List expected) throws TException { + private CASResult writeDdlLockWithCas(CassandraClient client, Column ourUpdate, List expected) + throws TException { try { return queryRunner.run(client, lockTable, () -> client.cas( diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java index aa3e1f9e206..46e4d5470c1 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java @@ -27,7 +27,6 @@ import java.util.regex.Pattern; import org.apache.cassandra.thrift.CASResult; -import org.apache.cassandra.thrift.Cassandra.Client; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; import org.apache.cassandra.thrift.ColumnPath; @@ -175,7 +174,7 @@ private long waitForSchemaMutationLock() { final long perOperationNodeId = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE - 2); try { - clientPool.runWithRetry((FunctionCheckedException) client -> { + clientPool.runWithRetry((FunctionCheckedException) client -> { Column ourUpdate = lockColumnFromIdAndHeartbeat(perOperationNodeId, 0); List expected = ImmutableList.of(lockColumnWithValue(GLOBAL_DDL_LOCK_CLEARED_VALUE)); @@ -343,7 +342,7 @@ private boolean trySchemaMutationUnlockOnce(long perOperationNodeId) { } } - private Optional queryExistingLockColumn(Client client) throws TException { + private Optional queryExistingLockColumn(CassandraClient client) throws TException { TableReference lockTableRef = lockTable.getOnlyTable(); ColumnPath columnPath = new ColumnPath(lockTableRef.getQualifiedName()); columnPath.setColumn(getGlobalDdlLockColumnName()); @@ -367,7 +366,7 @@ private void schemaMutationUnlockWithoutCas() { } private CASResult writeDdlLockWithCas( - Client client, + CassandraClient client, List expectedLockValue, Column newLockValue) throws TException { TableReference lockTableRef = lockTable.getOnlyTable(); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTables.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTables.java index 047a3e0efdc..a741aee3c66 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTables.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTables.java @@ -23,7 +23,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.CfDef; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -55,8 +54,8 @@ public Set getAllLockTables() throws TException { return lockTables; } - private Set getAllLockTablesInternal(Cassandra.Client client) throws TException { - return client.describe_keyspace(config.getKeyspaceOrThrow()).getCf_defs().stream() + private Set getAllLockTablesInternal(CassandraClient client) throws TException { + return client.rawClient().describe_keyspace(config.getKeyspaceOrThrow()).getCf_defs().stream() .map(CfDef::getName) .filter(IS_LOCK_TABLE) .map(TableReference::createWithEmptyNamespace) @@ -67,7 +66,7 @@ public TableReference createLockTable() throws TException { return clientPool.runWithRetry(this::createLockTable); } - private TableReference createLockTable(Cassandra.Client client) throws TException { + private TableReference createLockTable(CassandraClient client) throws TException { String lockTableName = LOCK_TABLE_PREFIX + "_" + getUniqueSuffix(); TableReference lockTable = TableReference.createWithEmptyNamespace(lockTableName); log.info("Creating lock table {}", SafeArg.of("schemaMutationTableName", lockTable)); @@ -80,14 +79,14 @@ private String getUniqueSuffix() { return UUID.randomUUID().toString().replace('-', '_'); } - private void createTableInternal(Cassandra.Client client, TableReference tableRef) throws TException { + private void createTableInternal(CassandraClient client, TableReference tableRef) throws TException { CfDef cf = ColumnFamilyDefinitions.getStandardCfDef( config.getKeyspaceOrThrow(), CassandraKeyValueServiceImpl.internalTableName(tableRef)); - client.system_add_column_family(cf); + client.rawClient().system_add_column_family(cf); CassandraKeyValueServices.waitForSchemaVersions( config, - client, + client.rawClient(), tableRef.getQualifiedName(), true); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingQueryRunner.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingQueryRunner.java index 4d6a141be20..424cbddddba 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingQueryRunner.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingQueryRunner.java @@ -20,7 +20,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.cassandra.thrift.Cassandra; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -43,7 +42,7 @@ public interface Action { V run() throws TException; } - public V run(Cassandra.Client client, Set tableRefs, Action action) throws TException { + public V run(CassandraClient client, Set tableRefs, Action action) throws TException { if (shouldTraceQuery(tableRefs)) { return trace(action, client, tableRefs); } else { @@ -56,12 +55,12 @@ public V run(Cassandra.Client client, Set tableRefs, Action< } } - public V run(Cassandra.Client client, TableReference tableRef, Action action) throws TException { + public V run(CassandraClient client, TableReference tableRef, Action action) throws TException { return run(client, ImmutableSet.of(tableRef), action); } - private V trace(Action action, Cassandra.Client client, Set tableRefs) throws TException { - ByteBuffer traceId = client.trace_next_query(); + private V trace(Action action, CassandraClient client, Set tableRefs) throws TException { + ByteBuffer traceId = client.rawClient().trace_next_query(); Stopwatch stopwatch = Stopwatch.createStarted(); boolean failed = false; try { diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java index 9418ba57557..a251a2e8e5e 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java @@ -18,7 +18,6 @@ import java.net.InetSocketAddress; import java.util.List; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.KeyRange; @@ -28,6 +27,7 @@ import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.cassandra.CassandraClient; import com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool; import com.palantir.atlasdb.keyvalue.cassandra.CassandraKeyValueServiceImpl; import com.palantir.atlasdb.keyvalue.cassandra.TracingQueryRunner; @@ -56,9 +56,9 @@ public List getRows(KeyRange keyRange, SlicePredicate slicePredicate) InetSocketAddress host = clientPool.getRandomHostForKey(keyRange.getStart_key()); return clientPool.runWithRetryOnHost( host, - new FunctionCheckedException, Exception>() { + new FunctionCheckedException, Exception>() { @Override - public List apply(Cassandra.Client client) throws Exception { + public List apply(CassandraClient client) throws Exception { try { return queryRunner.run(client, tableRef, () -> client.get_range_slices(colFam, slicePredicate, keyRange, consistency)); diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolTest.java index 56a1f3d42ad..e4363a4a5d7 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolTest.java @@ -32,7 +32,6 @@ import java.util.NoSuchElementException; import java.util.Optional; -import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.InvalidRequestException; import org.apache.cassandra.thrift.TimedOutException; import org.apache.cassandra.thrift.UnavailableException; @@ -258,7 +257,7 @@ private void verifyNumberOfAttemptsOnHost(InetSocketAddress host, int numAttempts) { Mockito.verify(cassandraClientPool.getCurrentPools().get(host), Mockito.times(numAttempts)) .runWithPooledResource( - Mockito.>any()); + Mockito.>any()); } @Test @@ -338,7 +337,7 @@ private CassandraClientPoolingContainer getMockPoolingContainerForHost(InetSocke private void setFailureModeForHost(CassandraClientPoolingContainer poolingContainer, Exception failureMode) { try { when(poolingContainer.runWithPooledResource( - Mockito.>any())) + Mockito.>any())) .thenThrow(failureMode); } catch (Exception e) { throw Throwables.propagate(e); @@ -371,10 +370,10 @@ private void runNoopOnHostWithRetryWithException(InetSocketAddress host, Cassand } } - private FunctionCheckedException noOp() { - return new FunctionCheckedException() { + private FunctionCheckedException noOp() { + return new FunctionCheckedException() { @Override - public Void apply(Cassandra.Client input) throws RuntimeException { + public Void apply(CassandraClient input) throws RuntimeException { return null; } From c0c05f64c7705fe27c4557df8b9b9e9daa523d3e Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 10 Nov 2017 10:40:26 +0000 Subject: [PATCH 08/67] client -> namespace [no release notes] (#2654) --- .../atlasdb/timelock/TimeLockResource.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/TimeLockResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/TimeLockResource.java index 77bef636fc3..1648fd75700 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/TimeLockResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/TimeLockResource.java @@ -35,12 +35,12 @@ import com.palantir.timestamp.TimestampManagementService; import com.palantir.timestamp.TimestampService; -@Path("/{client: [a-zA-Z0-9_-]+}") +@Path("/{namespace: [a-zA-Z0-9_-]+}") public class TimeLockResource { private final Logger log = LoggerFactory.getLogger(TimeLockResource.class); private final Function clientServicesFactory; - private final ConcurrentMap servicesByClient = Maps.newConcurrentMap(); + private final ConcurrentMap servicesByNamespace = Maps.newConcurrentMap(); private final Supplier maxNumberOfClients; public TimeLockResource( @@ -51,46 +51,46 @@ public TimeLockResource( } @Path("/lock") - public LockService getLockService(@Safe @PathParam("client") String client) { - return getOrCreateServices(client).getLockService(); + public LockService getLockService(@Safe @PathParam("namespace") String namespace) { + return getOrCreateServices(namespace).getLockService(); } @Path("/timestamp") - public TimestampService getTimeService(@Safe @PathParam("client") String client) { - return getOrCreateServices(client).getTimestampService(); + public TimestampService getTimeService(@Safe @PathParam("namespace") String namespace) { + return getOrCreateServices(namespace).getTimestampService(); } @Path("/timelock") - public Object getTimelockService(@Safe @PathParam("client") String client) { - return getOrCreateServices(client).getTimelockService().getPresentService(); + public Object getTimelockService(@Safe @PathParam("namespace") String namespace) { + return getOrCreateServices(namespace).getTimelockService().getPresentService(); } @Path("/timestamp-management") - public TimestampManagementService getTimestampManagementService(@Safe @PathParam("client") String client) { - return getOrCreateServices(client).getTimestampManagementService(); + public TimestampManagementService getTimestampManagementService(@Safe @PathParam("namespace") String namespace) { + return getOrCreateServices(namespace).getTimestampManagementService(); } @VisibleForTesting - TimeLockServices getOrCreateServices(String client) { - return servicesByClient.computeIfAbsent(client, this::createNewClient); + TimeLockServices getOrCreateServices(String namespace) { + return servicesByNamespace.computeIfAbsent(namespace, this::createNewClient); } - private TimeLockServices createNewClient(String client) { - Preconditions.checkArgument(!client.equals(PaxosTimeLockConstants.LEADER_ELECTION_NAMESPACE), + private TimeLockServices createNewClient(String namespace) { + Preconditions.checkArgument(!namespace.equals(PaxosTimeLockConstants.LEADER_ELECTION_NAMESPACE), "The client name '%s' is reserved for the leader election service, and may not be " + "used.", PaxosTimeLockConstants.LEADER_ELECTION_NAMESPACE); - if (servicesByClient.size() >= maxNumberOfClients.get()) { + if (servicesByNamespace.size() >= maxNumberOfClients.get()) { log.error( "Unable to create timelock services for client {}, as it would exceed the maximum number of " + "allowed clients ({}). If this is intentional, the maximum number of clients can be " + "increased via the maximum-number-of-clients runtime config property.", - SafeArg.of("client", client), + SafeArg.of("client", namespace), SafeArg.of("maxNumberOfClients", maxNumberOfClients)); throw new IllegalStateException("Maximum number of clients exceeded"); } - return clientServicesFactory.apply(client); + return clientServicesFactory.apply(namespace); } } From fede1c3479cd2d3a306880996688ff944a43f2ed Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Fri, 10 Nov 2017 12:39:07 +0000 Subject: [PATCH 09/67] 0.65.2 and 0.66.0 release notes (#2663) * Release notes banners * fix pr numbers --- docs/source/release_notes/release-notes.rst | 77 ++++++++++++++++++--- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index c1a1abefab8..bd2d1696896 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -64,26 +64,83 @@ develop ``SweepBatchConfig`` values will now be halved with each failure until they reach 1 (previously they only went to about 30% due to another bug). This ensures we fully backoff and gives us the best possible chance of success. Values will slowly increase with each successful run until they are back to their default level. (`Pull Request `__) - * - |fixed| - - Reverted the Cassandra KVS executor PR (`Pull Request `__) that caused a performance regression. - (`Pull Request `__) + * - |improved| |logs| + - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not + (`Pull Request `__) - * - |fixed| - - ``CassandraTimestampBackupRunner`` now logs the backup bound correctly when performing a backup as part of TimeLock migration. - Previously, the bound logged would have been logged as ``null`` or as a relatively arbitrary byte array, depending on the content of the timestamp table when performing migration. - (`Pull Request `__) +.. <<<<------------------------------------------------------------------------------------------------------------->>>> + +======= +v0.66.0 +======= + +7 November 2017 + +.. list-table:: + :widths: 5 40 + :header-rows: 1 + + * - Type + - Change * - |improved| - AtlasDB now depends on Tritium 0.8.3, allowing products to upgrade Tritium without running into ``NoClassDefFound`` and ``NoSuchField`` errors. + (`Pull Request `__) + +.. <<<<------------------------------------------------------------------------------------------------------------->>>> + +=========== +v0.66.0-rc2 +=========== + +6 November 2017 + +.. list-table:: + :widths: 5 40 + :header-rows: 1 + + * - Type + - Change + + * - |improved| + - AtlasDB now depends on Tritium 0.8.1. (`Pull Request `__) - (`Pull Request `__) * - |improved| - AtlasDB can now tag RC releases. (`Pull Request `__) - * - |improved| |logs| - - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not +.. <<<<------------------------------------------------------------------------------------------------------------->>>> + +=========== +v0.66.0-rc1 +=========== + +This version was skipped due to issues on release. No artifacts with this version were ever published. + +.. <<<<------------------------------------------------------------------------------------------------------------->>>> + +======= +v0.65.2 +======= + +6 November 2017 + +.. list-table:: + :widths: 5 40 + :header-rows: 1 + + * - Type + - Change + + * - |fixed| + - Reverted the Cassandra KVS executor PR (`Pull Request `__) that caused a performance regression. + (`Pull Request `__) + + * - |fixed| + - ``CassandraTimestampBackupRunner`` now logs the backup bound correctly when performing a backup as part of TimeLock migration. + Previously, the bound logged would have been logged as ``null`` or as a relatively arbitrary byte array, depending on the content of the timestamp table when performing migration. + (`Pull Request `__) .. <<<<------------------------------------------------------------------------------------------------------------->>>> From a2be74968e4cf41d3232bfdf8263fe10c7e30c12 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 10 Nov 2017 12:55:34 +0000 Subject: [PATCH 10/67] [QoS] Add getNamespace to AtlasDBConfig (#2661) * Add getNamespace [no release notes] * Timelock client config cannot be empty * Make it explicit that unspecified namespace is only possible for InMemoryKVS * CR comments --- .../atlasdb/config/AtlasDbConfig.java | 47 ++++++++++++++----- .../atlasdb/config/TimeLockClientConfig.java | 7 +++ .../atlasdb/config/AtlasDbConfigTest.java | 42 +++++++++++------ .../config/TimeLockClientConfigTest.java | 13 +++++ 4 files changed, 81 insertions(+), 28 deletions(-) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java index 6161854ad4c..7762fe99292 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.memory.InMemoryAtlasDbConfig; @@ -35,6 +36,11 @@ @Value.Immutable public abstract class AtlasDbConfig { + @VisibleForTesting + static final String UNSPECIFIED_NAMESPACE = "unspecified"; + + private String namespace; + public abstract KeyValueServiceConfig keyValueService(); public abstract Optional leader(); @@ -260,7 +266,7 @@ public int getDefaultLockTimeoutSeconds() { protected final void check() { checkLeaderAndTimelockBlocks(); checkLockAndTimestampBlocks(); - checkNamespaceConfig(); + checkNamespaceConfigAndGetNamespace(); } private void checkLeaderAndTimelockBlocks() { @@ -282,25 +288,25 @@ private void checkLockAndTimestampBlocks() { "Lock and timestamp server blocks must either both be present or both be absent."); } - private void checkNamespaceConfig() { + private String checkNamespaceConfigAndGetNamespace() { if (namespace().isPresent()) { - String namespace = namespace().get(); - + String namespaceConfigValue = namespace().get(); keyValueService().namespace().ifPresent(kvsNamespace -> - Preconditions.checkState(kvsNamespace.equals(namespace), + Preconditions.checkState(kvsNamespace.equals(namespaceConfigValue), "If present, keyspace/dbName/sid config should be the same as the" + " atlas root-level namespace config.")); timelock().ifPresent(timelock -> timelock.client().ifPresent(client -> - Preconditions.checkState(client.equals(namespace), + Preconditions.checkState(client.equals(namespaceConfigValue), "If present, the TimeLock client config should be the same as the" + " atlas root-level namespace config."))); + return namespaceConfigValue; } else if (!(keyValueService() instanceof InMemoryAtlasDbConfig)) { Preconditions.checkState(keyValueService().namespace().isPresent(), "Either the atlas root-level namespace" + " or the keyspace/dbName/sid config needs to be set."); - Optional keyValueServiceNamespace = keyValueService().namespace(); + String keyValueServiceNamespace = keyValueService().namespace().get(); if (timelock().isPresent()) { TimeLockClientConfig timeLockConfig = timelock().get(); @@ -313,17 +319,32 @@ private void checkNamespaceConfig() { // (C* keyspace / Postgres dbName / Oracle sid). But changing the name of the TimeLock client // will return the timestamp bound store to 0, so we also need to fast forward the new client bound // to a value above of the original bound. - Preconditions.checkState(timeLockConfig.client().equals(keyValueServiceNamespace), + Preconditions.checkState(timeLockConfig.client().equals(Optional.of(keyValueServiceNamespace)), "AtlasDB refused to start, in order to avoid potential data corruption." + " Please contact AtlasDB support to remediate this. Specific steps are required;" + " DO NOT ATTEMPT TO FIX THIS YOURSELF."); } - } else if (timelock().isPresent()) { - // Special case - empty timelock and empty namespace/keyspace does not make sense - boolean timelockClientNonEmpty = !timelock().get().client().orElse("").isEmpty(); - Preconditions.checkState(timelockClientNonEmpty, - "For InMemoryKVS, the TimeLock client should not be empty"); + return keyValueServiceNamespace; + } else { + Preconditions.checkState(keyValueService() instanceof InMemoryAtlasDbConfig, + "Expecting KeyvalueServiceConfig to be instance of InMemoryAtlasDbConfig, found %s", + keyValueService().getClass()); + if (timelock().isPresent()) { + return timelock().get().client() + .orElseThrow(() -> + new IllegalStateException("For InMemoryKVS, the TimeLock client should not be empty")); + } + return UNSPECIFIED_NAMESPACE; + } + } + + @Value.Derived + @JsonIgnore + public String getNamespaceString() { + if (namespace == null) { + namespace = checkNamespaceConfigAndGetNamespace(); } + return namespace; } private boolean areTimeAndLockConfigsAbsent() { diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java index deaef412333..73afab95986 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.Preconditions; @JsonSerialize(as = ImmutableTimeLockClientConfig.class) @JsonDeserialize(as = ImmutableTimeLockClientConfig.class) @@ -55,4 +56,10 @@ public ServerListConfig toNamespacedServerList() { return ImmutableServerListConfig.copyOf(serversList()) .withServers(serversWithNamespaces); } + + @Value.Check + protected final void check() { + Preconditions.checkArgument(!client().isPresent() || !client().get().isEmpty(), + "Timelock client string cannot be empty"); + } } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java index d9a02a12fd2..6c6758cfb9f 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java @@ -17,6 +17,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -54,23 +55,25 @@ public class AtlasDbConfigTest { private static final Optional OTHER_SSL_CONFIG = Optional.of(mock(SslConfiguration.class)); private static final Optional NO_SSL_CONFIG = Optional.empty(); - private static final TimeLockClientConfig TIMELOCK_CONFIG_WITH_EMPTY_CLIENT = ImmutableTimeLockClientConfig + private static final String TEST_NAMESPACE = "client"; + private static final String OTHER_CLIENT = "other-client"; + + private static final TimeLockClientConfig TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT = ImmutableTimeLockClientConfig .builder() .client(Optional.empty()) .serversList(DEFAULT_SERVER_LIST) .build(); - private static final TimeLockClientConfig TIMELOCK_CONFIG_WITH_OTHER_CLIENT = ImmutableTimeLockClientConfig .builder() - .client("other-client") + .client(OTHER_CLIENT) .serversList(DEFAULT_SERVER_LIST) .build(); @BeforeClass public static void setUp() { when(KVS_CONFIG_WITHOUT_NAMESPACE.namespace()).thenReturn(Optional.empty()); - when(KVS_CONFIG_WITH_OTHER_NAMESPACE.namespace()).thenReturn(Optional.of("other-client")); - when(KVS_CONFIG_WITH_NAMESPACE.namespace()).thenReturn(Optional.of("client")); + when(KVS_CONFIG_WITH_OTHER_NAMESPACE.namespace()).thenReturn(Optional.of(OTHER_CLIENT)); + when(KVS_CONFIG_WITH_NAMESPACE.namespace()).thenReturn(Optional.of(TEST_NAMESPACE)); } @Test @@ -92,6 +95,7 @@ public void configWithLeaderBlockIsValid() { .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .leader(LEADER_CONFIG) .build(); + assertThat(config.getNamespaceString(), equalTo(TEST_NAMESPACE)); assertThat(config, not(nullValue())); } @@ -101,6 +105,7 @@ public void configWithTimelockBlockIsValid() { .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .timelock(TIMELOCK_CONFIG) .build(); + assertThat(config.getNamespaceString(), equalTo(TEST_NAMESPACE)); assertThat(config, not(nullValue())); } @@ -111,6 +116,7 @@ public void remoteLockAndTimestampConfigIsValid() { .lock(DEFAULT_SERVER_LIST) .timestamp(DEFAULT_SERVER_LIST) .build(); + assertThat(config.getNamespaceString(), equalTo(TEST_NAMESPACE)); assertThat(config, not(nullValue())); } @@ -194,7 +200,7 @@ public void absentNamespaceRequiresKvsNamespace() { public void absentNamespaceRequiresTimelockClient() { assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .timelock(TIMELOCK_CONFIG_WITH_EMPTY_CLIENT) + .timelock(TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT) .build()) .isInstanceOf(IllegalStateException.class) .satisfies((exception) -> @@ -211,11 +217,12 @@ public void absentNamespaceRequiresMatchingKvsNamespaceAndTimelockClient() { @Test public void namespaceAcceptsEmptyKvsNamespaceAndTimelockClient() { - ImmutableAtlasDbConfig.builder() + AtlasDbConfig config = ImmutableAtlasDbConfig.builder() .namespace("a client") .keyValueService(KVS_CONFIG_WITHOUT_NAMESPACE) - .timelock(TIMELOCK_CONFIG_WITH_EMPTY_CLIENT) + .timelock(TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT) .build(); + assertThat(config.getNamespaceString(), equalTo("a client")); } @Test @@ -223,10 +230,11 @@ public void inMemoryConfigCanHaveEmptyNamespace() { InMemoryAtlasDbConfig kvsConfig = new InMemoryAtlasDbConfig(); assertFalse("This test assumes the InMemoryAtlasDbConfig has no namespace by default", kvsConfig.namespace().isPresent()); - ImmutableAtlasDbConfig.builder() + ImmutableAtlasDbConfig config = ImmutableAtlasDbConfig.builder() .namespace(Optional.empty()) .keyValueService(kvsConfig) .build(); + assertThat(config.getNamespaceString(), equalTo(AtlasDbConfig.UNSPECIFIED_NAMESPACE)); } @Test @@ -234,10 +242,11 @@ public void inMemoryConfigWorksWithNonTestNamespace() { InMemoryAtlasDbConfig kvsConfig = new InMemoryAtlasDbConfig(); assertFalse("This test assumes the InMemoryAtlasDbConfig has no namespace by default", kvsConfig.namespace().isPresent()); - ImmutableAtlasDbConfig.builder() + AtlasDbConfig config = ImmutableAtlasDbConfig.builder() .namespace("clive") .keyValueService(kvsConfig) .build(); + assertThat(config.getNamespaceString(), equalTo("clive")); } @Test @@ -248,7 +257,7 @@ public void inMemoryConfigCannotHaveEmptyNamespaceWithEmptyTimelockClient() { assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() .namespace(Optional.empty()) .keyValueService(kvsConfig) - .timelock(TIMELOCK_CONFIG_WITH_EMPTY_CLIENT) + .timelock(TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT) .build()) .isInstanceOf(IllegalStateException.class) .satisfies((exception) -> @@ -261,16 +270,17 @@ public void inMemoryKeyspaceAndTimelockClientCanBeDifferent() { InMemoryAtlasDbConfig kvsConfig = new InMemoryAtlasDbConfig(); assertFalse("This test assumes the InMemoryAtlasDbConfig has no namespace by default", kvsConfig.namespace().isPresent()); - ImmutableAtlasDbConfig.builder() + ImmutableAtlasDbConfig config = ImmutableAtlasDbConfig.builder() .keyValueService(kvsConfig) .timelock(TIMELOCK_CONFIG_WITH_OTHER_CLIENT) .build(); + assertThat(config.getNamespaceString(), equalTo(OTHER_CLIENT)); } @Test public void namespaceAndTimelockClientShouldMatch() { assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() - .namespace("client") + .namespace(TEST_NAMESPACE) .keyValueService(KVS_CONFIG_WITHOUT_NAMESPACE) .timelock(TIMELOCK_CONFIG_WITH_OTHER_CLIENT) .build()) @@ -282,9 +292,9 @@ public void namespaceAndTimelockClientShouldMatch() { @Test public void namespaceAndKvsNamespaceShouldMatch() { assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() - .namespace("client") + .namespace(TEST_NAMESPACE) .keyValueService(KVS_CONFIG_WITH_OTHER_NAMESPACE) - .timelock(TIMELOCK_CONFIG_WITH_EMPTY_CLIENT) + .timelock(TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT) .build()) .isInstanceOf(IllegalStateException.class) .satisfies((exception) -> @@ -299,6 +309,7 @@ public void addingFallbackSslAddsItToLeaderBlock() { .build(); AtlasDbConfig withSsl = AtlasDbConfigs.addFallbackSslConfigurationToAtlasDbConfig(withoutSsl, SSL_CONFIG); assertThat(withSsl.leader().get().sslConfiguration(), is(SSL_CONFIG)); + assertThat(withoutSsl.getNamespaceString(), equalTo(TEST_NAMESPACE)); } @Test @@ -310,6 +321,7 @@ public void addingFallbackSslAddsItToLockBlock() { .build(); AtlasDbConfig withSsl = AtlasDbConfigs.addFallbackSslConfigurationToAtlasDbConfig(withoutSsl, SSL_CONFIG); assertThat(withSsl.lock().get().sslConfiguration(), is(SSL_CONFIG)); + assertThat(withoutSsl.getNamespaceString(), equalTo(TEST_NAMESPACE)); } @Test diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/TimeLockClientConfigTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/TimeLockClientConfigTest.java index cb7c0d167f5..c7789f5d2c0 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/TimeLockClientConfigTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/TimeLockClientConfigTest.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.config; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.junit.Assert.assertThat; @@ -74,6 +75,18 @@ public void canCreateWithoutClientSpecified() { .build(); } + @Test + public void tmelockClientCannotBeAnEmptyString() { + assertThatThrownBy(() -> ImmutableTimeLockClientConfig + .builder() + .client("") + .serversList(SERVERS_LIST) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .satisfies((exception) -> + assertThat(exception.getMessage(), containsString("Timelock client string cannot be empty"))); + } + @Test public void throwsWhenReadingClientWithoutClientSpecified() { TimeLockClientConfig config = ImmutableTimeLockClientConfig.builder() From 18d35b6e567e8e5cc57762dcfc32f9e839a9fe1c Mon Sep 17 00:00:00 2001 From: Jeremy Kong Date: Fri, 10 Nov 2017 14:01:44 +0000 Subject: [PATCH 11/67] Live Reloading the TimeLock Block, Part 1: Pull to Push (#2621) * thoughts * More tests for RIH * Paranoid logging * statics * javadoc part 1 * polling refreshable * Unit tests * Remove the old RIH * lock lock * Tests that test how we deal with exceptions * logging * [no release notes] * CR comments part 1 * Make interval configurable * Standard nasty time edge cases * lastSeenValue does not need to be volatile --- atlasdb-cli-distribution/versions.lock | 8 + atlasdb-cli/versions.lock | 16 ++ atlasdb-config/versions.lock | 16 ++ atlasdb-console-distribution/versions.lock | 8 + atlasdb-console/versions.lock | 16 ++ atlasdb-dagger/versions.lock | 16 ++ atlasdb-dbkvs-tests/versions.lock | 16 ++ atlasdb-dropwizard-bundle/versions.lock | 16 ++ atlasdb-ete-tests/versions.lock | 16 ++ atlasdb-feign/build.gradle | 1 + .../atlasdb/http/PollingRefreshable.java | 118 +++++++++++ .../atlasdb/http/PollingRefreshableTest.java | 188 ++++++++++++++++++ atlasdb-feign/versions.lock | 18 +- atlasdb-jepsen-tests/versions.lock | 16 ++ atlasdb-perf/versions.lock | 16 ++ atlasdb-service-server/versions.lock | 16 ++ atlasdb-service/versions.lock | 16 ++ atlasdb-tests-shared/versions.lock | 16 ++ timelock-agent/versions.lock | 16 ++ timelock-impl/versions.lock | 16 ++ timelock-server-distribution/versions.lock | 8 + timelock-server/versions.lock | 16 ++ 22 files changed, 585 insertions(+), 4 deletions(-) create mode 100644 atlasdb-feign/src/main/java/com/palantir/atlasdb/http/PollingRefreshable.java create mode 100644 atlasdb-feign/src/test/java/com/palantir/atlasdb/http/PollingRefreshableTest.java diff --git a/atlasdb-cli-distribution/versions.lock b/atlasdb-cli-distribution/versions.lock index b53dc3f56b1..79576be86be 100644 --- a/atlasdb-cli-distribution/versions.lock +++ b/atlasdb-cli-distribution/versions.lock @@ -239,6 +239,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -258,6 +259,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -622,6 +624,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-cli/versions.lock b/atlasdb-cli/versions.lock index f8f3d9f8756..6f06ed437a5 100644 --- a/atlasdb-cli/versions.lock +++ b/atlasdb-cli/versions.lock @@ -222,6 +222,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -243,6 +244,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -539,6 +541,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1106,6 +1114,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1124,6 +1133,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1420,6 +1430,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-config/versions.lock b/atlasdb-config/versions.lock index 9da63b706f6..b1c67a0048b 100644 --- a/atlasdb-config/versions.lock +++ b/atlasdb-config/versions.lock @@ -191,6 +191,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -206,6 +207,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -417,6 +419,12 @@ "com.palantir.remoting3:keystores": { "locked": "3.5.1" }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -827,6 +835,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -842,6 +851,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1053,6 +1063,12 @@ "com.palantir.remoting3:keystores": { "locked": "3.5.1" }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-console-distribution/versions.lock b/atlasdb-console-distribution/versions.lock index ce08b574ca3..750755ac34a 100644 --- a/atlasdb-console-distribution/versions.lock +++ b/atlasdb-console-distribution/versions.lock @@ -237,6 +237,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -256,6 +257,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -604,6 +606,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-console/versions.lock b/atlasdb-console/versions.lock index 93459414be1..4a79af18d8f 100644 --- a/atlasdb-console/versions.lock +++ b/atlasdb-console/versions.lock @@ -198,6 +198,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -213,6 +214,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -459,6 +461,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -908,6 +916,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -923,6 +932,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1169,6 +1179,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-dagger/versions.lock b/atlasdb-dagger/versions.lock index bb1b1b4fe26..06ee9a23d6c 100644 --- a/atlasdb-dagger/versions.lock +++ b/atlasdb-dagger/versions.lock @@ -198,6 +198,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -216,6 +217,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -462,6 +464,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -898,6 +906,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -916,6 +925,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1162,6 +1172,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-dbkvs-tests/versions.lock b/atlasdb-dbkvs-tests/versions.lock index 417ccdac132..f1b5137710c 100644 --- a/atlasdb-dbkvs-tests/versions.lock +++ b/atlasdb-dbkvs-tests/versions.lock @@ -154,6 +154,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -165,6 +166,7 @@ "com.palantir.common:streams", "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -418,6 +420,12 @@ "com.palantir.atlasdb:atlasdb-impl-shared" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -840,6 +848,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -851,6 +860,7 @@ "com.palantir.common:streams", "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1104,6 +1114,12 @@ "com.palantir.atlasdb:atlasdb-impl-shared" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-dropwizard-bundle/versions.lock b/atlasdb-dropwizard-bundle/versions.lock index bdce814ff2c..54750bea084 100644 --- a/atlasdb-dropwizard-bundle/versions.lock +++ b/atlasdb-dropwizard-bundle/versions.lock @@ -242,6 +242,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -260,6 +261,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -570,6 +572,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1571,6 +1579,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1589,6 +1598,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1899,6 +1909,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-ete-tests/versions.lock b/atlasdb-ete-tests/versions.lock index 8145bb9301f..c3d088c5922 100644 --- a/atlasdb-ete-tests/versions.lock +++ b/atlasdb-ete-tests/versions.lock @@ -245,6 +245,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -263,6 +264,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -595,6 +597,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1642,6 +1650,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1661,6 +1670,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -2040,6 +2050,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-feign/build.gradle b/atlasdb-feign/build.gradle index faa61353207..53d7afbf1d4 100644 --- a/atlasdb-feign/build.gradle +++ b/atlasdb-feign/build.gradle @@ -27,6 +27,7 @@ configurations { dependencies { explicitShadow project(':lock-api') explicitShadow group: 'com.squareup.okhttp3', name: 'okhttp' + explicitShadow group: 'com.palantir.remoting3', name: 'refresh-utils' toBeShaded group: 'com.palantir.remoting2', name: 'error-handling' toBeShaded group: 'com.palantir.remoting2', name: 'jackson-support' diff --git a/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/PollingRefreshable.java b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/PollingRefreshable.java new file mode 100644 index 00000000000..b81ee202e3f --- /dev/null +++ b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/PollingRefreshable.java @@ -0,0 +1,118 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.http; + +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.palantir.common.concurrent.NamedThreadFactory; +import com.palantir.logsafe.UnsafeArg; +import com.palantir.remoting3.ext.refresh.Refreshable; + +/** + * A PollingRefreshable serves as a bridge between a {@link Supplier} and {@link Refreshable}, polling for changes + * in the value of the Supplier and, if detecting a change, pushing it to the linked Refreshable. + * + * @param type of the value supplied / pushed to the Refreshable + */ +public final class PollingRefreshable implements AutoCloseable { + @VisibleForTesting + static final Duration DEFAULT_REFRESH_INTERVAL = Duration.ofSeconds(5L); + + private static final Logger log = LoggerFactory.getLogger(PollingRefreshable.class); + + private final Supplier supplier; + private final Duration refreshInterval; + private final ScheduledExecutorService poller; + + private final Refreshable refreshable = Refreshable.empty(); + + private T lastSeenValue; + + private PollingRefreshable(Supplier supplier, + Duration refreshInterval, + ScheduledExecutorService poller) { + Preconditions.checkArgument(!refreshInterval.isNegative() && !refreshInterval.isZero(), + "Refresh interval must be positive, but found %s", refreshInterval); + + this.supplier = supplier; + this.refreshInterval = refreshInterval; + this.poller = poller; + + try { + lastSeenValue = supplier.get(); + refreshable.set(lastSeenValue); + } catch (Exception e) { + log.info("Exception occurred in supplier when trying to populate the initial value."); + lastSeenValue = null; + } + } + + public static PollingRefreshable create(Supplier supplier) { + return create(supplier, DEFAULT_REFRESH_INTERVAL); + } + + public static PollingRefreshable create(Supplier supplier, Duration refreshInterval) { + return createWithSpecificPoller(supplier, + refreshInterval, + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("polling-refreshable", true))); + } + + @VisibleForTesting + static PollingRefreshable createWithSpecificPoller( + Supplier supplier, + Duration refreshInterval, + ScheduledExecutorService poller) { + PollingRefreshable pollingRefreshable = new PollingRefreshable<>(supplier, refreshInterval, poller); + pollingRefreshable.scheduleUpdates(); + return pollingRefreshable; + } + + public Refreshable getRefreshable() { + return refreshable; + } + + private void scheduleUpdates() { + poller.scheduleAtFixedRate(() -> { + try { + T value = supplier.get(); + if (!value.equals(lastSeenValue)) { + lastSeenValue = value; + refreshable.set(lastSeenValue); + } + } catch (Exception e) { + log.info("Exception occurred in supplier when polling for a new value in our PollingRefreshable." + + " The last value we saw was {}.", + UnsafeArg.of("currentValue", lastSeenValue), + e); + } + }, refreshInterval.getSeconds(), refreshInterval.getSeconds(), TimeUnit.SECONDS); + } + + @Override + public void close() { + poller.shutdown(); + } +} diff --git a/atlasdb-feign/src/test/java/com/palantir/atlasdb/http/PollingRefreshableTest.java b/atlasdb-feign/src/test/java/com/palantir/atlasdb/http/PollingRefreshableTest.java new file mode 100644 index 00000000000..ce50eb87e5f --- /dev/null +++ b/atlasdb-feign/src/test/java/com/palantir/atlasdb/http/PollingRefreshableTest.java @@ -0,0 +1,188 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.http; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +import org.jmock.lib.concurrent.DeterministicScheduler; +import org.junit.Test; + +import com.palantir.remoting3.ext.refresh.Refreshable; + +public class PollingRefreshableTest { + private static final long FORTY_TWO = 42L; + private static final Duration REFRESH_INTERVAL = Duration.of(5, ChronoUnit.HALF_DAYS); + + private final DeterministicScheduler scheduler = new DeterministicScheduler(); + + @Test + public void refreshableIsInitializedWithTheSupplierValue() { + PollingRefreshable pollingRefreshable = createPollingRefreshableWithTestScheduler(() -> 1L); + assertRefreshableContainsAndClear(pollingRefreshable.getRefreshable(), 1L); + } + + @Test + public void refreshableIsNotRepopulatedWithStaleSupplierValuesEvenAfterTheRefreshInterval() { + PollingRefreshable pollingRefreshable = createPollingRefreshableWithTestScheduler(() -> 1L); + assertRefreshableContainsAndClear(pollingRefreshable.getRefreshable(), 1L); + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertThat(pollingRefreshable.getRefreshable().getAndClear()).isNotPresent(); + } + + @Test + public void refreshableIsRepopulatedWithNewSupplierValuesAfterTheRefreshIntervalPasses() { + AtomicLong atomicLong = new AtomicLong(); + PollingRefreshable pollingRefreshable = createPollingRefreshableWithTestScheduler( + atomicLong::incrementAndGet); + Refreshable refreshable = pollingRefreshable.getRefreshable(); + + assertRefreshableContainsAndClear(refreshable, 1L); + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, 2L); + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, 3L); + } + + @Test + public void refreshableIsPopulatedWithTheFreshestValueAfterTheRefreshIntervalPasses() { + AtomicLong atomicLong = new AtomicLong(); + PollingRefreshable pollingRefreshable = createPollingRefreshableWithTestScheduler(atomicLong::get); + Refreshable refreshable = pollingRefreshable.getRefreshable(); + + assertRefreshableContainsAndClear(refreshable, 0L); + atomicLong.set(FORTY_TWO + FORTY_TWO); + scheduler.tick(1, TimeUnit.MILLISECONDS); + atomicLong.set(FORTY_TWO); + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, FORTY_TWO); + } + + @Test + public void refreshableIsNotRepopulatedWithNewSupplierValuesBeforeTheRefreshIntervalPasses() { + AtomicLong atomicLong = new AtomicLong(); + PollingRefreshable pollingRefreshable = createPollingRefreshableWithTestScheduler( + atomicLong::incrementAndGet); + Refreshable refreshable = pollingRefreshable.getRefreshable(); + + assertRefreshableContainsAndClear(refreshable, 1L); + scheduler.tick(REFRESH_INTERVAL.toMillis() - 1, TimeUnit.MILLISECONDS); + assertThat(refreshable.getAndClear()).isEmpty(); + } + + @Test + public void canRecoverFromSupplierThrowingExceptionsInitially() { + Refreshable refreshable = getIncrementingLongRefreshableThrowingOnSpecificValue(1L); + + assertThat(refreshable.getAndClear()).isEmpty(); + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, 2L); + } + + @Test + public void canRecoverFromSupplierThrowingExceptionsLater() { + Refreshable refreshable = getIncrementingLongRefreshableThrowingOnSpecificValue(2L); + + assertRefreshableContainsAndClear(refreshable, 1L); + + // This execution will throw a RuntimeException. + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertThat(refreshable.getAndClear()).isEmpty(); + + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, 3L); + } + + @Test + public void shutsDownExecutorWhenClosed() { + ScheduledExecutorService scheduledExecutor = mock(ScheduledExecutorService.class); + PollingRefreshable pollingRefreshable = PollingRefreshable.createWithSpecificPoller( + () -> 1L, REFRESH_INTERVAL, scheduledExecutor); + pollingRefreshable.close(); + verify(scheduledExecutor).shutdown(); + } + + @Test + public void canCloseMultipleTimes() { + PollingRefreshable pollingRefreshable = PollingRefreshable.create(() -> 1L); + pollingRefreshable.close(); + pollingRefreshable.close(); + } + + @Test + public void canConfigureRefreshInterval() { + Duration doubleRefreshInterval = REFRESH_INTERVAL.multipliedBy(2L); + AtomicLong atomicLong = new AtomicLong(); + PollingRefreshable lessFrequentlyPollingRefreshable = PollingRefreshable.createWithSpecificPoller( + atomicLong::incrementAndGet, doubleRefreshInterval, scheduler); + Refreshable refreshable = lessFrequentlyPollingRefreshable.getRefreshable(); + + assertRefreshableContainsAndClear(refreshable, 1L); + + // This is only about half of our poller's refresh interval, so we shouldn't have polled yet + scheduler.tick(REFRESH_INTERVAL.toMillis() + 1, TimeUnit.MILLISECONDS); + assertThat(refreshable.getAndClear()).isEmpty(); + + // But now we cumulatively have ticked by 2*REFRESH_INTERVAL + 1, so we should have polled + scheduler.tick(REFRESH_INTERVAL.toMillis(), TimeUnit.MILLISECONDS); + assertRefreshableContainsAndClear(refreshable, 2L); + } + + @Test + public void cannotSetRefreshIntervalToZero() { + assertThatThrownBy(() -> PollingRefreshable.create(() -> 1L, Duration.ZERO)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void cannotSetNegativeRefreshInterval() { + assertThatThrownBy(() -> PollingRefreshable.create(() -> 1L, Duration.of(-1, ChronoUnit.SECONDS))) + .isInstanceOf(IllegalArgumentException.class); + } + + private PollingRefreshable createPollingRefreshableWithTestScheduler(Supplier supplier) { + return PollingRefreshable.createWithSpecificPoller(supplier, REFRESH_INTERVAL, scheduler); + } + + private void assertRefreshableContainsAndClear(Refreshable refreshable, T expectedValue) { + assertThat(refreshable.getAndClear()).isPresent().contains(expectedValue); + } + + private Refreshable getIncrementingLongRefreshableThrowingOnSpecificValue(long badValue) { + AtomicLong atomicLong = new AtomicLong(); + PollingRefreshable pollingRefreshable = PollingRefreshable.createWithSpecificPoller( + () -> { + long newValue = atomicLong.incrementAndGet(); + if (newValue == badValue) { + throw new RuntimeException(badValue + " is illegal"); + } + return newValue; + }, + REFRESH_INTERVAL, + scheduler); + return pollingRefreshable.getRefreshable(); + } +} diff --git a/atlasdb-feign/versions.lock b/atlasdb-feign/versions.lock index 11498799ff7..b9f5b599154 100644 --- a/atlasdb-feign/versions.lock +++ b/atlasdb-feign/versions.lock @@ -72,7 +72,8 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting2:error-handling" + "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -80,7 +81,8 @@ "transitive": [ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting2:error-handling" + "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils" ] }, "com.netflix.feign:feign-core": { @@ -130,6 +132,9 @@ "com.palantir.remoting2:error-handling" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1" + }, "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", "transitive": [ @@ -259,7 +264,8 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting2:error-handling" + "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -267,7 +273,8 @@ "transitive": [ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting2:error-handling" + "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils" ] }, "com.netflix.feign:feign-core": { @@ -317,6 +324,9 @@ "com.palantir.remoting2:error-handling" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1" + }, "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", "transitive": [ diff --git a/atlasdb-jepsen-tests/versions.lock b/atlasdb-jepsen-tests/versions.lock index 5fbb79fda03..69aaf58fd80 100644 --- a/atlasdb-jepsen-tests/versions.lock +++ b/atlasdb-jepsen-tests/versions.lock @@ -197,6 +197,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -212,6 +213,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -450,6 +452,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -884,6 +892,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -899,6 +908,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1137,6 +1147,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-perf/versions.lock b/atlasdb-perf/versions.lock index aebbe37f41b..8c2aeec33d3 100644 --- a/atlasdb-perf/versions.lock +++ b/atlasdb-perf/versions.lock @@ -245,6 +245,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -271,6 +272,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -630,6 +632,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1308,6 +1316,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1334,6 +1343,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1693,6 +1703,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-service-server/versions.lock b/atlasdb-service-server/versions.lock index 8d3947c335d..2b533f8d25d 100644 --- a/atlasdb-service-server/versions.lock +++ b/atlasdb-service-server/versions.lock @@ -216,6 +216,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -231,6 +232,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -478,6 +480,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1364,6 +1372,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1382,6 +1391,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1672,6 +1682,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-service/versions.lock b/atlasdb-service/versions.lock index 911bd254c18..4c16cc9c464 100644 --- a/atlasdb-service/versions.lock +++ b/atlasdb-service/versions.lock @@ -197,6 +197,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -212,6 +213,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -450,6 +452,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -882,6 +890,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -897,6 +906,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1135,6 +1145,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/atlasdb-tests-shared/versions.lock b/atlasdb-tests-shared/versions.lock index a8299c81c67..86b904273ca 100644 --- a/atlasdb-tests-shared/versions.lock +++ b/atlasdb-tests-shared/versions.lock @@ -145,6 +145,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -156,6 +157,7 @@ "com.palantir.common:streams", "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -332,6 +334,12 @@ "com.palantir.atlasdb:atlasdb-impl-shared" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -697,6 +705,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -708,6 +717,7 @@ "com.palantir.common:streams", "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -884,6 +894,12 @@ "com.palantir.atlasdb:atlasdb-impl-shared" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/timelock-agent/versions.lock b/timelock-agent/versions.lock index e927b0331a2..6a878b5483c 100644 --- a/timelock-agent/versions.lock +++ b/timelock-agent/versions.lock @@ -171,6 +171,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -185,6 +186,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -439,6 +441,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -830,6 +838,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -844,6 +853,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1098,6 +1108,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/timelock-impl/versions.lock b/timelock-impl/versions.lock index f5447986753..bab7238794a 100644 --- a/timelock-impl/versions.lock +++ b/timelock-impl/versions.lock @@ -170,6 +170,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -184,6 +185,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -427,6 +429,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -815,6 +823,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] }, @@ -829,6 +838,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1072,6 +1082,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/timelock-server-distribution/versions.lock b/timelock-server-distribution/versions.lock index 18d6a19f808..082f99a13e6 100644 --- a/timelock-server-distribution/versions.lock +++ b/timelock-server-distribution/versions.lock @@ -266,6 +266,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -286,6 +287,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -661,6 +663,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ diff --git a/timelock-server/versions.lock b/timelock-server/versions.lock index 4c84c755c6b..9fe89f281a9 100644 --- a/timelock-server/versions.lock +++ b/timelock-server/versions.lock @@ -227,6 +227,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -243,6 +244,7 @@ "com.palantir.remoting2:error-handling", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -507,6 +509,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ @@ -1461,6 +1469,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", + "com.palantir.remoting3:refresh-utils", "io.dropwizard:dropwizard-util", "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" ] @@ -1481,6 +1490,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:keystores", + "com.palantir.remoting3:refresh-utils", "com.palantir.remoting3:tracing", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1838,6 +1848,12 @@ "com.palantir.atlasdb:atlasdb-config" ] }, + "com.palantir.remoting3:refresh-utils": { + "locked": "3.5.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ From a953328904e4f718778a04182f50db4f23d27276 Mon Sep 17 00:00:00 2001 From: Jeremy Kong Date: Fri, 10 Nov 2017 22:22:52 +0000 Subject: [PATCH 12/67] Live Reloading the TimeLock Block, Part 2: TransactionManagers Plumbing (#2622) * ServiceCreator.applyDynamic() * Propagate config through TMs * Json Serialization fixes * Some refactoring * lock/lock * Fixed checkstyle * CR comments part 1 * Switch to RPIH * add test * [no release notes] forthcoming in part 4 * checkstyle --- atlasdb-cli-distribution/versions.lock | 6 +- atlasdb-cli/versions.lock | 12 ++- .../atlasdb/config/AtlasDbRuntimeConfig.java | 10 +++ .../atlasdb/config/ServerListConfigs.java | 49 ++++++++++++ .../atlasdb/config/TimeLockClientConfig.java | 15 ++-- .../atlasdb/config/TimeLockRuntimeConfig.java | 29 +++++++ .../atlasdb/factory/ServiceCreator.java | 29 ++++--- .../atlasdb/factory/TransactionManagers.java | 50 +++++++----- .../factory/startup/TimeLockMigrator.java | 17 ++-- .../atlasdb/http/AtlasDbHttpClients.java | 43 +++++++++- ...asDbRuntimeConfigDeserializationTest.java} | 13 ++- .../atlasdb/config/ServerListConfigsTest.java | 80 +++++++++++++++++++ .../factory/TransactionManagersTest.java | 7 +- .../factory/startup/TimeLockMigratorTest.java | 18 +++-- .../atlasdb/http/AtlasDbHttpClientsTest.java | 34 ++++++++ .../test/resources/runtime-config-block.yml | 7 ++ atlasdb-config/versions.lock | 16 +++- atlasdb-console-distribution/versions.lock | 6 +- atlasdb-console/versions.lock | 12 ++- atlasdb-dagger/versions.lock | 12 ++- atlasdb-dbkvs-tests/versions.lock | 50 ++++++++++-- atlasdb-dropwizard-bundle/versions.lock | 8 +- atlasdb-ete-tests/versions.lock | 8 +- atlasdb-feign/build.gradle | 2 + .../atlasdb/config/ServerListConfig.java | 0 .../http/AtlasDbFeignTargetFactory.java | 49 ++++++++++++ atlasdb-feign/versions.lock | 52 +++++++++++- atlasdb-jepsen-tests/versions.lock | 12 ++- atlasdb-perf/versions.lock | 12 ++- atlasdb-service-server/versions.lock | 8 +- atlasdb-service/versions.lock | 12 ++- atlasdb-tests-shared/versions.lock | 50 ++++++++++-- timelock-agent/versions.lock | 12 ++- timelock-impl/versions.lock | 12 ++- timelock-server-distribution/versions.lock | 2 + timelock-server/versions.lock | 4 + 36 files changed, 628 insertions(+), 130 deletions(-) create mode 100644 atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfigs.java create mode 100644 atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java rename atlasdb-config/src/test/java/com/palantir/atlasdb/config/{AtlasDbRuntimeConfigSerializationTest.java => AtlasDbRuntimeConfigDeserializationTest.java} (66%) create mode 100644 atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java rename {atlasdb-config => atlasdb-feign}/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java (100%) diff --git a/atlasdb-cli-distribution/versions.lock b/atlasdb-cli-distribution/versions.lock index 79576be86be..744fa29b876 100644 --- a/atlasdb-cli-distribution/versions.lock +++ b/atlasdb-cli-distribution/versions.lock @@ -561,7 +561,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -847,7 +848,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-cli/versions.lock b/atlasdb-cli/versions.lock index 6f06ed437a5..ed2e30591dc 100644 --- a/atlasdb-cli/versions.lock +++ b/atlasdb-cli/versions.lock @@ -485,7 +485,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -748,7 +749,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1374,7 +1376,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1636,7 +1639,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java index 5732795b2c8..792073986e8 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java @@ -16,6 +16,8 @@ package com.palantir.atlasdb.config; +import java.util.Optional; + import org.immutables.value.Value; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -58,6 +60,14 @@ public long getTimestampCacheSize() { return AtlasDbConstants.DEFAULT_TIMESTAMP_CACHE_SIZE; } + /** + * Runtime live-reloadable parameters for communicating with TimeLock. + * + * This value is ignored if the install config does not specify usage of TimeLock. + * We do not currently support live reloading from a leader block or using embedded services to using TimeLock. + */ + public abstract Optional timelockRuntime(); + public static ImmutableAtlasDbRuntimeConfig defaultRuntimeConfig() { return ImmutableAtlasDbRuntimeConfig.builder().build(); } diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfigs.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfigs.java new file mode 100644 index 00000000000..5919504863d --- /dev/null +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfigs.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.config; + +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public final class ServerListConfigs { + private ServerListConfigs() { + // utilities + } + + public static ServerListConfig parseInstallAndRuntimeConfigs(TimeLockClientConfig installClientConfig, + Supplier> runtimeConfig, + String namespace) { + ServerListConfig nonNamespacedConfig = runtimeConfig.get() + .map(TimeLockRuntimeConfig::serversList) + .orElse(installClientConfig.serversList()); + return namespaceUris(nonNamespacedConfig, namespace); + } + + public static ServerListConfig namespaceUris(ServerListConfig config, String namespace) { + Set serversWithNamespaces = config + .servers() + .stream() + .map(serverAddress -> serverAddress.replaceAll("/$", "") + "/" + namespace) + .collect(Collectors.toSet()); + return ImmutableServerListConfig.builder() + .from(config) + .servers(serversWithNamespaces) + .build(); + } +} diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java index 73afab95986..f1934d44d43 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java @@ -16,8 +16,6 @@ package com.palantir.atlasdb.config; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import org.immutables.value.Value; @@ -45,16 +43,15 @@ public String getClientOrThrow() { "Tried to read a client from a TimeLockClientConfig, but it hadn't been initialised.")); } + /** + * @deprecated Please use {@link TimeLockRuntimeConfig} to specify the {@link ServerListConfig} to be used + * for connecting to TimeLock. + */ + @Deprecated public abstract ServerListConfig serversList(); public ServerListConfig toNamespacedServerList() { - Set serversWithNamespaces = serversList() - .servers() - .stream() - .map(serverAddress -> serverAddress.replaceAll("/$", "") + "/" + getClientOrThrow()) - .collect(Collectors.toSet()); - return ImmutableServerListConfig.copyOf(serversList()) - .withServers(serversWithNamespaces); + return ServerListConfigs.namespaceUris(serversList(), getClientOrThrow()); } @Value.Check diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java new file mode 100644 index 00000000000..6335b5c3f93 --- /dev/null +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.config; + +import org.immutables.value.Value; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(as = ImmutableTimeLockRuntimeConfig.class) +@JsonDeserialize(as = ImmutableTimeLockRuntimeConfig.class) +@Value.Immutable +public abstract class TimeLockRuntimeConfig { + public abstract ServerListConfig serversList(); +} diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceCreator.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceCreator.java index 898f73db5e1..f50a912b881 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceCreator.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceCreator.java @@ -23,7 +23,7 @@ import java.net.URI; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.function.Supplier; import javax.net.ssl.SSLSocketFactory; @@ -49,9 +49,18 @@ public ServiceCreator(Class serviceClass, String userAgent) { @Override public T apply(ServerListConfig input) { - Optional sslSocketFactory = createSslSocketFactory(input.sslConfiguration()); - Optional proxySelector = input.proxyConfiguration().map(ServiceCreator::createProxySelector); - return createService(sslSocketFactory, proxySelector, input.servers(), serviceClass, userAgent); + return applyDynamic(() -> input); + } + + // Semi-horrible, but given that we create ServiceCreators explicitly and I'd rather not API break our + // implementation of Function, leaving this here for now. + public T applyDynamic(Supplier input) { + return createService( + input, + SslSocketFactories::createSslSocketFactory, + ServiceCreator::createProxySelector, + serviceClass, + userAgent); } /** @@ -62,13 +71,13 @@ public static Optional createSslSocketFactory(Optional T createService( - Optional sslSocketFactory, - Optional proxySelector, - Set uris, - Class serviceClass, + Supplier serverListConfigSupplier, + java.util.function.Function sslSocketFactoryCreator, + java.util.function.Function proxySelectorCreator, + Class type, String userAgent) { - return AtlasDbHttpClients.createProxyWithFailover( - sslSocketFactory, proxySelector, uris, serviceClass, userAgent); + return AtlasDbHttpClients.createLiveReloadingProxyWithFailover( + serverListConfigSupplier, sslSocketFactoryCreator, proxySelectorCreator, type, userAgent); } public static T createInstrumentedService(T service, Class serviceClass) { diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java index c51db40aed2..547fa5cf5b1 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java @@ -45,12 +45,13 @@ import com.palantir.atlasdb.config.AtlasDbConfig; import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; +import com.palantir.atlasdb.config.ImmutableAtlasDbRuntimeConfig; import com.palantir.atlasdb.config.ImmutableServerListConfig; import com.palantir.atlasdb.config.LeaderConfig; import com.palantir.atlasdb.config.ServerListConfig; +import com.palantir.atlasdb.config.ServerListConfigs; import com.palantir.atlasdb.config.SweepConfig; import com.palantir.atlasdb.config.TimeLockClientConfig; -import com.palantir.atlasdb.config.TimeLockClientConfigs; import com.palantir.atlasdb.config.TimestampClientConfig; import com.palantir.atlasdb.factory.Leaders.LocalPaxosServices; import com.palantir.atlasdb.factory.startup.TimeLockMigrator; @@ -313,7 +314,7 @@ SerializableTransactionManager serializable() { SimpleTimeDuration.of(config.getDefaultLockTimeoutSeconds(), TimeUnit.SECONDS)); LockAndTimestampServices lockAndTimestampServices = createLockAndTimestampServices( config, - () -> runtimeConfigSupplier.get().timestampClient(), + runtimeConfigSupplier, registrar(), () -> LockServiceImpl.create(lockServerOptions()), atlasFactory::getTimestampService, @@ -521,6 +522,7 @@ public static LockAndTimestampServices createLockAndTimestampServices( com.google.common.base.Supplier time) { LockAndTimestampServices lockAndTimestampServices = createRawInstrumentedServices(config, + () -> ImmutableAtlasDbRuntimeConfig.builder().build(), env, lock, time, @@ -535,16 +537,16 @@ public static LockAndTimestampServices createLockAndTimestampServices( @VisibleForTesting static LockAndTimestampServices createLockAndTimestampServices( AtlasDbConfig config, - java.util.function.Supplier runtimeConfigSupplier, + java.util.function.Supplier runtimeConfigSupplier, Consumer env, com.google.common.base.Supplier lock, com.google.common.base.Supplier time, TimestampStoreInvalidator invalidator, String userAgent) { LockAndTimestampServices lockAndTimestampServices = - createRawInstrumentedServices(config, env, lock, time, invalidator, userAgent); + createRawInstrumentedServices(config, runtimeConfigSupplier, env, lock, time, invalidator, userAgent); return withRequestBatchingTimestampService( - runtimeConfigSupplier, + () -> runtimeConfigSupplier.get().timestampClient(), withRefreshingLockService(lockAndTimestampServices)); } @@ -575,6 +577,7 @@ private static LockAndTimestampServices withRequestBatchingTimestampService( @VisibleForTesting static LockAndTimestampServices createRawInstrumentedServices( AtlasDbConfig config, + Supplier runtimeConfigSupplier, Consumer env, com.google.common.base.Supplier lock, com.google.common.base.Supplier time, @@ -585,7 +588,7 @@ static LockAndTimestampServices createRawInstrumentedServices( } else if (config.timestamp().isPresent() && config.lock().isPresent()) { return createRawRemoteServices(config, userAgent); } else if (config.timelock().isPresent()) { - return createRawServicesFromTimeLock(config, invalidator, userAgent); + return createRawServicesFromTimeLock(config, runtimeConfigSupplier, invalidator, userAgent); } else { return createRawEmbeddedServices(env, lock, time); } @@ -593,36 +596,39 @@ static LockAndTimestampServices createRawInstrumentedServices( private static LockAndTimestampServices createRawServicesFromTimeLock( AtlasDbConfig config, + Supplier runtimeConfigSupplier, TimestampStoreInvalidator invalidator, String userAgent) { - Preconditions.checkState(config.timelock().isPresent(), - "Cannot create raw services from timelock without a timelock block!"); - TimeLockClientConfig clientConfig = config.timelock().get(); - String resolvedClient = OptionalResolver.resolve(clientConfig.client(), config.namespace()); - TimeLockClientConfig timeLockClientConfig = - TimeLockClientConfigs.copyWithClient(config.timelock().get(), resolvedClient); + Supplier serverListConfigSupplier = + getServerListConfigSupplierForTimeLock(config, runtimeConfigSupplier); TimeLockMigrator migrator = - TimeLockMigrator.create(timeLockClientConfig, invalidator, userAgent, config.initializeAsync()); + TimeLockMigrator.create(serverListConfigSupplier, invalidator, userAgent, config.initializeAsync()); migrator.migrate(); // This can proceed async if config.initializeAsync() was set return ImmutableLockAndTimestampServices.copyOf( - createNamespacedRawRemoteServices(timeLockClientConfig, userAgent)) + getLockAndTimestampServices(serverListConfigSupplier, userAgent)) .withMigrator(migrator); } - private static LockAndTimestampServices createNamespacedRawRemoteServices( - TimeLockClientConfig config, - String userAgent) { - ServerListConfig namespacedServerListConfig = config.toNamespacedServerList(); - return getLockAndTimestampServices(namespacedServerListConfig, userAgent); + private static Supplier getServerListConfigSupplierForTimeLock( + AtlasDbConfig config, + Supplier runtimeConfigSupplier) { + Preconditions.checkState(config.timelock().isPresent(), + "Cannot create raw services from timelock without a timelock block!"); + TimeLockClientConfig clientConfig = config.timelock().get(); + String resolvedClient = OptionalResolver.resolve(clientConfig.client(), config.namespace()); + return () -> ServerListConfigs.parseInstallAndRuntimeConfigs( + clientConfig, + () -> runtimeConfigSupplier.get().timelockRuntime(), + resolvedClient); } private static LockAndTimestampServices getLockAndTimestampServices( - ServerListConfig timelockServerListConfig, + Supplier timelockServerListConfig, String userAgent) { LockService lockService = new ServiceCreator<>(LockService.class, userAgent) - .apply(timelockServerListConfig); + .applyDynamic(timelockServerListConfig); TimelockService timelockService = new ServiceCreator<>(TimelockService.class, userAgent) - .apply(timelockServerListConfig); + .applyDynamic(timelockServerListConfig); return ImmutableLockAndTimestampServices.builder() .lock(lockService) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/startup/TimeLockMigrator.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/startup/TimeLockMigrator.java index 96fd1dc9203..1a97f413843 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/startup/TimeLockMigrator.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/startup/TimeLockMigrator.java @@ -15,10 +15,11 @@ */ package com.palantir.atlasdb.factory.startup; +import java.util.function.Supplier; + import com.palantir.async.initializer.AsyncInitializer; import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.config.ServerListConfig; -import com.palantir.atlasdb.config.TimeLockClientConfig; import com.palantir.atlasdb.factory.ServiceCreator; import com.palantir.common.annotation.Idempotent; import com.palantir.timestamp.TimestampManagementService; @@ -40,19 +41,20 @@ private TimeLockMigrator( } public static TimeLockMigrator create( - TimeLockClientConfig config, + ServerListConfig serverListConfig, TimestampStoreInvalidator invalidator, String userAgent) { - return create(config, invalidator, userAgent, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC); + return create(() -> serverListConfig, invalidator, userAgent, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC); } public static TimeLockMigrator create( - TimeLockClientConfig config, + Supplier serverListConfigSupplier, TimestampStoreInvalidator invalidator, String userAgent, boolean initializeAsync) { TimestampManagementService remoteTimestampManagementService = - createRemoteManagementService(config, userAgent); + createRemoteManagementService( + serverListConfigSupplier, userAgent); return new TimeLockMigrator(invalidator, remoteTimestampManagementService, initializeAsync); } @@ -75,11 +77,10 @@ public void migrate() { } private static TimestampManagementService createRemoteManagementService( - TimeLockClientConfig timelockConfig, + Supplier serverListConfig, String userAgent) { - ServerListConfig serverListConfig = timelockConfig.toNamespacedServerList(); return new ServiceCreator<>(TimestampManagementService.class, userAgent) - .apply(serverListConfig); + .applyDynamic(serverListConfig); } @Override diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/http/AtlasDbHttpClients.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/http/AtlasDbHttpClients.java index 1305f155062..ed33543bccb 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/http/AtlasDbHttpClients.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/http/AtlasDbHttpClients.java @@ -19,17 +19,22 @@ import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; import javax.net.ssl.SSLSocketFactory; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import com.palantir.atlasdb.config.ServerListConfig; import com.palantir.atlasdb.util.AtlasDbMetrics; +import com.palantir.remoting.api.config.service.ProxyConfiguration; +import com.palantir.remoting.api.config.ssl.SslConfiguration; public final class AtlasDbHttpClients { - private static final int QUICK_FEIGN_TIMEOUT_MILLIS = 1000; - private static final int QUICK_MAX_BACKOFF_MILLIS = 1000; + private static final int QUICK_FEIGN_TIMEOUT_MILLIS = 100; + private static final int QUICK_MAX_BACKOFF_MILLIS = 100; private AtlasDbHttpClients() { // Utility class @@ -143,6 +148,40 @@ public static T createProxyWithFailover( MetricRegistry.name(type)); } + public static T createLiveReloadingProxyWithFailover( + Supplier serverListConfigSupplier, + Function sslSocketFactoryCreator, + Function proxySelectorCreator, + Class type, + String userAgent) { + return AtlasDbMetrics.instrument( + type, + AtlasDbFeignTargetFactory.createLiveReloadingProxyWithFailover( + serverListConfigSupplier, sslSocketFactoryCreator, proxySelectorCreator, type, userAgent), + MetricRegistry.name(type)); + } + + @VisibleForTesting + static T createLiveReloadingProxyWithQuickFailoverForTesting( + Supplier serverListConfigSupplier, + Function sslSocketFactoryCreator, + Function proxySelectorCreator, + Class type, + String userAgent) { + return AtlasDbMetrics.instrument( + type, + AtlasDbFeignTargetFactory.createLiveReloadingProxyWithFailover( + serverListConfigSupplier, + sslSocketFactoryCreator, + proxySelectorCreator, + QUICK_FEIGN_TIMEOUT_MILLIS, + QUICK_FEIGN_TIMEOUT_MILLIS, + QUICK_MAX_BACKOFF_MILLIS, + type, + userAgent), + MetricRegistry.name(type)); + } + @VisibleForTesting static T createProxyWithQuickFailoverForTesting( Optional sslSocketFactory, diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigSerializationTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java similarity index 66% rename from atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigSerializationTest.java rename to atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java index 693a3f597fa..fc557ad8f83 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigSerializationTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java @@ -23,14 +23,21 @@ import org.junit.Test; -public class AtlasDbRuntimeConfigSerializationTest { +public class AtlasDbRuntimeConfigDeserializationTest { private static final File TEST_RUNTIME_CONFIG_FILE = new File( - AtlasDbConfigDeserializationTest.class.getResource("/runtime-config-block.yml").getPath()); + AtlasDbRuntimeConfigDeserializationTest.class.getResource("/runtime-config-block.yml").getPath()); @Test - public void canDeserializeAtlasDbConfig() throws IOException { + public void canDeserializeRuntimeConfig() throws IOException { AtlasDbRuntimeConfig runtimeConfig = AtlasDbConfigs.OBJECT_MAPPER.readValue(TEST_RUNTIME_CONFIG_FILE, AtlasDbRuntimeConfig.class); assertThat(runtimeConfig.timestampClient().enableTimestampBatching()).isTrue(); + + assertThat(runtimeConfig.timelockRuntime()).isPresent(); + assertThat(runtimeConfig.timelockRuntime().get().serversList().servers()) + .containsExactlyInAnyOrder( + "https://foo1:12345", + "https://foo2:8421", + "https://foo3:9421"); } } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java new file mode 100644 index 00000000000..1f5b16c11a3 --- /dev/null +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Optional; + +import org.junit.Test; + +public class ServerListConfigsTest { + private static final String CLIENT = "client"; + + private static final ImmutableServerListConfig SERVERS_LIST_1 = ImmutableServerListConfig.builder() + .addServers("one") + .build(); + private static final ImmutableServerListConfig SERVERS_LIST_2 = ImmutableServerListConfig.builder() + .addServers("one", "two") + .build(); + private static final ImmutableServerListConfig SERVERS_LIST_3 = ImmutableServerListConfig.builder() + .addServers("three/") + .build(); + + private static final TimeLockClientConfig INSTALL_CONFIG = ImmutableTimeLockClientConfig.builder() + .serversList(SERVERS_LIST_1) + .build(); + private static final TimeLockRuntimeConfig RUNTIME_CONFIG = ImmutableTimeLockRuntimeConfig.builder() + .serversList(SERVERS_LIST_2) + .build(); + + @Test + public void namespacingAddsClientNameCorrectly() { + ServerListConfig namespacedServersList = ServerListConfigs.namespaceUris(SERVERS_LIST_1, CLIENT); + assertThat(namespacedServersList.servers()).containsExactly("one/client"); + } + + @Test + public void namespacingAddsClientNameToAllServers() { + ServerListConfig namespacedServersList = ServerListConfigs.namespaceUris(SERVERS_LIST_2, CLIENT); + assertThat(namespacedServersList.servers()).containsExactlyInAnyOrder("one/client", "two/client"); + } + + @Test + public void namespacingCanDealWithTrailingSlash() { + ServerListConfig namespacedServersList = ServerListConfigs.namespaceUris(SERVERS_LIST_3, CLIENT); + assertThat(namespacedServersList.servers()).containsExactlyInAnyOrder("three/client"); + } + + @Test + public void prioritisesRuntimeConfigIfAvailable() { + ServerListConfig resolvedConfig = ServerListConfigs.parseInstallAndRuntimeConfigs( + INSTALL_CONFIG, + () -> Optional.of(RUNTIME_CONFIG), + CLIENT); + assertThat(resolvedConfig.servers()).containsExactlyInAnyOrder("one/client", "two/client"); + } + + @Test + public void fallsBackToInstallConfigIfRuntimeConfigNotAvailable() { + ServerListConfig resolvedConfig = ServerListConfigs.parseInstallAndRuntimeConfigs( + INSTALL_CONFIG, + Optional::empty, + CLIENT); + assertThat(resolvedConfig.servers()).containsExactlyInAnyOrder("one/client"); + } +} diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java index 5bb5a1dca5f..d01c39fa6f2 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java @@ -193,6 +193,7 @@ public void setup() throws JsonProcessingException { runtimeConfig = mock(AtlasDbRuntimeConfig.class); when(runtimeConfig.timestampClient()).thenReturn(ImmutableTimestampClientConfig.of(false)); + when(runtimeConfig.timelockRuntime()).thenReturn(Optional.empty()); environment = mock(Consumer.class); @@ -487,7 +488,7 @@ private void setupLeaderBlockInConfig() throws IOException { private TransactionManagers.LockAndTimestampServices getLockAndTimestampServices() { return TransactionManagers.createLockAndTimestampServices( config, - () -> runtimeConfig.timestampClient(), + () -> runtimeConfig, environment, LockServiceImpl::create, InMemoryTimestampService::new, @@ -512,7 +513,7 @@ private void verifyUserAgentOnTimestampAndLockRequests(String timestampPath, Str TransactionManagers.LockAndTimestampServices lockAndTimestamp = TransactionManagers.createLockAndTimestampServices( config, - () -> ImmutableTimestampClientConfig.of(false), + () -> runtimeConfig, environment, LockServiceImpl::create, InMemoryTimestampService::new, @@ -544,7 +545,7 @@ private TransactionManagers.LockAndTimestampServices createLockAndTimestampServi AtlasDbConfig atlasDbConfig, AtlasDbRuntimeConfig atlasDbRuntimeConfig) { return TransactionManagers.createLockAndTimestampServices( atlasDbConfig, - atlasDbRuntimeConfig::timestampClient, + () -> atlasDbRuntimeConfig, environment, LockServiceImpl::create, InMemoryTimestampService::new, diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java index 4cce918b8ae..ca387aeccdf 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java @@ -61,7 +61,6 @@ public class TimeLockMigratorTest { private static final String USER_AGENT = "user-agent (123456789)"; - private ServerListConfig defaultServerListConfig; private TimeLockClientConfig timelockConfig; private final TimestampStoreInvalidator invalidator = mock(TimestampStoreInvalidator.class); @@ -83,7 +82,7 @@ public void setUp() { String serverUri = String.format("http://%s:%s", WireMockConfiguration.DEFAULT_BIND_ADDRESS, wireMockRule.port()); - defaultServerListConfig = ImmutableServerListConfig.builder().addServers(serverUri).build(); + ServerListConfig defaultServerListConfig = ImmutableServerListConfig.builder().addServers(serverUri).build(); timelockConfig = ImmutableTimeLockClientConfig.builder() .client("testClient") .serversList(defaultServerListConfig) @@ -94,7 +93,8 @@ public void setUp() { public void propagatesBackupTimestampToFastForwardOnRemoteService() { wireMockRule.stubFor(TEST_MAPPING.willReturn(aResponse().withStatus(204))); - TimeLockMigrator migrator = TimeLockMigrator.create(timelockConfig, invalidator, USER_AGENT); + TimeLockMigrator migrator = + TimeLockMigrator.create(timelockConfig.toNamespacedServerList(), invalidator, USER_AGENT); migrator.migrate(); wireMockRule.verify(getRequestedFor(urlEqualTo(PING_ENDPOINT))); @@ -106,7 +106,8 @@ public void propagatesBackupTimestampToFastForwardOnRemoteService() { public void invalidationDoesNotProceedIfTimelockPingUnsuccessful() { wireMockRule.stubFor(PING_MAPPING.willReturn(aResponse().withStatus(500))); - TimeLockMigrator migrator = TimeLockMigrator.create(timelockConfig, invalidator, USER_AGENT); + TimeLockMigrator migrator = + TimeLockMigrator.create(timelockConfig.toNamespacedServerList(), invalidator, USER_AGENT); assertThatThrownBy(migrator::migrate).isInstanceOf(IllegalStateException.class); verify(invalidator, never()).backupAndInvalidate(); } @@ -115,7 +116,8 @@ public void invalidationDoesNotProceedIfTimelockPingUnsuccessful() { public void migrationDoesNotProceedIfInvalidationFails() { when(invalidator.backupAndInvalidate()).thenThrow(new IllegalStateException()); - TimeLockMigrator migrator = TimeLockMigrator.create(timelockConfig, invalidator, USER_AGENT); + TimeLockMigrator migrator = + TimeLockMigrator.create(timelockConfig.toNamespacedServerList(), invalidator, USER_AGENT); assertThatThrownBy(migrator::migrate).isInstanceOf(IllegalStateException.class); wireMockRule.verify(0, postRequestedFor(urlEqualTo(TEST_ENDPOINT))); } @@ -135,7 +137,8 @@ public void asyncMigrationProceedsIfTimeLockInitiallyUnavailable() throws Interr wireMockRule.stubFor(TEST_MAPPING.willReturn(aResponse().withStatus(204))); - TimeLockMigrator migrator = TimeLockMigrator.create(timelockConfig, invalidator, USER_AGENT, true); + TimeLockMigrator migrator = + TimeLockMigrator.create(() -> timelockConfig.toNamespacedServerList(), invalidator, USER_AGENT, true); migrator.migrate(); Awaitility.await() @@ -163,7 +166,8 @@ public Long answer(InvocationOnMock invocation) throws Throwable { }); wireMockRule.stubFor(TEST_MAPPING.willReturn(aResponse().withStatus(204))); - TimeLockMigrator migrator = TimeLockMigrator.create(timelockConfig, invalidator, USER_AGENT, true); + TimeLockMigrator migrator = + TimeLockMigrator.create(() -> timelockConfig.toNamespacedServerList(), invalidator, USER_AGENT, true); migrator.migrate(); Awaitility.await() diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java index 8259a958517..77deb189615 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java @@ -15,6 +15,7 @@ */ package com.palantir.atlasdb.http; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; @@ -25,8 +26,10 @@ import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import java.net.ProxySelector; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLSocketFactory; import javax.ws.rs.GET; @@ -43,8 +46,12 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Uninterruptibles; +import com.palantir.atlasdb.config.ImmutableServerListConfig; import com.palantir.atlasdb.factory.ServiceCreator; import com.palantir.remoting.api.config.service.ProxyConfiguration; +import com.palantir.remoting3.config.ssl.SslSocketFactories; public class AtlasDbHttpClientsTest { private static final Optional NO_SSL = Optional.empty(); @@ -138,6 +145,33 @@ public void httpProxyIsConfigurableOnClientRequests() { availableServer.verify(0, getRequestedFor(urlMatching(TEST_ENDPOINT))); } + @Test + public void canLiveReloadServersList() { + unavailableServer.stubFor(ENDPOINT_MAPPING.willReturn(aResponse().withStatus(503))); + + List servers = Lists.newArrayList(getUriForPort(unavailablePort)); + + TestResource client = AtlasDbHttpClients.createLiveReloadingProxyWithQuickFailoverForTesting( + () -> ImmutableServerListConfig.builder() + .servers(servers) + .build(), + SslSocketFactories::createSslSocketFactory, + unused -> ProxySelector.getDefault(), + TestResource.class, + "user (123)"); + + // actually a Feign RetryableException but that's not on our classpath + assertThatThrownBy(client::getTestNumber).isInstanceOf(RuntimeException.class); + + servers.add(getUriForPort(availablePort)); + Uninterruptibles.sleepUninterruptibly( + PollingRefreshable.DEFAULT_REFRESH_INTERVAL.getSeconds() + 1, TimeUnit.SECONDS); + + int response = client.getTestNumber(); + assertThat(response, equalTo(TEST_NUMBER)); + unavailableServer.verify(getRequestedFor(urlMatching(TEST_ENDPOINT))); + } + private static String getUriForPort(int port) { return String.format("http://%s:%s", WireMockConfiguration.DEFAULT_BIND_ADDRESS, port); } diff --git a/atlasdb-config/src/test/resources/runtime-config-block.yml b/atlasdb-config/src/test/resources/runtime-config-block.yml index 855da31869d..138f646b146 100644 --- a/atlasdb-config/src/test/resources/runtime-config-block.yml +++ b/atlasdb-config/src/test/resources/runtime-config-block.yml @@ -1,2 +1,9 @@ timestampClient: enableTimestampBatching: true + +timelockRuntime: + serversList: + servers: + - "https://foo1:12345" + - "https://foo2:8421" + - "https://foo3:9421" \ No newline at end of file diff --git a/atlasdb-config/versions.lock b/atlasdb-config/versions.lock index b1c67a0048b..53ac2e59f43 100644 --- a/atlasdb-config/versions.lock +++ b/atlasdb-config/versions.lock @@ -372,7 +372,10 @@ ] }, "com.palantir.remoting-api:service-config": { - "locked": "1.4.0" + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", @@ -560,7 +563,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1016,7 +1020,10 @@ ] }, "com.palantir.remoting-api:service-config": { - "locked": "1.4.0" + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", @@ -1204,7 +1211,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-console-distribution/versions.lock b/atlasdb-console-distribution/versions.lock index 750755ac34a..6f36e4794a1 100644 --- a/atlasdb-console-distribution/versions.lock +++ b/atlasdb-console-distribution/versions.lock @@ -543,7 +543,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -827,7 +828,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-console/versions.lock b/atlasdb-console/versions.lock index 4a79af18d8f..1a95fe81a62 100644 --- a/atlasdb-console/versions.lock +++ b/atlasdb-console/versions.lock @@ -407,7 +407,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -617,7 +618,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1125,7 +1127,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1335,7 +1338,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-dagger/versions.lock b/atlasdb-dagger/versions.lock index 06ee9a23d6c..b086407177d 100644 --- a/atlasdb-dagger/versions.lock +++ b/atlasdb-dagger/versions.lock @@ -410,7 +410,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -618,7 +619,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1118,7 +1120,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1326,7 +1329,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-dbkvs-tests/versions.lock b/atlasdb-dbkvs-tests/versions.lock index f1b5137710c..e93fc7897a8 100644 --- a/atlasdb-dbkvs-tests/versions.lock +++ b/atlasdb-dbkvs-tests/versions.lock @@ -54,7 +54,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { @@ -80,7 +81,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -376,10 +378,17 @@ "com.palantir.remoting3:error-handling" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.remoting-api:service-config" ] }, "com.palantir.remoting-api:tracing": { @@ -456,6 +465,12 @@ "com.palantir.tritium:tritium-tracing" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.tritium:tritium-api": { "locked": "0.8.3", "transitive": [ @@ -560,7 +575,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -671,6 +687,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -748,7 +765,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { @@ -774,7 +792,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -1070,10 +1089,17 @@ "com.palantir.remoting3:error-handling" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.remoting-api:service-config" ] }, "com.palantir.remoting-api:tracing": { @@ -1150,6 +1176,12 @@ "com.palantir.tritium:tritium-tracing" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.tritium:tritium-api": { "locked": "0.8.3", "transitive": [ @@ -1254,7 +1286,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1365,6 +1398,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", diff --git a/atlasdb-dropwizard-bundle/versions.lock b/atlasdb-dropwizard-bundle/versions.lock index 54750bea084..c49ac9257ff 100644 --- a/atlasdb-dropwizard-bundle/versions.lock +++ b/atlasdb-dropwizard-bundle/versions.lock @@ -516,7 +516,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -926,6 +927,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" @@ -1853,7 +1855,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -2263,6 +2266,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" diff --git a/atlasdb-ete-tests/versions.lock b/atlasdb-ete-tests/versions.lock index c3d088c5922..e84fdf95984 100644 --- a/atlasdb-ete-tests/versions.lock +++ b/atlasdb-ete-tests/versions.lock @@ -541,7 +541,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -978,6 +979,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" @@ -1987,7 +1989,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -2440,6 +2443,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" diff --git a/atlasdb-feign/build.gradle b/atlasdb-feign/build.gradle index 53d7afbf1d4..ed21fda2ff6 100644 --- a/atlasdb-feign/build.gradle +++ b/atlasdb-feign/build.gradle @@ -27,7 +27,9 @@ configurations { dependencies { explicitShadow project(':lock-api') explicitShadow group: 'com.squareup.okhttp3', name: 'okhttp' + explicitShadow group: 'com.palantir.remoting-api', name: 'service-config' explicitShadow group: 'com.palantir.remoting3', name: 'refresh-utils' + explicitShadow group: 'javax.validation', name: 'validation-api' toBeShaded group: 'com.palantir.remoting2', name: 'error-handling' toBeShaded group: 'com.palantir.remoting2', name: 'jackson-support' diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java b/atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java similarity index 100% rename from atlasdb-config/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java rename to atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java diff --git a/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java index c821f907e7e..f808b2c8f14 100644 --- a/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java +++ b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java @@ -19,11 +19,18 @@ import java.net.ProxySelector; import java.util.Collection; import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; import javax.net.ssl.SSLSocketFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.google.common.reflect.Reflection; +import com.palantir.atlasdb.config.ServerListConfig; +import com.palantir.remoting.api.config.service.ProxyConfiguration; +import com.palantir.remoting.api.config.ssl.SslConfiguration; +import com.palantir.remoting3.ext.refresh.RefreshableProxyInvocationHandler; import feign.Client; import feign.Contract; @@ -140,4 +147,46 @@ private static T createProxyWithFailover( .target(failoverFeignTarget); } + public static T createLiveReloadingProxyWithFailover( + Supplier serverListConfigSupplier, + Function sslSocketFactoryCreator, + Function proxySelectorCreator, + Class type, + String userAgent) { + return createLiveReloadingProxyWithFailover( + serverListConfigSupplier, + sslSocketFactoryCreator, + proxySelectorCreator, + DEFAULT_FEIGN_OPTIONS.connectTimeoutMillis(), + DEFAULT_FEIGN_OPTIONS.readTimeoutMillis(), + FailoverFeignTarget.DEFAULT_MAX_BACKOFF_MILLIS, + type, + userAgent); + } + + public static T createLiveReloadingProxyWithFailover( + Supplier serverListConfigSupplier, + Function sslSocketFactoryCreator, + Function proxySelectorCreator, + int feignConnectTimeout, + int feignReadTimeout, + int maxBackoffMillis, + Class type, + String userAgent) { + PollingRefreshable configPollingRefreshable = + PollingRefreshable.create(serverListConfigSupplier); + return Reflection.newProxy( + type, + RefreshableProxyInvocationHandler.create( + configPollingRefreshable.getRefreshable(), + serverListConfig -> createProxyWithFailover( + serverListConfig.sslConfiguration().map(sslSocketFactoryCreator), + serverListConfig.proxyConfiguration().map(proxySelectorCreator), + serverListConfig.servers(), + feignConnectTimeout, + feignReadTimeout, + maxBackoffMillis, + type, + userAgent))); + } } diff --git a/atlasdb-feign/versions.lock b/atlasdb-feign/versions.lock index b9f5b599154..5fbf802ac48 100644 --- a/atlasdb-feign/versions.lock +++ b/atlasdb-feign/versions.lock @@ -28,8 +28,10 @@ "com.fasterxml.jackson.module:jackson-module-afterburner", "com.netflix.feign:feign-jackson", "com.palantir.atlasdb:lock-api", + "com.palantir.remoting-api:ssl-config", "com.palantir.remoting2:error-handling", - "com.palantir.remoting2:jackson-support" + "com.palantir.remoting2:jackson-support", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-guava": { @@ -43,7 +45,8 @@ "locked": "2.6.7", "transitive": [ "com.palantir.remoting2:error-handling", - "com.palantir.remoting2:jackson-support" + "com.palantir.remoting2:jackson-support", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -123,6 +126,15 @@ "com.palantir.atlasdb:lock-api" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0" + }, + "com.palantir.remoting-api:ssl-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.remoting2:error-handling": { "locked": "2.3.0" }, @@ -143,6 +155,12 @@ "com.palantir.atlasdb:timestamp-api" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.squareup.okhttp3:okhttp": { "locked": "3.8.1", "transitive": [ @@ -161,6 +179,9 @@ "com.palantir.atlasdb:atlasdb-commons" ] }, + "javax.validation:validation-api": { + "locked": "1.1.0.Final" + }, "javax.ws.rs:javax.ws.rs-api": { "locked": "2.0.1", "transitive": [ @@ -187,6 +208,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting2:error-handling", + "com.palantir.tokens:auth-tokens", "io.dropwizard.metrics:metrics-core" ] } @@ -220,8 +242,10 @@ "com.fasterxml.jackson.module:jackson-module-afterburner", "com.netflix.feign:feign-jackson", "com.palantir.atlasdb:lock-api", + "com.palantir.remoting-api:ssl-config", "com.palantir.remoting2:error-handling", - "com.palantir.remoting2:jackson-support" + "com.palantir.remoting2:jackson-support", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-guava": { @@ -235,7 +259,8 @@ "locked": "2.6.7", "transitive": [ "com.palantir.remoting2:error-handling", - "com.palantir.remoting2:jackson-support" + "com.palantir.remoting2:jackson-support", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -315,6 +340,15 @@ "com.palantir.atlasdb:lock-api" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0" + }, + "com.palantir.remoting-api:ssl-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.remoting2:error-handling": { "locked": "2.3.0" }, @@ -335,6 +369,12 @@ "com.palantir.atlasdb:timestamp-api" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.squareup.okhttp3:okhttp": { "locked": "3.8.1", "transitive": [ @@ -353,6 +393,9 @@ "com.palantir.atlasdb:atlasdb-commons" ] }, + "javax.validation:validation-api": { + "locked": "1.1.0.Final" + }, "javax.ws.rs:javax.ws.rs-api": { "locked": "2.0.1", "transitive": [ @@ -379,6 +422,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting2:error-handling", + "com.palantir.tokens:auth-tokens", "io.dropwizard.metrics:metrics-core" ] } diff --git a/atlasdb-jepsen-tests/versions.lock b/atlasdb-jepsen-tests/versions.lock index 69aaf58fd80..67b3eed33da 100644 --- a/atlasdb-jepsen-tests/versions.lock +++ b/atlasdb-jepsen-tests/versions.lock @@ -398,7 +398,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -599,7 +600,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1093,7 +1095,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1294,7 +1297,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-perf/versions.lock b/atlasdb-perf/versions.lock index 8c2aeec33d3..6b7e6cf667a 100644 --- a/atlasdb-perf/versions.lock +++ b/atlasdb-perf/versions.lock @@ -569,7 +569,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -854,7 +855,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1640,7 +1642,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1925,7 +1928,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-service-server/versions.lock b/atlasdb-service-server/versions.lock index 2b533f8d25d..9470ffa273c 100644 --- a/atlasdb-service-server/versions.lock +++ b/atlasdb-service-server/versions.lock @@ -426,7 +426,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -782,6 +783,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" @@ -1626,7 +1628,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -2032,6 +2035,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" diff --git a/atlasdb-service/versions.lock b/atlasdb-service/versions.lock index 4c16cc9c464..eaa4b96a425 100644 --- a/atlasdb-service/versions.lock +++ b/atlasdb-service/versions.lock @@ -398,7 +398,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -603,7 +604,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1091,7 +1093,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1296,7 +1299,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/atlasdb-tests-shared/versions.lock b/atlasdb-tests-shared/versions.lock index 86b904273ca..668499a09e3 100644 --- a/atlasdb-tests-shared/versions.lock +++ b/atlasdb-tests-shared/versions.lock @@ -53,7 +53,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { @@ -79,7 +80,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -294,10 +296,17 @@ "com.palantir.remoting3:error-handling" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.remoting-api:service-config" ] }, "com.palantir.remoting-api:tracing": { @@ -369,6 +378,12 @@ "com.palantir.tritium:tritium-tracing" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.tritium:tritium-api": { "locked": "0.8.3", "transitive": [ @@ -453,7 +468,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -538,6 +554,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -613,7 +630,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { @@ -639,7 +657,8 @@ "com.palantir.remoting2:jackson-support", "com.palantir.remoting2:tracing", "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens" ] }, "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { @@ -854,10 +873,17 @@ "com.palantir.remoting3:error-handling" ] }, + "com.palantir.remoting-api:service-config": { + "locked": "1.4.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-feign" + ] + }, "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.remoting-api:service-config" ] }, "com.palantir.remoting-api:tracing": { @@ -929,6 +955,12 @@ "com.palantir.tritium:tritium-tracing" ] }, + "com.palantir.tokens:auth-tokens": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.remoting-api:service-config" + ] + }, "com.palantir.tritium:tritium-api": { "locked": "0.8.3", "transitive": [ @@ -1013,7 +1045,8 @@ "javax.validation:validation-api": { "locked": "1.1.0.Final", "transitive": [ - "com.palantir.atlasdb:atlasdb-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1098,6 +1131,7 @@ "com.palantir.remoting2:tracing", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:tracing", + "com.palantir.tokens:auth-tokens", "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", diff --git a/timelock-agent/versions.lock b/timelock-agent/versions.lock index 6a878b5483c..27c3cb91876 100644 --- a/timelock-agent/versions.lock +++ b/timelock-agent/versions.lock @@ -387,7 +387,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -579,7 +580,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1054,7 +1056,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1246,7 +1249,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/timelock-impl/versions.lock b/timelock-impl/versions.lock index bab7238794a..a01408833e4 100644 --- a/timelock-impl/versions.lock +++ b/timelock-impl/versions.lock @@ -375,7 +375,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -565,7 +566,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -1028,7 +1030,8 @@ "com.palantir.remoting-api:service-config": { "locked": "1.4.0", "transitive": [ - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "com.palantir.remoting-api:ssl-config": { @@ -1218,7 +1221,8 @@ "locked": "1.1.0.Final", "transitive": [ "com.palantir.atlasdb:atlasdb-api", - "com.palantir.atlasdb:atlasdb-config" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign" ] }, "javax.ws.rs:javax.ws.rs-api": { diff --git a/timelock-server-distribution/versions.lock b/timelock-server-distribution/versions.lock index 082f99a13e6..9b7e8b7ac3b 100644 --- a/timelock-server-distribution/versions.lock +++ b/timelock-server-distribution/versions.lock @@ -598,6 +598,7 @@ "locked": "1.4.0", "transitive": [ "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "com.palantir.atlasdb:timelock-agent" ] }, @@ -1052,6 +1053,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" diff --git a/timelock-server/versions.lock b/timelock-server/versions.lock index 9fe89f281a9..2ba066b95ab 100644 --- a/timelock-server/versions.lock +++ b/timelock-server/versions.lock @@ -455,6 +455,7 @@ "locked": "1.4.0", "transitive": [ "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "com.palantir.atlasdb:timelock-agent" ] }, @@ -829,6 +830,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" @@ -1785,6 +1787,7 @@ "locked": "1.4.0", "transitive": [ "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "com.palantir.atlasdb:timelock-agent" ] }, @@ -2230,6 +2233,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:atlasdb-config", + "com.palantir.atlasdb:atlasdb-feign", "org.glassfish.jersey.core:jersey-server", "org.glassfish.jersey.ext:jersey-bean-validation", "org.hibernate:hibernate-validator" From 8fdd50b0aa33c1872737c6ff8d8fda999eae3a2f Mon Sep 17 00:00:00 2001 From: Jeremy Kong Date: Mon, 13 Nov 2017 13:29:41 +0000 Subject: [PATCH 13/67] [TTT] [no release notes] Document behaviour regarding index rows (#2658) * [no release notes] Document behaviour regarding index rows * fix compile bug * ``List`` --- docs/source/schemas/tables_and_indices.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/schemas/tables_and_indices.rst b/docs/source/schemas/tables_and_indices.rst index 0ee627320c3..fc67ba6c16a 100644 --- a/docs/source/schemas/tables_and_indices.rst +++ b/docs/source/schemas/tables_and_indices.rst @@ -495,6 +495,17 @@ succeeded by a ``partitioner()`` call, in the exact same manner as for table rows. For more information, see the Partitioners subsection of Table Rows. +.. note:: + + Internally, index rows are stored including a reference to the source column, + but this is stripped out in the generated code before being returned to the user. + Thus, if one uses a ``List`` of results returned from an index table (e.g. through ``getRowColumns``, + one may encounter multiple values that appear to be the same). The standard workaround is to use + a ``Set`` to deduplicate the results. + + Please see discussion on `issue 604 `__ for more details + regarding this behaviour. + Constraints ----------- From 6fed36fd741a30af47fb838647f460d84cd3e321 Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Mon, 13 Nov 2017 15:47:35 +0000 Subject: [PATCH 14/67] Refactor and Instrument CassandraClient api (#2665) * Sanitize Client API * Instrument CassandraClient * checkstyle * Address comment * [no release notes] * checkstyle * Fix cas --- .../keyvalue/cassandra/CassandraClient.java | 16 ++++---- .../cassandra/CassandraClientFactory.java | 3 +- .../cassandra/CassandraClientImpl.java | 41 +++++++++++++++---- .../cassandra/CassandraClientPoolImpl.java | 10 ----- .../CassandraKeyValueServiceImpl.java | 16 +++----- .../CassandraTimestampBoundStore.java | 8 ++-- .../atlasdb/keyvalue/cassandra/Heartbeat.java | 2 +- .../cassandra/SchemaMutationLock.java | 8 ++-- .../keyvalue/cassandra/paging/RowGetter.java | 7 +--- 9 files changed, 58 insertions(+), 53 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 8a9c40e1957..d6719a4e6a3 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -24,8 +24,6 @@ import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; -import org.apache.cassandra.thrift.ColumnParent; -import org.apache.cassandra.thrift.ColumnPath; import org.apache.cassandra.thrift.Compression; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.CqlResult; @@ -39,30 +37,34 @@ import org.apache.cassandra.thrift.TimedOutException; import org.apache.cassandra.thrift.UnavailableException; +import com.palantir.atlasdb.keyvalue.api.TableReference; + @SuppressWarnings({"all"}) // thrift variable names. public interface CassandraClient { Cassandra.Client rawClient(); - Map> multiget_slice(List keys, ColumnParent column_parent, + Map> multiget_slice(TableReference tableRef, + List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - List get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, + List get_range_slices(TableReference tableRef, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; void batch_mutate(Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) + ColumnOrSuperColumn get(TableReference tableReference, ByteBuffer key, + byte[] column, ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException; - CASResult cas(ByteBuffer key, String column_family, List expected, List updates, + CASResult cas(TableReference tableReference, ByteBuffer key, List expected, List updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) + CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException; } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index 74ecb10e19c..72ded6c1d88 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -46,6 +46,7 @@ import com.google.common.collect.Maps; import com.palantir.atlasdb.cassandra.CassandraCredentialsConfig; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; +import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.common.exception.AtlasDbDependencyException; import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.UnsafeArg; @@ -88,7 +89,7 @@ public CassandraClient create() throws Exception { } private CassandraClient instrumentClient(Client client) { - return new CassandraClientImpl(client); + return AtlasDbMetrics.instrument(CassandraClient.class, new CassandraClientImpl(client)); } private static Cassandra.Client getClient(InetSocketAddress addr, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java index 7efc9087bf9..164486cfa2d 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java @@ -40,6 +40,9 @@ import org.apache.cassandra.thrift.UnavailableException; import org.apache.thrift.TException; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.impl.AbstractKeyValueService; + @SuppressWarnings({"all"}) // thrift variable names. public class CassandraClientImpl implements CassandraClient { private Cassandra.Client client; @@ -54,17 +57,22 @@ public Cassandra.Client rawClient() { } @Override - public Map> multiget_slice(List keys, ColumnParent column_parent, + public Map> multiget_slice( + TableReference tableRef, List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - return client.multiget_slice(keys, column_parent, predicate, consistency_level); + ColumnParent colFam = getColumnParent(tableRef); + return client.multiget_slice(keys, colFam, predicate, consistency_level); } @Override - public List get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, + public List get_range_slices(TableReference tableRef, + SlicePredicate predicate, + KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - return client.get_range_slices(column_parent, predicate, range, consistency_level); + ColumnParent colFam = getColumnParent(tableRef); + return client.get_range_slices(colFam, predicate, range, consistency_level); } @Override @@ -75,16 +83,27 @@ public void batch_mutate(Map>> mutation_m } @Override - public ColumnOrSuperColumn get(ByteBuffer key, ColumnPath column_path, ConsistencyLevel consistency_level) + public ColumnOrSuperColumn get(TableReference tableReference, + ByteBuffer key, + byte[] column, + ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { - return client.get(key, column_path, consistency_level); + ColumnPath columnPath = new ColumnPath(tableReference.getQualifiedName()); + columnPath.setColumn(column); + return client.get(key, columnPath, consistency_level); } @Override - public CASResult cas(ByteBuffer key, String column_family, List expected, List updates, - ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) + public CASResult cas(TableReference tableReference, + ByteBuffer key, + List expected, + List updates, + ConsistencyLevel serial_consistency_level, + ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - return client.cas(key, column_family, expected, updates, serial_consistency_level, commit_consistency_level); + String internalTableName = AbstractKeyValueService.internalTableName(tableReference); + return client.cas(key, internalTableName, expected, updates, serial_consistency_level, + commit_consistency_level); } @Override @@ -93,4 +112,8 @@ public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, C TException { return client.execute_cql3_query(query, compression, consistency); } + + private ColumnParent getColumnParent(TableReference tableRef) { + return new ColumnParent(AbstractKeyValueService.internalTableName(tableRef)); + } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java index 8efeb1a8259..72fe3d3d31c 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java @@ -70,7 +70,6 @@ import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; -import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.cassandra.CassandraClientFactory.ClientCreationFailedException; import com.palantir.atlasdb.util.MetricsManager; import com.palantir.common.base.FunctionCheckedException; @@ -486,15 +485,6 @@ void runOneTimeStartupChecks() { } } - //todo dedupe this into a name-demangling class that everyone can access - protected static String internalTableName(TableReference tableRef) { - String tableName = tableRef.getQualifiedName(); - if (tableName.startsWith("_")) { - return tableName; - } - return tableName.replaceFirst("\\.", "__"); - } - private InetSocketAddress getAddressForHostThrowUnchecked(String host) { try { return getAddressForHost(host); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java index 99068879d07..c2f810cc205 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java @@ -448,12 +448,10 @@ public Map apply(CassandraClient client) throws Exception { List rowNames = wrap(batch); - ColumnParent colFam = new ColumnParent(internalTableName(tableRef)); Map> results = multigetInternal( client, tableRef, rowNames, - colFam, pred, readConsistency); Map ret = Maps.newHashMapWithExpectedSize(batch.size()); @@ -648,7 +646,7 @@ public Void apply(CassandraClient client) throws Exception { } Map> results = - multigetInternal(client, tableRef, rowNames, colFam, predicate, consistency); + multigetInternal(client, tableRef, rowNames, predicate, consistency); visitor.visit(results); return null; } @@ -792,9 +790,8 @@ public RowColumnRangeExtractor.RowColumnRangeResult apply(CassandraClient client Limit limit = Limit.of(batchColumnRangeSelection.getBatchHint()); SlicePredicate pred = SlicePredicates.create(range, limit); - ColumnParent colFam = new ColumnParent(internalTableName(tableRef)); Map> results = - multigetInternal(client, tableRef, wrap(rows), colFam, pred, readConsistency); + multigetInternal(client, tableRef, wrap(rows), pred, readConsistency); RowColumnRangeExtractor extractor = new RowColumnRangeExtractor(); extractor.extractResults(rows, results, startTs); @@ -849,9 +846,8 @@ public TokenBackedBasicResultsPage, byte[]> apply(CassandraCl ByteBuffer rowByteBuffer = ByteBuffer.wrap(row); - ColumnParent colFam = new ColumnParent(internalTableName(tableRef)); Map> results = multigetInternal(client, tableRef, - ImmutableList.of(rowByteBuffer), colFam, pred, readConsistency); + ImmutableList.of(rowByteBuffer), pred, readConsistency); if (results.isEmpty()) { return SimpleTokenBackedResultsPage.create(startCol, ImmutableList.of(), false); @@ -1200,11 +1196,11 @@ private Map> multigetInternal( CassandraClient client, TableReference tableRef, List rowNames, - ColumnParent colFam, SlicePredicate pred, ConsistencyLevel consistency) throws TException { try { - return queryRunner.run(client, tableRef, () -> client.multiget_slice(rowNames, colFam, pred, consistency)); + return queryRunner.run(client, tableRef, () -> client.multiget_slice(tableRef, rowNames, pred, + consistency)); } catch (UnavailableException e) { throw new InsufficientConsistencyException( "This get operation requires " + consistency + " Cassandra nodes to be up and available.", e); @@ -2165,8 +2161,8 @@ private CASResult executeCheckAndSet(CassandraClient client, CheckAndSetRequest Column newColumn = makeColumn(colName, request.newValue(), timestamp); return queryRunner.run(client, table, () -> client.cas( + table, rowName, - internalTableName(table), oldColumns, ImmutableList.of(newColumn), ConsistencyLevel.SERIAL, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java index cc220f536a6..04bf263d03a 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBoundStore.java @@ -23,7 +23,6 @@ import org.apache.cassandra.thrift.CASResult; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; -import org.apache.cassandra.thrift.ColumnPath; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.NotFoundException; import org.slf4j.Logger; @@ -130,11 +129,10 @@ public synchronized long getUpperLimit() { @Override public Long apply(CassandraClient client) { ByteBuffer rowName = getRowName(); - ColumnPath columnPath = new ColumnPath(AtlasDbConstants.TIMESTAMP_TABLE.getQualifiedName()); - columnPath.setColumn(getColumnName()); ColumnOrSuperColumn result; try { - result = client.get(rowName, columnPath, ConsistencyLevel.LOCAL_QUORUM); + result = client.get(AtlasDbConstants.TIMESTAMP_TABLE, rowName, getColumnName(), + ConsistencyLevel.LOCAL_QUORUM); } catch (NotFoundException e) { result = null; } catch (Exception e) { @@ -190,8 +188,8 @@ private void cas(CassandraClient client, Long oldVal, long newVal) { try { DebugLogger.logger.info("[CAS] Trying to set upper limit from {} to {}.", oldVal, newVal); result = client.cas( + AtlasDbConstants.TIMESTAMP_TABLE, getRowName(), - AtlasDbConstants.TIMESTAMP_TABLE.getQualifiedName(), oldVal == null ? ImmutableList.of() : ImmutableList.of(makeColumn(oldVal)), ImmutableList.of(makeColumn(newVal)), ConsistencyLevel.SERIAL, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java index 340a57b517f..d1d47837599 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/Heartbeat.java @@ -94,8 +94,8 @@ private CASResult writeDdlLockWithCas(CassandraClient client, Column ourUpdate, try { return queryRunner.run(client, lockTable, () -> client.cas( + lockTable, SchemaMutationLock.getGlobalDdlLockRowName(), - lockTable.getQualifiedName(), expected, ImmutableList.of(ourUpdate), ConsistencyLevel.SERIAL, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java index 46e4d5470c1..b6e7e6a85c2 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLock.java @@ -29,7 +29,6 @@ import org.apache.cassandra.thrift.CASResult; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; -import org.apache.cassandra.thrift.ColumnPath; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.NotFoundException; import org.apache.cassandra.thrift.UnavailableException; @@ -344,13 +343,12 @@ private boolean trySchemaMutationUnlockOnce(long perOperationNodeId) { private Optional queryExistingLockColumn(CassandraClient client) throws TException { TableReference lockTableRef = lockTable.getOnlyTable(); - ColumnPath columnPath = new ColumnPath(lockTableRef.getQualifiedName()); - columnPath.setColumn(getGlobalDdlLockColumnName()); Column existingColumn = null; ConsistencyLevel localQuorum = ConsistencyLevel.LOCAL_QUORUM; try { ColumnOrSuperColumn result = queryRunner.run(client, lockTableRef, - () -> client.get(getGlobalDdlLockRowName(), columnPath, localQuorum)); + () -> client.get(lockTableRef, getGlobalDdlLockRowName(), getGlobalDdlLockColumnName(), + localQuorum)); existingColumn = result.getColumn(); } catch (UnavailableException e) { throw new InsufficientConsistencyException( @@ -372,8 +370,8 @@ private CASResult writeDdlLockWithCas( TableReference lockTableRef = lockTable.getOnlyTable(); return queryRunner.run(client, lockTableRef, () -> client.cas( + lockTableRef, getGlobalDdlLockRowName(), - lockTableRef.getQualifiedName(), expectedLockValue, ImmutableList.of(newLockValue), ConsistencyLevel.SERIAL, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java index a251a2e8e5e..4fe67978bd7 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java @@ -18,7 +18,6 @@ import java.net.InetSocketAddress; import java.util.List; -import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.KeyRange; import org.apache.cassandra.thrift.KeySlice; @@ -29,7 +28,6 @@ import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.cassandra.CassandraClient; import com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool; -import com.palantir.atlasdb.keyvalue.cassandra.CassandraKeyValueServiceImpl; import com.palantir.atlasdb.keyvalue.cassandra.TracingQueryRunner; import com.palantir.common.base.FunctionCheckedException; import com.palantir.common.base.Throwables; @@ -52,7 +50,6 @@ public RowGetter( } public List getRows(KeyRange keyRange, SlicePredicate slicePredicate) throws Exception { - ColumnParent colFam = new ColumnParent(CassandraKeyValueServiceImpl.internalTableName(tableRef)); InetSocketAddress host = clientPool.getRandomHostForKey(keyRange.getStart_key()); return clientPool.runWithRetryOnHost( host, @@ -61,7 +58,7 @@ public List getRows(KeyRange keyRange, SlicePredicate slicePredicate) public List apply(CassandraClient client) throws Exception { try { return queryRunner.run(client, tableRef, - () -> client.get_range_slices(colFam, slicePredicate, keyRange, consistency)); + () -> client.get_range_slices(tableRef, slicePredicate, keyRange, consistency)); } catch (UnavailableException e) { throw new InsufficientConsistencyException("get_range_slices requires " + consistency + " Cassandra nodes to be up and available.", e); @@ -72,7 +69,7 @@ public List apply(CassandraClient client) throws Exception { @Override public String toString() { - return "get_range_slices(" + colFam + ")"; + return "get_range_slices(" + tableRef + ")"; } }); } From e8e85f96d77eaa14024365a47259002bec9f889e Mon Sep 17 00:00:00 2001 From: Jeremy Kong Date: Mon, 13 Nov 2017 17:29:08 +0000 Subject: [PATCH 15/67] Live Reloading the TimeLock Block, Part 3: Working with 0 Nodes (#2647) * 0 nodes part 1 * add support for 0 servers in a ServerListConfig * extend deserialization tests * More tests * code defensively * [no release notes] defer to 2648 * Fixed CR nits * singleton server list --- .../atlasdb/config/AtlasDbConfig.java | 10 +++ .../atlasdb/config/TimeLockClientConfig.java | 5 +- .../atlasdb/config/TimeLockRuntimeConfig.java | 5 +- .../atlasdb/config/AtlasDbConfigTest.java | 70 ++++++++++++++----- ...lasDbRuntimeConfigDeserializationTest.java | 25 +++++++ .../atlasdb/config/ServerListConfigsTest.java | 17 +++++ .../atlasdb/http/AtlasDbHttpClientsTest.java | 40 +++++++++++ .../runtime-config-block-no-servers.yml | 6 ++ .../test/resources/runtime-config-block.yml | 6 +- .../atlasdb/config/ServerListConfig.java | 8 +-- .../http/AtlasDbFeignTargetFactory.java | 31 +++++--- 11 files changed, 188 insertions(+), 35 deletions(-) create mode 100644 atlasdb-config/src/test/resources/runtime-config-block-no-servers.yml diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java index 7762fe99292..d8cc2d5aec8 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java @@ -275,6 +275,8 @@ private void checkLeaderAndTimelockBlocks() { "If the leader block is present, then the lock and timestamp server blocks must both be absent."); Preconditions.checkState(!timelock().isPresent(), "If the leader block is present, then the timelock block must be absent."); + Preconditions.checkState(!leader().get().leaders().isEmpty(), + "Leader config must have at least one server."); } if (timelock().isPresent()) { @@ -286,6 +288,14 @@ private void checkLeaderAndTimelockBlocks() { private void checkLockAndTimestampBlocks() { Preconditions.checkState(lock().isPresent() == timestamp().isPresent(), "Lock and timestamp server blocks must either both be present or both be absent."); + checkServersListHasAtLeastOneServerIfPresent(lock()); + checkServersListHasAtLeastOneServerIfPresent(timestamp()); + } + + private static void checkServersListHasAtLeastOneServerIfPresent(Optional serverListOptional) { + serverListOptional.ifPresent( + serverList -> Preconditions.checkState(serverList.hasAtLeastOneServer(), + "Server list must have at least one server.")); } private String checkNamespaceConfigAndGetNamespace() { diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java index f1934d44d43..8623cee00b2 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockClientConfig.java @@ -48,7 +48,10 @@ public String getClientOrThrow() { * for connecting to TimeLock. */ @Deprecated - public abstract ServerListConfig serversList(); + @Value.Default + public ServerListConfig serversList() { + return ImmutableServerListConfig.builder().build(); + } public ServerListConfig toNamespacedServerList() { return ServerListConfigs.namespaceUris(serversList(), getClientOrThrow()); diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java index 6335b5c3f93..e94f8834b07 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/TimeLockRuntimeConfig.java @@ -25,5 +25,8 @@ @JsonDeserialize(as = ImmutableTimeLockRuntimeConfig.class) @Value.Immutable public abstract class TimeLockRuntimeConfig { - public abstract ServerListConfig serversList(); + @Value.Default + public ServerListConfig serversList() { + return ImmutableServerListConfig.builder().build(); + } } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java index 6c6758cfb9f..34f8878b5a9 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbConfigTest.java @@ -44,12 +44,12 @@ public class AtlasDbConfigTest { .localServer("me") .addLeaders("me") .build(); - private static final ServerListConfig DEFAULT_SERVER_LIST = ImmutableServerListConfig.builder() + private static final ServerListConfig SINGLETON_SERVER_LIST = ImmutableServerListConfig.builder() .addServers("server") .build(); private static final TimeLockClientConfig TIMELOCK_CONFIG = ImmutableTimeLockClientConfig.builder() .client("client") - .serversList(DEFAULT_SERVER_LIST) + .serversList(SINGLETON_SERVER_LIST) .build(); private static final Optional SSL_CONFIG = Optional.of(mock(SslConfiguration.class)); private static final Optional OTHER_SSL_CONFIG = Optional.of(mock(SslConfiguration.class)); @@ -61,13 +61,14 @@ public class AtlasDbConfigTest { private static final TimeLockClientConfig TIMELOCK_CONFIG_WITH_OPTIONAL_EMPTY_CLIENT = ImmutableTimeLockClientConfig .builder() .client(Optional.empty()) - .serversList(DEFAULT_SERVER_LIST) + .serversList(SINGLETON_SERVER_LIST) .build(); private static final TimeLockClientConfig TIMELOCK_CONFIG_WITH_OTHER_CLIENT = ImmutableTimeLockClientConfig .builder() .client(OTHER_CLIENT) - .serversList(DEFAULT_SERVER_LIST) + .serversList(SINGLETON_SERVER_LIST) .build(); + private static final String CLIENT_NAMESPACE = "client"; @BeforeClass public static void setUp() { @@ -113,8 +114,8 @@ public void configWithTimelockBlockIsValid() { public void remoteLockAndTimestampConfigIsValid() { AtlasDbConfig config = ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .lock(DEFAULT_SERVER_LIST) - .timestamp(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); assertThat(config.getNamespaceString(), equalTo(TEST_NAMESPACE)); assertThat(config, not(nullValue())); @@ -125,8 +126,8 @@ public void leaderBlockNotPermittedWithLockAndTimestampBlocks() { ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .leader(LEADER_CONFIG) - .lock(DEFAULT_SERVER_LIST) - .timestamp(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); } @@ -136,9 +137,9 @@ public void timelockBlockNotPermittedWithLockAndTimestampBlocks() { .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .timelock(ImmutableTimeLockClientConfig.builder() .client("testClient") - .serversList(DEFAULT_SERVER_LIST).build()) - .lock(DEFAULT_SERVER_LIST) - .timestamp(DEFAULT_SERVER_LIST) + .serversList(SINGLETON_SERVER_LIST).build()) + .lock(SINGLETON_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); } @@ -156,7 +157,7 @@ public void leaderBlockNotPermittedWithLockBlock() { ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .leader(LEADER_CONFIG) - .lock(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) .build(); } @@ -165,7 +166,7 @@ public void leaderBlockNotPermittedWithTimestampBlock() { ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) .leader(LEADER_CONFIG) - .timestamp(DEFAULT_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); } @@ -173,7 +174,7 @@ public void leaderBlockNotPermittedWithTimestampBlock() { public void lockBlockRequiresTimestampBlock() { ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .lock(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) .build(); } @@ -181,7 +182,7 @@ public void lockBlockRequiresTimestampBlock() { public void timestampBlockRequiresLockBlock() { ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .timestamp(DEFAULT_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); } @@ -316,8 +317,8 @@ public void addingFallbackSslAddsItToLeaderBlock() { public void addingFallbackSslAddsItToLockBlock() { AtlasDbConfig withoutSsl = ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .lock(DEFAULT_SERVER_LIST) - .timestamp(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); AtlasDbConfig withSsl = AtlasDbConfigs.addFallbackSslConfigurationToAtlasDbConfig(withoutSsl, SSL_CONFIG); assertThat(withSsl.lock().get().sslConfiguration(), is(SSL_CONFIG)); @@ -338,8 +339,8 @@ public void addingFallbackSslAddsItToTimelockServersBlock() { public void addingFallbackSslAddsItToTimestampBlock() { AtlasDbConfig withoutSsl = ImmutableAtlasDbConfig.builder() .keyValueService(KVS_CONFIG_WITH_NAMESPACE) - .lock(DEFAULT_SERVER_LIST) - .timestamp(DEFAULT_SERVER_LIST) + .lock(SINGLETON_SERVER_LIST) + .timestamp(SINGLETON_SERVER_LIST) .build(); AtlasDbConfig withSsl = AtlasDbConfigs.addFallbackSslConfigurationToAtlasDbConfig(withoutSsl, SSL_CONFIG); assertThat(withSsl.timestamp().get().sslConfiguration(), is(SSL_CONFIG)); @@ -367,4 +368,35 @@ public void addingAbsentFallbackSslWhenItDoesntExistsLeavesItAsAbsent() { AtlasDbConfig withSsl = AtlasDbConfigs.addFallbackSslConfigurationToAtlasDbConfig(withoutSsl, NO_SSL_CONFIG); assertThat(withSsl.leader().get().sslConfiguration(), is(NO_SSL_CONFIG)); } + + @Test + public void cannotSpecifyZeroServersIfUsingTimestampBlock() { + assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() + .keyValueService(KVS_CONFIG_WITHOUT_NAMESPACE) + .namespace(CLIENT_NAMESPACE) + .timestamp(ImmutableServerListConfig.builder().build()) + .lock(SINGLETON_SERVER_LIST) + .build()).isInstanceOf(IllegalStateException.class); + } + + @Test + public void cannotSpecifyZeroServersIfUsingLockBlock() { + assertThatThrownBy(() -> ImmutableAtlasDbConfig.builder() + .keyValueService(KVS_CONFIG_WITHOUT_NAMESPACE) + .namespace(CLIENT_NAMESPACE) + .timestamp(SINGLETON_SERVER_LIST) + .lock(ImmutableServerListConfig.builder().build()) + .build()).isInstanceOf(IllegalStateException.class); + } + + @Test + public void canSpecifyZeroServersIfUsingTimelockBlock() { + ImmutableAtlasDbConfig.builder() + .keyValueService(KVS_CONFIG_WITHOUT_NAMESPACE) + .namespace(CLIENT_NAMESPACE) + .timelock(ImmutableTimeLockClientConfig.builder() + .serversList(ImmutableServerListConfig.builder().build()) + .build()) + .build(); + } } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java index fc557ad8f83..16d82f4dbff 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfigDeserializationTest.java @@ -20,12 +20,18 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import org.junit.Test; +import com.palantir.remoting.api.config.ssl.SslConfiguration; + public class AtlasDbRuntimeConfigDeserializationTest { private static final File TEST_RUNTIME_CONFIG_FILE = new File( AtlasDbRuntimeConfigDeserializationTest.class.getResource("/runtime-config-block.yml").getPath()); + private static final File TEST_RUNTIME_CONFIG_NO_SERVERS_FILE = new File( + AtlasDbRuntimeConfigDeserializationTest.class.getResource("/runtime-config-block-no-servers.yml") + .getPath()); @Test public void canDeserializeRuntimeConfig() throws IOException { @@ -39,5 +45,24 @@ public void canDeserializeRuntimeConfig() throws IOException { "https://foo1:12345", "https://foo2:8421", "https://foo3:9421"); + assertThat(runtimeConfig.timelockRuntime().get().serversList().sslConfiguration()).satisfies( + sslConfiguration -> sslConfiguration.ifPresent(this::assertSslConfigDeserializedCorrectly)); + } + + private void assertSslConfigDeserializedCorrectly(SslConfiguration sslConfiguration) { + assertThat(sslConfiguration.keyStorePassword()).hasValue("0987654321"); + assertThat(sslConfiguration.keyStorePath()).hasValue(Paths.get("var", "security", "keyStore.jks")); + assertThat(sslConfiguration.trustStorePath()).isEqualTo(Paths.get("var", "security", "trustStore.jks")); + } + + @Test + public void canDeserializeRuntimeConfigWithZeroServers() throws IOException { + AtlasDbRuntimeConfig runtimeConfig = + AtlasDbConfigs.OBJECT_MAPPER.readValue(TEST_RUNTIME_CONFIG_NO_SERVERS_FILE, AtlasDbRuntimeConfig.class); + assertThat(runtimeConfig.timestampClient().enableTimestampBatching()).isTrue(); + + assertThat(runtimeConfig.timelockRuntime()).isPresent(); + assertThat(runtimeConfig.timelockRuntime().get().serversList().servers()).isEmpty(); + assertThat(runtimeConfig.timelockRuntime().get().serversList().sslConfiguration()).isEmpty(); } } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java index 1f5b16c11a3..38cadd52592 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/config/ServerListConfigsTest.java @@ -34,6 +34,8 @@ public class ServerListConfigsTest { private static final ImmutableServerListConfig SERVERS_LIST_3 = ImmutableServerListConfig.builder() .addServers("three/") .build(); + private static final ImmutableServerListConfig SERVERS_LIST_EMPTY = ImmutableServerListConfig.builder() + .build(); private static final TimeLockClientConfig INSTALL_CONFIG = ImmutableTimeLockClientConfig.builder() .serversList(SERVERS_LIST_1) @@ -60,6 +62,12 @@ public void namespacingCanDealWithTrailingSlash() { assertThat(namespacedServersList.servers()).containsExactlyInAnyOrder("three/client"); } + @Test + public void namespacingCanDealWithServerListConfigsWithZeroNodes() { + ServerListConfig namespacedServersList = ServerListConfigs.namespaceUris(SERVERS_LIST_EMPTY, CLIENT); + assertThat(namespacedServersList.servers()).isEmpty(); + } + @Test public void prioritisesRuntimeConfigIfAvailable() { ServerListConfig resolvedConfig = ServerListConfigs.parseInstallAndRuntimeConfigs( @@ -69,6 +77,15 @@ public void prioritisesRuntimeConfigIfAvailable() { assertThat(resolvedConfig.servers()).containsExactlyInAnyOrder("one/client", "two/client"); } + @Test + public void prioritisesRuntimeConfigEvenIfThatHasNoClients() { + ServerListConfig resolvedConfig = ServerListConfigs.parseInstallAndRuntimeConfigs( + INSTALL_CONFIG, + () -> Optional.of(ImmutableTimeLockRuntimeConfig.builder().build()), + CLIENT); + assertThat(resolvedConfig.servers()).isEmpty(); + } + @Test public void fallsBackToInstallConfigIfRuntimeConfigNotAvailable() { ServerListConfig resolvedConfig = ServerListConfigs.parseInstallAndRuntimeConfigs( diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java index 77deb189615..4241d8c07c0 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/AtlasDbHttpClientsTest.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.net.ssl.SSLSocketFactory; import javax.ws.rs.GET; @@ -49,7 +50,9 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.config.ImmutableServerListConfig; +import com.palantir.atlasdb.config.ServerListConfig; import com.palantir.atlasdb.factory.ServiceCreator; +import com.palantir.common.remoting.ServiceNotAvailableException; import com.palantir.remoting.api.config.service.ProxyConfiguration; import com.palantir.remoting3.config.ssl.SslSocketFactories; @@ -172,6 +175,43 @@ public void canLiveReloadServersList() { unavailableServer.verify(getRequestedFor(urlMatching(TEST_ENDPOINT))); } + @Test + public void httpProxyThrowsServiceNotAvailableExceptionIfConfiguredWithZeroNodes() { + TestResource testResource = AtlasDbHttpClients.createLiveReloadingProxyWithQuickFailoverForTesting( + () -> ImmutableServerListConfig.builder().build(), + SslSocketFactories::createSslSocketFactory, + proxyConfiguration -> ProxySelector.getDefault(), + TestResource.class, + UserAgents.DEFAULT_VALUE); + + assertThatThrownBy(testResource::getTestNumber).isInstanceOf(ServiceNotAvailableException.class); + } + + @Test + public void httpProxyCanBeCommissionedAndDecommissionedIfNodeAvailabilityChanges() { + AtomicReference config = new AtomicReference<>(ImmutableServerListConfig.builder().build()); + + TestResource testResource = AtlasDbHttpClients.createLiveReloadingProxyWithQuickFailoverForTesting( + config::get, + SslSocketFactories::createSslSocketFactory, + proxyConfiguration -> ProxySelector.getDefault(), + TestResource.class, + UserAgents.DEFAULT_VALUE); + + // At this point, there are zero nodes in the config, so we should get ServiceNotAvailable. + assertThatThrownBy(testResource::getTestNumber).isInstanceOf(ServiceNotAvailableException.class); + + config.set(ImmutableServerListConfig.builder().addServers(getUriForPort(availablePort)).build()); + Uninterruptibles.sleepUninterruptibly( + PollingRefreshable.DEFAULT_REFRESH_INTERVAL.getSeconds() + 1, TimeUnit.SECONDS); + assertThat(testResource.getTestNumber(), equalTo(TEST_NUMBER)); + + config.set(ImmutableServerListConfig.builder().build()); + Uninterruptibles.sleepUninterruptibly( + PollingRefreshable.DEFAULT_REFRESH_INTERVAL.getSeconds() + 1, TimeUnit.SECONDS); + assertThatThrownBy(testResource::getTestNumber).isInstanceOf(ServiceNotAvailableException.class); + } + private static String getUriForPort(int port) { return String.format("http://%s:%s", WireMockConfiguration.DEFAULT_BIND_ADDRESS, port); } diff --git a/atlasdb-config/src/test/resources/runtime-config-block-no-servers.yml b/atlasdb-config/src/test/resources/runtime-config-block-no-servers.yml new file mode 100644 index 00000000000..69f199a5601 --- /dev/null +++ b/atlasdb-config/src/test/resources/runtime-config-block-no-servers.yml @@ -0,0 +1,6 @@ +timestampClient: + enableTimestampBatching: true + +timelockRuntime: + serversList: + servers: [] \ No newline at end of file diff --git a/atlasdb-config/src/test/resources/runtime-config-block.yml b/atlasdb-config/src/test/resources/runtime-config-block.yml index 138f646b146..0009f64358b 100644 --- a/atlasdb-config/src/test/resources/runtime-config-block.yml +++ b/atlasdb-config/src/test/resources/runtime-config-block.yml @@ -6,4 +6,8 @@ timelockRuntime: servers: - "https://foo1:12345" - "https://foo2:8421" - - "https://foo3:9421" \ No newline at end of file + - "https://foo3:9421" + sslConfiguration: + trustStorePath: var/security/trustStore.jks + keyStorePath: var/security/keyStore.jks + keyStorePassword: 0987654321 \ No newline at end of file diff --git a/atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java b/atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java index fc64ba743cc..744ff7b36f5 100644 --- a/atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java +++ b/atlasdb-feign/src/main/java/com/palantir/atlasdb/config/ServerListConfig.java @@ -18,8 +18,6 @@ import java.util.Optional; import java.util.Set; -import javax.validation.constraints.Size; - import org.immutables.value.Value; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -31,11 +29,13 @@ @JsonSerialize(as = ImmutableServerListConfig.class) @Value.Immutable public interface ServerListConfig { - - @Size(min = 1) Set servers(); Optional sslConfiguration(); Optional proxyConfiguration(); + + default boolean hasAtLeastOneServer() { + return servers().size() >= 1; + } } diff --git a/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java index f808b2c8f14..8fadda71b53 100644 --- a/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java +++ b/atlasdb-feign/src/main/java/com/palantir/atlasdb/http/AtlasDbFeignTargetFactory.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.google.common.reflect.Reflection; import com.palantir.atlasdb.config.ServerListConfig; +import com.palantir.common.remoting.ServiceNotAvailableException; import com.palantir.remoting.api.config.service.ProxyConfiguration; import com.palantir.remoting.api.config.ssl.SslConfiguration; import com.palantir.remoting3.ext.refresh.RefreshableProxyInvocationHandler; @@ -179,14 +180,26 @@ public static T createLiveReloadingProxyWithFailover( type, RefreshableProxyInvocationHandler.create( configPollingRefreshable.getRefreshable(), - serverListConfig -> createProxyWithFailover( - serverListConfig.sslConfiguration().map(sslSocketFactoryCreator), - serverListConfig.proxyConfiguration().map(proxySelectorCreator), - serverListConfig.servers(), - feignConnectTimeout, - feignReadTimeout, - maxBackoffMillis, - type, - userAgent))); + serverListConfig -> { + if (serverListConfig.hasAtLeastOneServer()) { + return createProxyWithFailover( + serverListConfig.sslConfiguration().map(sslSocketFactoryCreator), + serverListConfig.proxyConfiguration().map(proxySelectorCreator), + serverListConfig.servers(), + feignConnectTimeout, + feignReadTimeout, + maxBackoffMillis, + type, + userAgent); + } + return createProxyForZeroNodes(type); + })); + } + + private static T createProxyForZeroNodes(Class type) { + return Reflection.newProxy(type, (unused1, unused2, unused3) -> { + throw new ServiceNotAvailableException("The " + type.getSimpleName() + " is currently unavailable," + + " because configuration contains zero servers."); + }); } } From 74180cf9704e03de546a03f87fdf6e9bca32be84 Mon Sep 17 00:00:00 2001 From: nziebart Date: Mon, 13 Nov 2017 19:23:16 +0000 Subject: [PATCH 16/67] check immutable ts (#2406) * check immutable ts * checkstyle * release notes * Fix TM creation * checkstyle --- .../transaction/impl/SnapshotTransaction.java | 4 +++ .../impl/TransactionManagerTest.java | 35 +++++++++++++++++++ docs/source/release_notes/release-notes.rst | 5 +++ 3 files changed, 44 insertions(+) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 2bca1ec48d5..641284ce688 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -1301,6 +1301,10 @@ private void checkConstraints() { private void commitWrites(TransactionService transactionService) { if (!hasWrites()) { + // if there are no writes, we must still make sure the immutable timestamp lock is still valid, + // to ensure that sweep hasn't thoroughly deleted cells we tried to read + // TODO(nziebart): we actually only need to do this if we've read from tables with THOROUGH sweep strategy + throwIfImmutableTsOrCommitLocksExpired(null); return; } diff --git a/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/transaction/impl/TransactionManagerTest.java b/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/transaction/impl/TransactionManagerTest.java index fb45108b992..2ae5d4a3bb7 100644 --- a/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/transaction/impl/TransactionManagerTest.java +++ b/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/transaction/impl/TransactionManagerTest.java @@ -17,23 +17,31 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import java.util.UUID; + import org.junit.Test; import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.cleaner.NoOpCleaner; import com.palantir.atlasdb.keyvalue.api.KeyValueService; import com.palantir.atlasdb.keyvalue.impl.InMemoryKeyValueService; +import com.palantir.atlasdb.monitoring.TimestampTrackerImpl; import com.palantir.atlasdb.transaction.api.AtlasDbConstraintCheckingMode; +import com.palantir.atlasdb.transaction.api.TransactionFailedRetriableException; import com.palantir.atlasdb.transaction.api.TransactionManager; import com.palantir.atlasdb.transaction.api.TransactionTask; import com.palantir.common.concurrent.PTExecutors; import com.palantir.lock.LockClient; import com.palantir.lock.LockService; +import com.palantir.lock.v2.LockImmutableTimestampResponse; +import com.palantir.lock.v2.LockToken; +import com.palantir.lock.v2.TimelockService; import com.palantir.remoting2.tracing.Tracers; import com.palantir.timestamp.TimestampService; @@ -105,6 +113,33 @@ public void shouldNotMakeRemoteCallsInAReadonlyTransactionIfNoWorkIsDone() { verifyNoMoreInteractions(mockTimestampService); } + @Test + public void shouldConflictIfImmutableTimestampLockExpiresEvenIfNoWrites() { + TimelockService timelock = mock(TimelockService.class); + LockService mockLockService = mock(LockService.class); + TransactionManager txnManagerWithMocks = new SerializableTransactionManager(keyValueService, + timelock, + mockLockService, + transactionService, + () -> AtlasDbConstraintCheckingMode.FULL_CONSTRAINT_CHECKING_THROWS_EXCEPTIONS, + conflictDetectionManager, + sweepStrategyManager, + NoOpCleaner.INSTANCE, + TimestampTrackerImpl.createNoOpTracker(), + () -> AtlasDbConstants.DEFAULT_TIMESTAMP_CACHE_SIZE, + false, + () -> AtlasDbConstants.DEFAULT_TRANSACTION_LOCK_ACQUIRE_TIMEOUT_MS, + AbstractTransactionTest.GET_RANGES_THREAD_POOL_SIZE, + AbstractTransactionTest.DEFAULT_GET_RANGES_CONCURRENCY); + + when(timelock.getFreshTimestamp()).thenReturn(1L); + when(timelock.lockImmutableTimestamp(any())).thenReturn( + LockImmutableTimestampResponse.of(2L, LockToken.of(UUID.randomUUID()))); + + assertThatThrownBy(() -> txnManagerWithMocks.runTaskThrowOnConflict(txn -> null)) + .isInstanceOf(TransactionFailedRetriableException.class); + } + @Override protected KeyValueService getKeyValueService() { return new InMemoryKeyValueService(false, diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index bd2d1696896..5891b87703f 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -68,6 +68,11 @@ develop - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not (`Pull Request `__) + * - |fixed| + - Check that immutable timestamp is locked on write transactions with no writes. + This could cause long-running readers to read an incorrect empty value when using the ``Sweep.THOROUGH`` strategy. + (`Pull Request `__) + .. <<<<------------------------------------------------------------------------------------------------------------->>>> ======= From d4bf8053b8dcb4ca497b0d08411dcc65d23949a2 Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Mon, 13 Nov 2017 19:47:23 +0000 Subject: [PATCH 17/67] Propagate top-level KVS method names to CassandraClient (#2669) * Propagate method names down to multiget_slice * Add the corresponding KVS method to remaining methods * Add TODO * [no release notes] * nit --- .../keyvalue/cassandra/CassandraClient.java | 30 +++-- .../cassandra/CassandraClientFactory.java | 1 + .../cassandra/CassandraClientImpl.java | 13 +- .../CassandraExpiringKeyValueService.java | 25 ++-- .../CassandraKeyValueServiceImpl.java | 117 +++++++++++------- .../paging/CassandraRangePagingIterable.java | 2 +- .../keyvalue/cassandra/paging/RowGetter.java | 10 +- 7 files changed, 125 insertions(+), 73 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index d6719a4e6a3..9308718d920 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -43,28 +43,42 @@ public interface CassandraClient { Cassandra.Client rawClient(); - Map> multiget_slice(TableReference tableRef, + Map> multiget_slice(String kvsMethodName, + TableReference tableRef, List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - List get_range_slices(TableReference tableRef, SlicePredicate predicate, KeyRange range, + List get_range_slices(String kvsMethodName, + TableReference tableRef, + SlicePredicate predicate, + KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - void batch_mutate(Map>> mutation_map, ConsistencyLevel consistency_level) + void batch_mutate(String kvsMethodName, + Map>> mutation_map, + ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - ColumnOrSuperColumn get(TableReference tableReference, ByteBuffer key, - byte[] column, ConsistencyLevel consistency_level) + ColumnOrSuperColumn get(TableReference tableReference, + ByteBuffer key, + byte[] column, + ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, org.apache.thrift.TException; - CASResult cas(TableReference tableReference, ByteBuffer key, List expected, List updates, - ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) + CASResult cas(TableReference tableReference, + ByteBuffer key, + List expected, + List updates, + ConsistencyLevel serial_consistency_level, + ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) + CqlResult execute_cql3_query(ByteBuffer query, + Compression compression, + ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, org.apache.thrift.TException; } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index 72ded6c1d88..e477dc87665 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -89,6 +89,7 @@ public CassandraClient create() throws Exception { } private CassandraClient instrumentClient(Client client) { + // TODO(ssouza): use the kvsMethodName to tag the timers. return AtlasDbMetrics.instrument(CassandraClient.class, new CassandraClientImpl(client)); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java index 164486cfa2d..3bcc3740e41 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java @@ -58,15 +58,19 @@ public Cassandra.Client rawClient() { @Override public Map> multiget_slice( - TableReference tableRef, List keys, - SlicePredicate predicate, ConsistencyLevel consistency_level) + String kvsMethodName, + TableReference tableRef, + List keys, + SlicePredicate predicate, + ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { ColumnParent colFam = getColumnParent(tableRef); return client.multiget_slice(keys, colFam, predicate, consistency_level); } @Override - public List get_range_slices(TableReference tableRef, + public List get_range_slices(String kvsMethodName, + TableReference tableRef, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) @@ -76,7 +80,8 @@ public List get_range_slices(TableReference tableRef, } @Override - public void batch_mutate(Map>> mutation_map, + public void batch_mutate(String kvsMethodName, + Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { client.batch_mutate(mutation_map, consistency_level); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java index 49058f401cb..f4aabf0e305 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java @@ -73,20 +73,14 @@ private CassandraExpiringKeyValueService( @Override public void put(TableReference tableRef, Map values, long timestamp, long time, TimeUnit unit) { - try { - putInternal( - tableRef, - KeyValueServices.toConstantTimestampValues(values.entrySet(), timestamp), - CassandraKeyValueServices.convertTtl(time, unit)); - } catch (Exception e) { - throw Throwables.throwUncheckedException(e); - } + expiringPut("put", tableRef, values, timestamp, time, unit); } @Override public void putWithTimestamps(TableReference tableRef, Multimap values, long time, TimeUnit unit) { try { - putInternal(tableRef, values.entries(), CassandraKeyValueServices.convertTtl(time, unit)); + putInternal("putWithTimestamps", tableRef, values.entries(), + CassandraKeyValueServices.convertTtl(time, unit)); } catch (Exception e) { throw Throwables.throwUncheckedException(e); } @@ -112,7 +106,7 @@ public void multiPut( for (final List> p : partitions) { callables.add(() -> { Thread.currentThread().setName("Atlas expiry multiPut of " + p.size() + " cells into " + table); - put(table, Maps2.fromEntries(p), timestamp, time, unit); + expiringPut("multiPut", table, Maps2.fromEntries(p), timestamp, time, unit); return null; }); } @@ -135,4 +129,15 @@ public void multiPut( } } + private void expiringPut(String kvsMethodName, TableReference tableRef, Map values, long timestamp, + long time, TimeUnit unit) { + try { + putInternal(kvsMethodName, + tableRef, + KeyValueServices.toConstantTimestampValues(values.entrySet(), timestamp), + CassandraKeyValueServices.convertTtl(time, unit)); + } catch (Exception e) { + throw Throwables.throwUncheckedException(e); + } + } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java index c2f810cc205..d5601e85216 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java @@ -448,7 +448,7 @@ public Map apply(CassandraClient client) throws Exception { List rowNames = wrap(batch); - Map> results = multigetInternal( + Map> results = multigetInternal("getRows", client, tableRef, rowNames, @@ -502,7 +502,7 @@ private Map getRowsForSpecificColumns(final TableReference tableRef } StartTsResultsCollector collector = new StartTsResultsCollector(startTs); - loadWithTs(tableRef, cells, startTs, false, collector, readConsistency); + loadWithTs("getRows", tableRef, cells, startTs, false, collector, readConsistency); return collector.getCollectedResults(); } @@ -532,7 +532,7 @@ public Map get(TableReference tableRef, Map timestampBy try { Long firstTs = timestampByCell.values().iterator().next(); if (Iterables.all(timestampByCell.values(), Predicates.equalTo(firstTs))) { - return get(tableRef, timestampByCell.keySet(), firstTs); + return get("get", tableRef, timestampByCell.keySet(), firstTs); } SetMultimap cellsByTs = Multimaps.invertFrom( @@ -540,7 +540,7 @@ public Map get(TableReference tableRef, Map timestampBy Builder builder = ImmutableMap.builder(); for (long ts : cellsByTs.keySet()) { StartTsResultsCollector collector = new StartTsResultsCollector(ts); - loadWithTs(tableRef, cellsByTs.get(ts), ts, false, collector, readConsistency); + loadWithTs("get", tableRef, cellsByTs.get(ts), ts, false, collector, readConsistency); builder.putAll(collector.getCollectedResults()); } return builder.build(); @@ -549,18 +549,20 @@ public Map get(TableReference tableRef, Map timestampBy } } - private Map get(TableReference tableRef, Set cells, long maxTimestampExclusive) { + private Map get(String kvsMethodName, TableReference tableRef, Set cells, + long maxTimestampExclusive) { StartTsResultsCollector collector = new StartTsResultsCollector(maxTimestampExclusive); - loadWithTs(tableRef, cells, maxTimestampExclusive, false, collector, readConsistency); + loadWithTs(kvsMethodName, tableRef, cells, maxTimestampExclusive, false, collector, readConsistency); return collector.getCollectedResults(); } - private void loadWithTs(TableReference tableRef, - Set cells, - long startTs, - boolean loadAllTs, - ThreadSafeResultVisitor visitor, - ConsistencyLevel consistency) { + private void loadWithTs(String kvsMethodName, + TableReference tableRef, + Set cells, + long startTs, + boolean loadAllTs, + ThreadSafeResultVisitor visitor, + ConsistencyLevel consistency) { Map> hostsAndCells = partitionByHost(cells, Cells.getRowFunction()); int totalPartitions = hostsAndCells.keySet().size(); @@ -584,7 +586,8 @@ private void loadWithTs(TableReference tableRef, SafeArg.of("ipPort", hostAndCells.getKey())); } - tasks.addAll(getLoadWithTsTasksForSingleHost(hostAndCells.getKey(), + tasks.addAll(getLoadWithTsTasksForSingleHost(kvsMethodName, + hostAndCells.getKey(), tableRef, hostAndCells.getValue(), startTs, @@ -596,7 +599,8 @@ private void loadWithTs(TableReference tableRef, } // TODO(unknown): after cassandra api change: handle different column select per row - private List> getLoadWithTsTasksForSingleHost(final InetSocketAddress host, + private List> getLoadWithTsTasksForSingleHost(final String kvsMethodName, + final InetSocketAddress host, final TableReference tableRef, final Collection cells, final long startTs, @@ -646,7 +650,8 @@ public Void apply(CassandraClient client) throws Exception { } Map> results = - multigetInternal(client, tableRef, rowNames, predicate, consistency); + multigetInternal(kvsMethodName, client, tableRef, rowNames, predicate, + consistency); visitor.visit(results); return null; } @@ -791,7 +796,8 @@ public RowColumnRangeExtractor.RowColumnRangeResult apply(CassandraClient client SlicePredicate pred = SlicePredicates.create(range, limit); Map> results = - multigetInternal(client, tableRef, wrap(rows), pred, readConsistency); + multigetInternal("getRowsColumnRange", client, tableRef, wrap(rows), pred, + readConsistency); RowColumnRangeExtractor extractor = new RowColumnRangeExtractor(); extractor.extractResults(rows, results, startTs); @@ -846,8 +852,8 @@ public TokenBackedBasicResultsPage, byte[]> apply(CassandraCl ByteBuffer rowByteBuffer = ByteBuffer.wrap(row); - Map> results = multigetInternal(client, tableRef, - ImmutableList.of(rowByteBuffer), pred, readConsistency); + Map> results = multigetInternal("getRowsColumnRange", + client, tableRef, ImmutableList.of(rowByteBuffer), pred, readConsistency); if (results.isEmpty()) { return SimpleTokenBackedResultsPage.create(startCol, ImmutableList.of(), false); @@ -945,7 +951,7 @@ public Map getLatestTimestamps(TableReference tableRef, Map values, final long timestamp) { try { - putInternal(tableRef, KeyValueServices.toConstantTimestampValues(values.entrySet(), timestamp)); + putInternal("put", tableRef, KeyValueServices.toConstantTimestampValues(values.entrySet(), timestamp)); } catch (Exception e) { throw Throwables.unwrapAndThrowAtlasDbDependencyException(e); } @@ -965,7 +971,7 @@ public void put(final TableReference tableRef, final Map values, f @Override public void putWithTimestamps(TableReference tableRef, Multimap values) { try { - putInternal(tableRef, values.entries()); + putInternal("putWithTimestamps", tableRef, values.entries()); } catch (Exception e) { throw Throwables.unwrapAndThrowAtlasDbDependencyException(e); } @@ -976,14 +982,16 @@ protected int getMultiPutBatchCount() { return configManager.getConfig().mutationBatchCount(); } - private void putInternal(final TableReference tableRef, - final Iterable> values) throws Exception { - putInternal(tableRef, values, CassandraConstants.NO_TTL); + private void putInternal(final String kvsMethodName, + final TableReference tableRef, + final Iterable> values) throws Exception { + putInternal(kvsMethodName, tableRef, values, CassandraConstants.NO_TTL); } - protected void putInternal(final TableReference tableRef, - Iterable> values, - final int ttl) throws Exception { + protected void putInternal(String kvsMethodName, + final TableReference tableRef, + Iterable> values, + final int ttl) throws Exception { Map> cellsByHost = partitionMapByHost(values); List> tasks = Lists.newArrayListWithCapacity(cellsByHost.size()); for (final Map.Entry> entry : cellsByHost.entrySet()) { @@ -991,17 +999,22 @@ protected void putInternal(final TableReference tableRef, "Atlas putInternal " + entry.getValue().size() + " cell values to " + tableRef + " on " + entry.getKey(), () -> { - putForSingleHostInternal(entry.getKey(), tableRef, entry.getValue().entrySet(), ttl); + putForSingleHostInternal(kvsMethodName, + entry.getKey(), + tableRef, + entry.getValue().entrySet(), + ttl); return null; })); } runAllTasksCancelOnFailure(tasks); } - private void putForSingleHostInternal(final InetSocketAddress host, - final TableReference tableRef, - final Iterable> values, - final int ttl) throws Exception { + private void putForSingleHostInternal(String kvsMethodName, + final InetSocketAddress host, + final TableReference tableRef, + final Iterable> values, + final int ttl) throws Exception { clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { @Override public Void apply(CassandraClient client) throws Exception { @@ -1030,7 +1043,7 @@ public Void apply(CassandraClient client) throws Exception { tableMutations.add(mutation); } - batchMutateInternal(client, tableRef, map, writeConsistency); + batchMutateInternal(kvsMethodName, client, tableRef, map, writeConsistency); } return null; } @@ -1110,7 +1123,7 @@ private Void multiPutForSingleHostInternal(final InetSocketAddress host, return clientPool.runWithRetryOnHost(host, new FunctionCheckedException() { @Override public Void apply(CassandraClient client) throws Exception { - return batchMutateInternal(client, tableRefs, map, writeConsistency); + return batchMutateInternal("multiPut", client, tableRefs, map, writeConsistency); } @Override @@ -1169,20 +1182,22 @@ private Column createColumn(Cell cell, Value value, final int ttl) { return col; } - private void batchMutateInternal(CassandraClient client, - TableReference tableRef, - Map>> map, - ConsistencyLevel consistency) throws TException { - batchMutateInternal(client, ImmutableSet.of(tableRef), map, consistency); + private void batchMutateInternal(String kvsMethodName, + CassandraClient client, + TableReference tableRef, + Map>> map, + ConsistencyLevel consistency) throws TException { + batchMutateInternal(kvsMethodName, client, ImmutableSet.of(tableRef), map, consistency); } - private Void batchMutateInternal(CassandraClient client, + private Void batchMutateInternal(String kvsMethodName, + CassandraClient client, Set tableRefs, Map>> map, ConsistencyLevel consistency) throws TException { try { return queryRunner.run(client, tableRefs, () -> { - client.batch_mutate(map, consistency); + client.batch_mutate(kvsMethodName, map, consistency); return null; }); } catch (UnavailableException e) { @@ -1193,14 +1208,15 @@ private Void batchMutateInternal(CassandraClient client, } private Map> multigetInternal( + String kvsMethodName, CassandraClient client, TableReference tableRef, List rowNames, SlicePredicate pred, ConsistencyLevel consistency) throws TException { try { - return queryRunner.run(client, tableRef, () -> client.multiget_slice(tableRef, rowNames, pred, - consistency)); + return queryRunner.run(client, tableRef, () -> client.multiget_slice(kvsMethodName, tableRef, rowNames, + pred, consistency)); } catch (UnavailableException e) { throw new InsufficientConsistencyException( "This get operation requires " + consistency + " Cassandra nodes to be up and available.", e); @@ -1363,7 +1379,7 @@ public Void apply(CassandraClient client) throws Exception { for (Map>> map : maps.values()) { // NOTE: we run with ConsistencyLevel.ALL here instead of ConsistencyLevel.QUORUM // because we want to remove all copies of this data - batchMutateInternal(client, tableRef, map, deleteConsistency); + batchMutateInternal("delete", client, tableRef, map, deleteConsistency); } return null; } @@ -1448,7 +1464,7 @@ public ClosableIterator>> getRangeOfTimestamps( .addTimestampsToIgnore() .shouldCheckIfLatestValueIsEmpty(false) .build(); - return getCandidateRowsForSweeping(tableRef, request) + return getCandidateRowsForSweeping("getRangeOfTimestamps", tableRef, request) .flatMap(rows -> rows) .map(row -> row.toRowResult()) .stopWhen(rowResult -> !rangeRequest.inRange(rowResult.getRowName())); @@ -1458,7 +1474,7 @@ public ClosableIterator>> getRangeOfTimestamps( public ClosableIterator> getCandidateCellsForSweeping( TableReference tableRef, CandidateCellForSweepingRequest request) { - return getCandidateRowsForSweeping(tableRef, request) + return getCandidateRowsForSweeping("getCandidateCellsForSweeping", tableRef, request) .map(rows -> rows.stream() .map(CandidateRowForSweeping::cells) .flatMap(List::stream) @@ -1466,9 +1482,13 @@ public ClosableIterator> getCandidateCellsForSwee } private ClosableIterator> getCandidateRowsForSweeping( + String kvsMethodName, TableReference tableRef, CandidateCellForSweepingRequest request) { - return new CandidateRowsForSweepingIterator(this::get, new CqlExecutor(clientPool, ConsistencyLevel.ALL), + return new CandidateRowsForSweepingIterator( + (iteratorTableRef, cells, maxTimestampExclusive) -> + get(kvsMethodName, iteratorTableRef, cells, maxTimestampExclusive), + new CqlExecutor(clientPool, ConsistencyLevel.ALL), tableRef, request); } @@ -2040,7 +2060,8 @@ public void close() { public void addGarbageCollectionSentinelValues(TableReference tableRef, Iterable cells) { try { final Value value = Value.create(PtBytes.EMPTY_BYTE_ARRAY, Value.INVALID_VALUE_TIMESTAMP); - putInternal(tableRef, Iterables.transform(cells, cell -> Maps.immutableEntry(cell, value))); + putInternal("addGarbageCollectionSentinelValues", + tableRef, Iterables.transform(cells, cell -> Maps.immutableEntry(cell, value))); } catch (Exception e) { throw Throwables.unwrapAndThrowAtlasDbDependencyException(e); } @@ -2064,7 +2085,7 @@ public void addGarbageCollectionSentinelValues(TableReference tableRef, Iterable @Override public Multimap getAllTimestamps(TableReference tableRef, Set cells, long ts) { AllTimestampsCollector collector = new AllTimestampsCollector(); - loadWithTs(tableRef, cells, ts, true, collector, deleteConsistency); + loadWithTs("getAllTimestamps", tableRef, cells, ts, true, collector, deleteConsistency); return collector.getCollectedResults(); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/CassandraRangePagingIterable.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/CassandraRangePagingIterable.java index 4540d168251..a10b1d8c935 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/CassandraRangePagingIterable.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/CassandraRangePagingIterable.java @@ -91,7 +91,7 @@ private TokenBackedBasicResultsPage, byte[]> getSinglePage(byte[] s private List getRows(byte[] startKey) throws Exception { KeyRange keyRange = getKeyRange(startKey, rangeRequest.getEndExclusive()); - return rowGetter.getRows(keyRange, slicePredicate); + return rowGetter.getRows("getRange", keyRange, slicePredicate); } private Map> getColumns(List firstPage) { diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java index 4fe67978bd7..f27f853f1f0 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/paging/RowGetter.java @@ -49,7 +49,9 @@ public RowGetter( this.tableRef = tableRef; } - public List getRows(KeyRange keyRange, SlicePredicate slicePredicate) throws Exception { + public List getRows(String kvsMethodName, KeyRange keyRange, SlicePredicate slicePredicate) + throws Exception { + InetSocketAddress host = clientPool.getRandomHostForKey(keyRange.getStart_key()); return clientPool.runWithRetryOnHost( host, @@ -58,7 +60,11 @@ public List getRows(KeyRange keyRange, SlicePredicate slicePredicate) public List apply(CassandraClient client) throws Exception { try { return queryRunner.run(client, tableRef, - () -> client.get_range_slices(tableRef, slicePredicate, keyRange, consistency)); + () -> client.get_range_slices(kvsMethodName, + tableRef, + slicePredicate, + keyRange, + consistency)); } catch (UnavailableException e) { throw new InsufficientConsistencyException("get_range_slices requires " + consistency + " Cassandra nodes to be up and available.", e); From 247f60c1460468563df351e7875af12af83a4a82 Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Mon, 13 Nov 2017 20:56:49 +0000 Subject: [PATCH 18/67] Extract cql executor interface (#2670) * Instrument CqlExecutor * [no release notes] --- .../CassandraKeyValueServiceImpl.java | 7 +- .../keyvalue/cassandra/CqlExecutor.java | 226 +--------------- .../keyvalue/cassandra/CqlExecutorImpl.java | 247 ++++++++++++++++++ .../keyvalue/cassandra/CqlExecutorTest.java | 4 +- 4 files changed, 267 insertions(+), 217 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorImpl.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java index d5601e85216..b42056aeb6d 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java @@ -122,6 +122,7 @@ import com.palantir.atlasdb.logging.LoggingArgs; import com.palantir.atlasdb.util.AnnotatedCallable; import com.palantir.atlasdb.util.AnnotationType; +import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.common.annotation.Idempotent; import com.palantir.common.base.ClosableIterator; import com.palantir.common.base.ClosableIterators; @@ -1488,10 +1489,14 @@ private ClosableIterator> getCandidateRowsForSweep return new CandidateRowsForSweepingIterator( (iteratorTableRef, cells, maxTimestampExclusive) -> get(kvsMethodName, iteratorTableRef, cells, maxTimestampExclusive), - new CqlExecutor(clientPool, ConsistencyLevel.ALL), + newInstrumentedCqlExecutor(), tableRef, request); } + private CqlExecutor newInstrumentedCqlExecutor() { + return AtlasDbMetrics.instrument(CqlExecutor.class, new CqlExecutorImpl(clientPool, ConsistencyLevel.ALL)); + } + private ClosableIterator> getRangeWithPageCreator( TableReference tableRef, RangeRequest rangeRequest, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java index 587a957d47b..ff764085830 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutor.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Palantir Technologies, Inc. All rights reserved. + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. * * Licensed under the BSD-3 License (the "License"); * you may not use this file except in compliance with the License. @@ -12,234 +12,32 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package com.palantir.atlasdb.keyvalue.cassandra; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.cassandra.thrift.Compression; -import org.apache.cassandra.thrift.ConsistencyLevel; -import org.apache.cassandra.thrift.CqlResult; -import org.apache.cassandra.thrift.CqlRow; -import org.apache.cassandra.thrift.UnavailableException; -import org.apache.thrift.TException; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import com.palantir.atlasdb.encoding.PtBytes; -import com.palantir.atlasdb.keyvalue.api.Cell; -import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.cassandra.sweep.CellWithTimestamp; -import com.palantir.atlasdb.logging.KvsProfilingLogger; -import com.palantir.atlasdb.logging.LoggingArgs; -import com.palantir.common.base.FunctionCheckedException; -import com.palantir.common.base.Throwables; -import com.palantir.logsafe.Arg; -import com.palantir.logsafe.SafeArg; -import com.palantir.logsafe.UnsafeArg; - -public class CqlExecutor { - private QueryExecutor queryExecutor; - - public interface QueryExecutor { - CqlResult execute(byte[] rowHintForHostSelection, String query); - } - - CqlExecutor(CassandraClientPool clientPool, ConsistencyLevel consistency) { - this.queryExecutor = new QueryExecutorImpl(clientPool, consistency); - } - - @VisibleForTesting - CqlExecutor(QueryExecutor queryExecutor) { - this.queryExecutor = queryExecutor; - } - - /** - * Returns a list of {@link CellWithTimestamp}s within the given {@code row}, starting at the given - * {@code startRowInclusive}, potentially spanning across multiple rows. - */ - public List getTimestamps( - TableReference tableRef, - byte[] startRowInclusive, - int limit) { - CqlQuery query = new CqlQuery( - "SELECT key, column1, column2 FROM %s WHERE token(key) >= token(%s) LIMIT %s;", - quotedTableName(tableRef), - key(startRowInclusive), - limit(limit)); - return query.executeAndGetCells(startRowInclusive, this::getCellFromRow); - } +public interface CqlExecutor { /** * Returns a list of {@link CellWithTimestamp}s within the given {@code row}, starting at the (column, timestamp) * pair represented by ({@code startColumnInclusive}, {@code startTimestampExclusive}). */ - public List getTimestampsWithinRow( + List getTimestampsWithinRow( TableReference tableRef, byte[] row, byte[] startColumnInclusive, long startTimestampExclusive, - int limit) { - long invertedTimestamp = ~startTimestampExclusive; - CqlQuery query = new CqlQuery( - "SELECT column1, column2 FROM %s WHERE key = %s AND (column1, column2) > (%s, %s) LIMIT %s;", - quotedTableName(tableRef), - key(row), - column1(startColumnInclusive), - column2(invertedTimestamp), - limit(limit)); - return query.executeAndGetCells(row, result -> getCellFromKeylessRow(result, row)); - } - - private CellWithTimestamp getCellFromKeylessRow(CqlRow row, byte[] key) { - byte[] rowName = key; - byte[] columnName = row.getColumns().get(0).getValue(); - long timestamp = extractTimestamp(row, 1); - - return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); - } - - private CellWithTimestamp getCellFromRow(CqlRow row) { - byte[] rowName = row.getColumns().get(0).getValue(); - byte[] columnName = row.getColumns().get(1).getValue(); - long timestamp = extractTimestamp(row, 2); - - return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); - } - - private long extractTimestamp(CqlRow row, int columnIndex) { - byte[] flippedTimestampAsBytes = row.getColumns().get(columnIndex).getValue(); - return ~PtBytes.toLong(flippedTimestampAsBytes); - } - - private Arg key(byte[] row) { - return UnsafeArg.of("key", CassandraKeyValueServices.encodeAsHex(row)); - } - - private Arg column1(byte[] column) { - return UnsafeArg.of("column1", CassandraKeyValueServices.encodeAsHex(column)); - } - - private Arg column2(long invertedTimestamp) { - return SafeArg.of("column2", invertedTimestamp); - } - - private Arg limit(long limit) { - return SafeArg.of("limit", limit); - } - - private Arg quotedTableName(TableReference tableRef) { - String tableNameWithQuotes = "\"" + CassandraKeyValueServiceImpl.internalTableName(tableRef) + "\""; - return LoggingArgs.customTableName(tableRef, tableNameWithQuotes); - } - - private List getCells(Function cellTsExtractor, CqlResult cqlResult) { - return cqlResult.getRows() - .stream() - .map(cellTsExtractor) - .collect(Collectors.toList()); - } - - private final class CqlQuery { - private final String queryFormat; - private final Arg[] queryArgs; - - CqlQuery(String queryFormat, Arg... args) { - this.queryFormat = queryFormat; - this.queryArgs = args; - } - - public List executeAndGetCells( - byte[] rowHintForHostSelection, - Function cellTsExtractor) { - CqlResult cqlResult = KvsProfilingLogger.maybeLog( - () -> queryExecutor.execute(rowHintForHostSelection, toString()), - this::logSlowResult, - this::logResultSize); - return getCells(cellTsExtractor, cqlResult); - } - - private void logSlowResult(KvsProfilingLogger.LoggingFunction log, Stopwatch timer) { - Object[] allArgs = new Object[queryArgs.length + 3]; - allArgs[0] = SafeArg.of("queryFormat", queryFormat); - allArgs[1] = UnsafeArg.of("fullQuery", toString()); - allArgs[2] = LoggingArgs.durationMillis(timer); - System.arraycopy(queryArgs, 0, allArgs, 3, queryArgs.length); - - log.log("A CQL query was slow: queryFormat = [{}], fullQuery = [{}], durationMillis = {}", allArgs); - } - - private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { - log.log("and returned {} rows", - SafeArg.of("numRows", result.getRows().size())); - } - - @Override - public String toString() { - return String.format(queryFormat, (Object[]) queryArgs); - } - } - - private static class QueryExecutorImpl implements QueryExecutor { - private final CassandraClientPool clientPool; - private final ConsistencyLevel consistency; - - QueryExecutorImpl(CassandraClientPool clientPool, ConsistencyLevel consistency) { - this.clientPool = clientPool; - this.consistency = consistency; - } - - @Override - public CqlResult execute(byte[] rowHintForHostSelection, String query) { - return executeQueryOnHost(query, getHostForRow(rowHintForHostSelection)); - } - - private InetSocketAddress getHostForRow(byte[] row) { - return clientPool.getRandomHostForKey(row); - } - - private CqlResult executeQueryOnHost(String query, InetSocketAddress host) { - try { - return clientPool.runWithRetryOnHost(host, createCqlFunction(query)); - } catch (UnavailableException e) { - throw wrapIfConsistencyAll(e); - } catch (TException e) { - throw Throwables.throwUncheckedException(e); - } - } - - private RuntimeException wrapIfConsistencyAll(UnavailableException ex) { - if (consistency.equals(ConsistencyLevel.ALL)) { - throw new InsufficientConsistencyException("This operation requires all Cassandra" - + " nodes to be up and available.", ex); - } else { - throw Throwables.throwUncheckedException(ex); - } - } - - private FunctionCheckedException createCqlFunction(String query) { - ByteBuffer queryBytes = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); - - return new FunctionCheckedException() { - @Override - public CqlResult apply(CassandraClient client) throws TException { - return client.execute_cql3_query(queryBytes, Compression.NONE, consistency); - } - - @Override - public String toString() { - return query; - } - }; - } - } - + int limit); + /** + * Returns a list of {@link CellWithTimestamp}s within the given {@code row}, starting at the given + * {@code startRowInclusive}, potentially spanning across multiple rows. + */ + List getTimestamps( + TableReference tableRef, + byte[] startRowInclusive, + int limit); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorImpl.java new file mode 100644 index 00000000000..00f7b10a6a6 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorImpl.java @@ -0,0 +1,247 @@ +/* + * Copyright 2016 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlRow; +import org.apache.cassandra.thrift.UnavailableException; +import org.apache.thrift.TException; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.palantir.atlasdb.encoding.PtBytes; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.cassandra.sweep.CellWithTimestamp; +import com.palantir.atlasdb.logging.KvsProfilingLogger; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.common.base.FunctionCheckedException; +import com.palantir.common.base.Throwables; +import com.palantir.logsafe.Arg; +import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.UnsafeArg; + +public class CqlExecutorImpl implements CqlExecutor { + private QueryExecutor queryExecutor; + + public interface QueryExecutor { + CqlResult execute(byte[] rowHintForHostSelection, String query); + } + + CqlExecutorImpl(CassandraClientPool clientPool, ConsistencyLevel consistency) { + this.queryExecutor = new QueryExecutorImpl(clientPool, consistency); + } + + @VisibleForTesting + CqlExecutorImpl(QueryExecutor queryExecutor) { + this.queryExecutor = queryExecutor; + } + + /** + * Returns a list of {@link CellWithTimestamp}s within the given {@code row}, starting at the given + * {@code startRowInclusive}, potentially spanning across multiple rows. + */ + @Override + public List getTimestamps( + TableReference tableRef, + byte[] startRowInclusive, + int limit) { + CqlQuery query = new CqlQuery( + "SELECT key, column1, column2 FROM %s WHERE token(key) >= token(%s) LIMIT %s;", + quotedTableName(tableRef), + key(startRowInclusive), + limit(limit)); + return query.executeAndGetCells(startRowInclusive, this::getCellFromRow); + } + + /** + * Returns a list of {@link CellWithTimestamp}s within the given {@code row}, starting at the (column, timestamp) + * pair represented by ({@code startColumnInclusive}, {@code startTimestampExclusive}). + */ + @Override + public List getTimestampsWithinRow( + TableReference tableRef, + byte[] row, + byte[] startColumnInclusive, + long startTimestampExclusive, + int limit) { + long invertedTimestamp = ~startTimestampExclusive; + CqlQuery query = new CqlQuery( + "SELECT column1, column2 FROM %s WHERE key = %s AND (column1, column2) > (%s, %s) LIMIT %s;", + quotedTableName(tableRef), + key(row), + column1(startColumnInclusive), + column2(invertedTimestamp), + limit(limit)); + return query.executeAndGetCells(row, result -> getCellFromKeylessRow(result, row)); + } + + private CellWithTimestamp getCellFromKeylessRow(CqlRow row, byte[] key) { + byte[] rowName = key; + byte[] columnName = row.getColumns().get(0).getValue(); + long timestamp = extractTimestamp(row, 1); + + return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); + } + + private CellWithTimestamp getCellFromRow(CqlRow row) { + byte[] rowName = row.getColumns().get(0).getValue(); + byte[] columnName = row.getColumns().get(1).getValue(); + long timestamp = extractTimestamp(row, 2); + + return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); + } + + private long extractTimestamp(CqlRow row, int columnIndex) { + byte[] flippedTimestampAsBytes = row.getColumns().get(columnIndex).getValue(); + return ~PtBytes.toLong(flippedTimestampAsBytes); + } + + private Arg key(byte[] row) { + return UnsafeArg.of("key", CassandraKeyValueServices.encodeAsHex(row)); + } + + private Arg column1(byte[] column) { + return UnsafeArg.of("column1", CassandraKeyValueServices.encodeAsHex(column)); + } + + private Arg column2(long invertedTimestamp) { + return SafeArg.of("column2", invertedTimestamp); + } + + private Arg limit(long limit) { + return SafeArg.of("limit", limit); + } + + private Arg quotedTableName(TableReference tableRef) { + String tableNameWithQuotes = "\"" + CassandraKeyValueServiceImpl.internalTableName(tableRef) + "\""; + return LoggingArgs.customTableName(tableRef, tableNameWithQuotes); + } + + private List getCells(Function cellTsExtractor, CqlResult cqlResult) { + return cqlResult.getRows() + .stream() + .map(cellTsExtractor) + .collect(Collectors.toList()); + } + + private final class CqlQuery { + private final String queryFormat; + private final Arg[] queryArgs; + + CqlQuery(String queryFormat, Arg... args) { + this.queryFormat = queryFormat; + this.queryArgs = args; + } + + public List executeAndGetCells( + byte[] rowHintForHostSelection, + Function cellTsExtractor) { + CqlResult cqlResult = KvsProfilingLogger.maybeLog( + () -> queryExecutor.execute(rowHintForHostSelection, toString()), + this::logSlowResult, + this::logResultSize); + return getCells(cellTsExtractor, cqlResult); + } + + private void logSlowResult(KvsProfilingLogger.LoggingFunction log, Stopwatch timer) { + Object[] allArgs = new Object[queryArgs.length + 3]; + allArgs[0] = SafeArg.of("queryFormat", queryFormat); + allArgs[1] = UnsafeArg.of("fullQuery", toString()); + allArgs[2] = LoggingArgs.durationMillis(timer); + System.arraycopy(queryArgs, 0, allArgs, 3, queryArgs.length); + + log.log("A CQL query was slow: queryFormat = [{}], fullQuery = [{}], durationMillis = {}", allArgs); + } + + private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { + log.log("and returned {} rows", + SafeArg.of("numRows", result.getRows().size())); + } + + @Override + public String toString() { + return String.format(queryFormat, (Object[]) queryArgs); + } + } + + private static class QueryExecutorImpl implements QueryExecutor { + private final CassandraClientPool clientPool; + private final ConsistencyLevel consistency; + + QueryExecutorImpl(CassandraClientPool clientPool, ConsistencyLevel consistency) { + this.clientPool = clientPool; + this.consistency = consistency; + } + + @Override + public CqlResult execute(byte[] rowHintForHostSelection, String query) { + return executeQueryOnHost(query, getHostForRow(rowHintForHostSelection)); + } + + private InetSocketAddress getHostForRow(byte[] row) { + return clientPool.getRandomHostForKey(row); + } + + private CqlResult executeQueryOnHost(String query, InetSocketAddress host) { + try { + return clientPool.runWithRetryOnHost(host, createCqlFunction(query)); + } catch (UnavailableException e) { + throw wrapIfConsistencyAll(e); + } catch (TException e) { + throw Throwables.throwUncheckedException(e); + } + } + + private RuntimeException wrapIfConsistencyAll(UnavailableException ex) { + if (consistency.equals(ConsistencyLevel.ALL)) { + throw new InsufficientConsistencyException("This operation requires all Cassandra" + + " nodes to be up and available.", ex); + } else { + throw Throwables.throwUncheckedException(ex); + } + } + + private FunctionCheckedException createCqlFunction(String query) { + ByteBuffer queryBytes = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); + + return new FunctionCheckedException() { + @Override + public CqlResult apply(CassandraClient client) throws TException { + return client.execute_cql3_query(queryBytes, Compression.NONE, consistency); + } + + @Override + public String toString() { + return query; + } + }; + } + } + +} diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java index 86203f3acb8..d07a83ca4bc 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java @@ -34,8 +34,8 @@ public class CqlExecutorTest { - private final CqlExecutor.QueryExecutor queryExecutor = mock(CqlExecutor.QueryExecutor.class); - private final CqlExecutor executor = new CqlExecutor(queryExecutor); + private final CqlExecutorImpl.QueryExecutor queryExecutor = mock(CqlExecutorImpl.QueryExecutor.class); + private final CqlExecutor executor = new CqlExecutorImpl(queryExecutor); private long queryDelayMillis = 0L; From 913e5e7704d15cdf24bb2ef3bc4230f3079dd5b6 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Mon, 13 Nov 2017 22:40:21 +0000 Subject: [PATCH 19/67] bump awaitility (#2668) * Upgrade to newer Awaitility. * locks [no release notes] * unused import --- .../cassandra/CassandraTestTools.java | 4 +- .../multinode/NodesDownTestSetup.java | 2 +- atlasdb-cassandra/build.gradle | 1 + atlasdb-config/build.gradle | 3 +- .../factory/TransactionManagersTest.java | 4 +- .../factory/startup/TimeLockMigratorTest.java | 2 +- atlasdb-container-test-utils/build.gradle | 1 + .../atlasdb/containers/Containers.java | 4 +- atlasdb-container-test-utils/versions.lock | 20 +++++-- .../dbkvs/DbkvsPostgresTestSuite.java | 4 +- atlasdb-dbkvs-tests/versions.lock | 52 +++++++------------ .../com/palantir/atlasdb/ete/EteSetup.java | 4 +- .../atlasdb/ete/TimeLockMigrationEteTest.java | 4 +- atlasdb-perf/build.gradle | 1 + .../backend/DatabasesContainer.java | 5 +- atlasdb-perf/versions.lock | 20 +++++-- atlasdb-tests-shared/build.gradle | 2 +- .../atlasdb/sweep/SweepTestUtils.java | 8 +-- atlasdb-tests-shared/versions.lock | 36 +++++-------- .../lock/AwaitedLocksCollectionTest.java | 7 ++- .../PaxosTimeLockServerIntegrationTest.java | 2 +- .../timelock/TestableTimelockCluster.java | 2 +- timestamp-impl/build.gradle | 2 +- versions.props | 2 +- 24 files changed, 97 insertions(+), 95 deletions(-) diff --git a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTestTools.java b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTestTools.java index 12ef70c60a7..bfccf445867 100644 --- a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTestTools.java +++ b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTestTools.java @@ -29,10 +29,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.joda.time.Duration; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.core.ConditionTimeoutException; import com.palantir.common.base.Throwables; /** diff --git a/atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/NodesDownTestSetup.java b/atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/NodesDownTestSetup.java index 44c33b15965..e74e13568ae 100644 --- a/atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/NodesDownTestSetup.java +++ b/atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/NodesDownTestSetup.java @@ -19,12 +19,12 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.awaitility.Awaitility; import org.junit.AfterClass; import org.junit.ClassRule; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; -import com.jayway.awaitility.Awaitility; import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfigManager; diff --git a/atlasdb-cassandra/build.gradle b/atlasdb-cassandra/build.gradle index 07e405d85c1..1276db7237e 100644 --- a/atlasdb-cassandra/build.gradle +++ b/atlasdb-cassandra/build.gradle @@ -51,6 +51,7 @@ dependencies { testCompile group: 'org.mockito', name: 'mockito-core' testCompile group: 'org.assertj', name: 'assertj-core' testCompile group: 'org.hamcrest', name: 'hamcrest-library' + testCompile group: 'org.awaitility', name: 'awaitility' processor group: 'org.immutables', name: 'value' processor 'com.google.auto.service:auto-service:1.0-rc2' diff --git a/atlasdb-config/build.gradle b/atlasdb-config/build.gradle index 1d9740c72fa..dd66e0ae39d 100644 --- a/atlasdb-config/build.gradle +++ b/atlasdb-config/build.gradle @@ -41,8 +41,7 @@ dependencies { testCompile group: 'org.mockito', name: 'mockito-core' testCompile group: 'com.github.tomakehurst', name: 'wiremock' testCompile group: 'org.assertj', name: 'assertj-core' - testCompile group: 'com.jayway.awaitility', name: 'awaitility' - + testCompile group: 'org.awaitility', name: 'awaitility' // Needed for Jersey Response-based tests testCompile group: 'org.glassfish.jersey.core', name: 'jersey-common' } diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java index d01c39fa6f2..1e4fd09115b 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java @@ -44,6 +44,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -61,7 +62,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; -import com.jayway.awaitility.Awaitility; import com.palantir.atlasdb.config.AtlasDbConfig; import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; @@ -163,7 +163,7 @@ public class TransactionManagersTest { public void setup() throws JsonProcessingException { // Change code to run synchronously, but with a timeout in case something's gone horribly wrong originalAsyncMethod = TransactionManagers.runAsync; - TransactionManagers.runAsync = task -> Awaitility.await().atMost(2, TimeUnit.SECONDS).until(task); + TransactionManagers.runAsync = task -> Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(task::run); availableServer.stubFor(LEADER_UUID_MAPPING.willReturn(aResponse().withStatus(200).withBody( ("\"" + UUID.randomUUID().toString() + "\"").getBytes()))); diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java index ca387aeccdf..7dd4055a2e9 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/startup/TimeLockMigratorTest.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.awaitility.Awaitility; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -42,7 +43,6 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.github.tomakehurst.wiremock.stubbing.Scenario; -import com.jayway.awaitility.Awaitility; import com.palantir.atlasdb.config.ImmutableServerListConfig; import com.palantir.atlasdb.config.ImmutableTimeLockClientConfig; import com.palantir.atlasdb.config.ServerListConfig; diff --git a/atlasdb-container-test-utils/build.gradle b/atlasdb-container-test-utils/build.gradle index faa0f3b63f9..230701fc612 100644 --- a/atlasdb-container-test-utils/build.gradle +++ b/atlasdb-container-test-utils/build.gradle @@ -10,6 +10,7 @@ dependencies { compile group: 'com.google.guava', name: 'guava' compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-core' compile group: 'com.palantir.docker.proxy', name: 'docker-proxy-rule' + compile group: 'org.awaitility', name: 'awaitility' processor group: 'org.immutables', name: 'value' diff --git a/atlasdb-container-test-utils/src/main/java/com/palantir/atlasdb/containers/Containers.java b/atlasdb-container-test-utils/src/main/java/com/palantir/atlasdb/containers/Containers.java index e8cff803308..79c46bbc056 100644 --- a/atlasdb-container-test-utils/src/main/java/com/palantir/atlasdb/containers/Containers.java +++ b/atlasdb-container-test-utils/src/main/java/com/palantir/atlasdb/containers/Containers.java @@ -24,6 +24,8 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.rules.ExternalResource; import com.google.common.annotations.VisibleForTesting; @@ -33,8 +35,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; import com.palantir.docker.compose.DockerComposeRule; import com.palantir.docker.compose.configuration.DockerComposeFiles; import com.palantir.docker.compose.configuration.ProjectName; diff --git a/atlasdb-container-test-utils/versions.lock b/atlasdb-container-test-utils/versions.lock index 8185e82cc65..f7899b03eef 100644 --- a/atlasdb-container-test-utils/versions.lock +++ b/atlasdb-container-test-utils/versions.lock @@ -537,11 +537,15 @@ "org.apache.cassandra:cassandra-thrift" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library" ] }, @@ -549,7 +553,8 @@ "locked": "1.3", "transitive": [ "com.github.almondtools:conmatch", - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -573,7 +578,8 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.slf4j:jcl-over-slf4j": { @@ -1151,11 +1157,15 @@ "org.apache.cassandra:cassandra-thrift" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library" ] }, @@ -1163,7 +1173,8 @@ "locked": "1.3", "transitive": [ "com.github.almondtools:conmatch", - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -1187,7 +1198,8 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.slf4j:jcl-over-slf4j": { diff --git a/atlasdb-dbkvs-tests/src/test/java/com/palantir/atlasdb/keyvalue/dbkvs/DbkvsPostgresTestSuite.java b/atlasdb-dbkvs-tests/src/test/java/com/palantir/atlasdb/keyvalue/dbkvs/DbkvsPostgresTestSuite.java index 170ed5c417c..9ebb1955632 100644 --- a/atlasdb-dbkvs-tests/src/test/java/com/palantir/atlasdb/keyvalue/dbkvs/DbkvsPostgresTestSuite.java +++ b/atlasdb-dbkvs-tests/src/test/java/com/palantir/atlasdb/keyvalue/dbkvs/DbkvsPostgresTestSuite.java @@ -18,14 +18,14 @@ import java.net.InetSocketAddress; import java.util.concurrent.Callable; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.junit.runners.Suite.SuiteClasses; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; import com.palantir.atlasdb.keyvalue.dbkvs.impl.ConnectionManagerAwareDbKvs; import com.palantir.atlasdb.keyvalue.dbkvs.impl.postgres.DbKvsPostgresGetCandidateCellsForSweepingTest; import com.palantir.docker.compose.DockerComposeRule; diff --git a/atlasdb-dbkvs-tests/versions.lock b/atlasdb-dbkvs-tests/versions.lock index e93fc7897a8..6ccc7ef6622 100644 --- a/atlasdb-dbkvs-tests/versions.lock +++ b/atlasdb-dbkvs-tests/versions.lock @@ -1,11 +1,5 @@ { "compileClasspath": { - "cglib:cglib-nodep": { - "locked": "3.1", - "transitive": [ - "com.jayway.awaitility:awaitility" - ] - }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ @@ -194,12 +188,6 @@ "com.palantir.atlasdb:atlasdb-client" ] }, - "com.jayway.awaitility:awaitility": { - "locked": "1.6.5", - "transitive": [ - "com.palantir.atlasdb:atlasdb-tests-shared" - ] - }, "com.jcraft:jzlib": { "locked": "1.1.3", "transitive": [ @@ -629,12 +617,18 @@ "com.palantir.atlasdb:atlasdb-tests-shared" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-tests-shared" + ] + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", "com.palantir.atlasdb:atlasdb-tests-shared", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library", "org.mockito:mockito-core" ] @@ -642,8 +636,8 @@ "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", - "com.palantir.atlasdb:atlasdb-tests-shared" + "com.palantir.atlasdb:atlasdb-tests-shared", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -669,7 +663,7 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility", "org.mockito:mockito-core" ] }, @@ -711,12 +705,6 @@ } }, "runtime": { - "cglib:cglib-nodep": { - "locked": "3.1", - "transitive": [ - "com.jayway.awaitility:awaitility" - ] - }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ @@ -905,12 +893,6 @@ "com.palantir.atlasdb:atlasdb-client" ] }, - "com.jayway.awaitility:awaitility": { - "locked": "1.6.5", - "transitive": [ - "com.palantir.atlasdb:atlasdb-tests-shared" - ] - }, "com.jcraft:jzlib": { "locked": "1.1.3", "transitive": [ @@ -1340,12 +1322,18 @@ "com.palantir.atlasdb:atlasdb-tests-shared" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-tests-shared" + ] + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", "com.palantir.atlasdb:atlasdb-tests-shared", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library", "org.mockito:mockito-core" ] @@ -1353,8 +1341,8 @@ "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", - "com.palantir.atlasdb:atlasdb-tests-shared" + "com.palantir.atlasdb:atlasdb-tests-shared", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -1380,7 +1368,7 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility", "org.mockito:mockito-core" ] }, diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/EteSetup.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/EteSetup.java index 54afd930162..425f24a853b 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/EteSetup.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/EteSetup.java @@ -25,14 +25,14 @@ import javax.net.ssl.SSLSocketFactory; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.rules.ExternalResource; import org.junit.rules.RuleChain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; import com.palantir.atlasdb.http.AtlasDbHttpClients; import com.palantir.atlasdb.todo.TodoResource; import com.palantir.docker.compose.DockerComposeRule; diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/TimeLockMigrationEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/TimeLockMigrationEteTest.java index 68c935d1246..14ed0011bf5 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/TimeLockMigrationEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/TimeLockMigrationEteTest.java @@ -22,6 +22,8 @@ import java.util.concurrent.Callable; import org.assertj.core.api.JUnitSoftAssertions; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -29,8 +31,6 @@ import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; import com.palantir.atlasdb.http.AtlasDbHttpClients; import com.palantir.atlasdb.http.errors.AtlasDbRemoteException; import com.palantir.atlasdb.todo.ImmutableTodo; diff --git a/atlasdb-perf/build.gradle b/atlasdb-perf/build.gradle index eb8e91781e2..3e865017f26 100644 --- a/atlasdb-perf/build.gradle +++ b/atlasdb-perf/build.gradle @@ -17,6 +17,7 @@ dependencies { compile group: 'org.reflections', name: 'reflections', version: '0.9.10' compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-junit4' compile group: 'ch.qos.logback', name: 'logback-classic' + compile group: 'org.awaitility', name: 'awaitility' testCompile group: 'org.assertj', name: 'assertj-core' testCompile group: 'org.mockito', name: 'mockito-core' diff --git a/atlasdb-perf/src/main/java/com/palantir/atlasdb/performance/backend/DatabasesContainer.java b/atlasdb-perf/src/main/java/com/palantir/atlasdb/performance/backend/DatabasesContainer.java index 68d0976dc57..e405373de1e 100644 --- a/atlasdb-perf/src/main/java/com/palantir/atlasdb/performance/backend/DatabasesContainer.java +++ b/atlasdb-perf/src/main/java/com/palantir/atlasdb/performance/backend/DatabasesContainer.java @@ -17,9 +17,10 @@ import java.util.List; +import org.awaitility.Awaitility; +import org.awaitility.Duration; + import com.google.common.collect.Lists; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; public final class DatabasesContainer implements AutoCloseable { diff --git a/atlasdb-perf/versions.lock b/atlasdb-perf/versions.lock index 6b7e6cf667a..4871de77d5e 100644 --- a/atlasdb-perf/versions.lock +++ b/atlasdb-perf/versions.lock @@ -954,18 +954,23 @@ "org.apache.cassandra:cassandra-thrift" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library" ] }, "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -1003,7 +1008,8 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.openjdk.jmh:jmh-core": { @@ -2027,18 +2033,23 @@ "org.apache.cassandra:cassandra-thrift" ] }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library" ] }, "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -2076,7 +2087,8 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility" + "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility" ] }, "org.openjdk.jmh:jmh-core": { diff --git a/atlasdb-tests-shared/build.gradle b/atlasdb-tests-shared/build.gradle index f9ebd35ee29..30cd457cf4d 100644 --- a/atlasdb-tests-shared/build.gradle +++ b/atlasdb-tests-shared/build.gradle @@ -20,7 +20,7 @@ dependencies { compile group: 'org.hamcrest', name: 'hamcrest-library' compile group: 'org.assertj', name: 'assertj-core' compile group: 'org.mockito', name: 'mockito-core' - compile group: "com.jayway.awaitility", name: "awaitility" + compile group: "org.awaitility", name: "awaitility" testCompile project(path: ":atlasdb-client", configuration: "testArtifacts") testCompile group: "org.jmock", name: "jmock", version: libVersions.jmock diff --git a/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/sweep/SweepTestUtils.java b/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/sweep/SweepTestUtils.java index ebc76a9c70a..fd06fbd41e6 100644 --- a/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/sweep/SweepTestUtils.java +++ b/atlasdb-tests-shared/src/main/java/com/palantir/atlasdb/sweep/SweepTestUtils.java @@ -15,9 +15,10 @@ */ package com.palantir.atlasdb.sweep; +import org.awaitility.Awaitility; +import org.awaitility.Duration; + import com.google.common.base.Supplier; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.cleaner.Cleaner; import com.palantir.atlasdb.cleaner.NoOpCleaner; @@ -94,8 +95,7 @@ private static void tearDownTables(KeyValueService kvs) { Awaitility.await() .timeout(Duration.FIVE_MINUTES) .until(() -> { - kvs.getAllTableNames().stream() - .forEach(tableRef -> kvs.dropTable(tableRef)); + kvs.getAllTableNames().forEach(kvs::dropTable); return true; }); TransactionTables.deleteTables(kvs); diff --git a/atlasdb-tests-shared/versions.lock b/atlasdb-tests-shared/versions.lock index 668499a09e3..47f06d62a2c 100644 --- a/atlasdb-tests-shared/versions.lock +++ b/atlasdb-tests-shared/versions.lock @@ -1,11 +1,5 @@ { "compileClasspath": { - "cglib:cglib-nodep": { - "locked": "3.1", - "transitive": [ - "com.jayway.awaitility:awaitility" - ] - }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ @@ -185,9 +179,6 @@ "com.palantir.atlasdb:atlasdb-client" ] }, - "com.jayway.awaitility:awaitility": { - "locked": "1.6.5" - }, "com.jcraft:jzlib": { "locked": "1.1.3", "transitive": [ @@ -508,11 +499,14 @@ "org.assertj:assertj-core": { "locked": "3.5.2" }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library", "org.mockito:mockito-core" ] @@ -520,7 +514,7 @@ "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility" + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -543,7 +537,7 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility", "org.mockito:mockito-core" ] }, @@ -577,12 +571,6 @@ } }, "runtime": { - "cglib:cglib-nodep": { - "locked": "3.1", - "transitive": [ - "com.jayway.awaitility:awaitility" - ] - }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ @@ -762,9 +750,6 @@ "com.palantir.atlasdb:atlasdb-client" ] }, - "com.jayway.awaitility:awaitility": { - "locked": "1.6.5" - }, "com.jcraft:jzlib": { "locked": "1.1.3", "transitive": [ @@ -1085,11 +1070,14 @@ "org.assertj:assertj-core": { "locked": "3.5.2" }, + "org.awaitility:awaitility": { + "locked": "3.0.0" + }, "org.hamcrest:hamcrest-core": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility", "junit:junit", + "org.awaitility:awaitility", "org.hamcrest:hamcrest-library", "org.mockito:mockito-core" ] @@ -1097,7 +1085,7 @@ "org.hamcrest:hamcrest-library": { "locked": "1.3", "transitive": [ - "com.jayway.awaitility:awaitility" + "org.awaitility:awaitility" ] }, "org.hdrhistogram:HdrHistogram": { @@ -1120,7 +1108,7 @@ "org.objenesis:objenesis": { "locked": "2.2", "transitive": [ - "com.jayway.awaitility:awaitility", + "org.awaitility:awaitility", "org.mockito:mockito-core" ] }, diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/lock/AwaitedLocksCollectionTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/lock/AwaitedLocksCollectionTest.java index 66577af8ad5..7e029b9cd4c 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/lock/AwaitedLocksCollectionTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/lock/AwaitedLocksCollectionTest.java @@ -29,11 +29,10 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.Test; -import com.jayway.awaitility.Awaitility; -import com.jayway.awaitility.Duration; - public class AwaitedLocksCollectionTest { private static final UUID REQUEST_1 = UUID.randomUUID(); @@ -107,7 +106,7 @@ private void assertRequestsWereRemoved(UUID... requests) { Awaitility.await() .atMost(Duration.ONE_SECOND) .pollInterval(5, TimeUnit.MILLISECONDS) - .until(() -> { + .untilAsserted(() -> { for (UUID requestId : Arrays.asList(requests)) { assertThat(awaitedLocks.requestsById).doesNotContainKey(requestId); } diff --git a/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/PaxosTimeLockServerIntegrationTest.java b/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/PaxosTimeLockServerIntegrationTest.java index 8a18609f8a0..19864f7122b 100644 --- a/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/PaxosTimeLockServerIntegrationTest.java +++ b/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/PaxosTimeLockServerIntegrationTest.java @@ -40,6 +40,7 @@ import javax.ws.rs.BadRequestException; import org.assertj.core.util.Lists; +import org.awaitility.Awaitility; import org.eclipse.jetty.http.HttpStatus; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -52,7 +53,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; -import com.jayway.awaitility.Awaitility; import com.palantir.atlasdb.http.AtlasDbHttpClients; import com.palantir.atlasdb.http.FeignOkHttpClients; import com.palantir.atlasdb.http.errors.AtlasDbRemoteException; diff --git a/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/TestableTimelockCluster.java b/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/TestableTimelockCluster.java index 9070061b0cd..be27bd5770f 100644 --- a/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/TestableTimelockCluster.java +++ b/timelock-server/src/integTest/java/com/palantir/atlasdb/timelock/TestableTimelockCluster.java @@ -26,13 +26,13 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.awaitility.Awaitility; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Uninterruptibles; -import com.jayway.awaitility.Awaitility; import com.palantir.atlasdb.timelock.util.TestProxies; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockService; diff --git a/timestamp-impl/build.gradle b/timestamp-impl/build.gradle index f632fb3719f..a3e873b0bff 100644 --- a/timestamp-impl/build.gradle +++ b/timestamp-impl/build.gradle @@ -17,6 +17,6 @@ dependencies { testCompile project(":atlasdb-tests-shared") testCompile group: 'junit', name: 'junit' testCompile group: 'org.jmock', name: 'jmock', version: libVersions.jmock - testCompile group: 'com.jayway.awaitility', name: 'awaitility' + testCompile group: 'org.awaitility', name: 'awaitility' testCompile group: 'org.mockito', name: 'mockito-core' } diff --git a/versions.props b/versions.props index d3add193d0c..5f6da12fb31 100644 --- a/versions.props +++ b/versions.props @@ -9,7 +9,7 @@ com.google.dagger:dagger = 2.0.2 com.google.dagger:dagger-compiler = 2.0.2 com.google.guava:* = 18.0 com.googlecode.json-simple:json-simple = 1.1.1 -com.jayway.awaitility:awaitility = 1.6.5 +org.awaitility:awaitility = 3.0.0 com.palantir.common:streams = 1.9.0 com.palantir.config.crypto:encrypted-config-value-module = 1.0.0 com.palantir.docker.compose:docker-compose-rule* = 0.31.0 From cafc479643d8919953b928274fa60a41e529c0a3 Mon Sep 17 00:00:00 2001 From: datval Date: Tue, 14 Nov 2017 02:14:45 -0800 Subject: [PATCH 20/67] Bump Atlas on Tritium 0.8.4 to fix dependency conflicts (#2662) * Bump Atlas on Tritium 0.8.4 to fix dependency conflicts * Add changes into missing file * Doc changes * Exclude Tracing and HdrHistogram from Tritium dependencies * update locks * Add excluded dependencies explicitly * Fix merge conflict in relase notes * Uncomment dependencies * Regenerate locks --- atlasdb-cassandra/versions.lock | 40 ++++++++--------- atlasdb-cli-distribution/versions.lock | 24 +++++----- atlasdb-cli/versions.lock | 48 +++++++++----------- atlasdb-client/build.gradle | 3 +- atlasdb-client/versions.lock | 44 ++++++++---------- atlasdb-config/build.gradle | 4 ++ atlasdb-config/versions.lock | 48 +++++++++----------- atlasdb-console-distribution/versions.lock | 24 +++++----- atlasdb-console/versions.lock | 48 +++++++++----------- atlasdb-container-test-utils/versions.lock | 40 ++++++++--------- atlasdb-dagger/versions.lock | 48 +++++++++----------- atlasdb-dbkvs-tests/versions.lock | 46 +++++++++---------- atlasdb-dbkvs/versions.lock | 40 ++++++++--------- atlasdb-dropwizard-bundle/versions.lock | 48 +++++++++----------- atlasdb-ete-tests/versions.lock | 50 ++++++++++----------- atlasdb-hikari/versions.lock | 40 ++++++++--------- atlasdb-impl-shared/versions.lock | 40 ++++++++--------- atlasdb-jdbc/versions.lock | 40 ++++++++--------- atlasdb-jepsen-tests/versions.lock | 48 +++++++++----------- atlasdb-perf/versions.lock | 48 +++++++++----------- atlasdb-service-server/versions.lock | 48 +++++++++----------- atlasdb-service/versions.lock | 48 +++++++++----------- atlasdb-tests-shared/versions.lock | 46 +++++++++---------- docs/source/release_notes/release-notes.rst | 4 ++ examples/profile-client/versions.lock | 40 ++++++++--------- timelock-agent/versions.lock | 48 +++++++++----------- timelock-impl/versions.lock | 48 +++++++++----------- timelock-server-distribution/versions.lock | 25 +++++------ timelock-server/versions.lock | 50 ++++++++++----------- timestamp-impl/versions.lock | 40 ++++++++--------- versions.props | 2 +- 31 files changed, 535 insertions(+), 635 deletions(-) diff --git a/atlasdb-cassandra/versions.lock b/atlasdb-cassandra/versions.lock index a55073e8cae..53fb42b4de2 100644 --- a/atlasdb-cassandra/versions.lock +++ b/atlasdb-cassandra/versions.lock @@ -110,8 +110,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -245,8 +244,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -265,7 +263,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -275,7 +273,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -284,31 +282,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -591,8 +589,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -726,8 +723,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -746,7 +742,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -756,7 +752,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -765,31 +761,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-cli-distribution/versions.lock b/atlasdb-cli-distribution/versions.lock index 744fa29b876..f8f397de5ad 100644 --- a/atlasdb-cli-distribution/versions.lock +++ b/atlasdb-cli-distribution/versions.lock @@ -240,8 +240,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -636,11 +635,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -673,7 +672,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -683,7 +682,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -692,32 +691,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -931,8 +930,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.jboss.marshalling:jboss-marshalling": { diff --git a/atlasdb-cli/versions.lock b/atlasdb-cli/versions.lock index ed2e30591dc..c36e036d9ce 100644 --- a/atlasdb-cli/versions.lock +++ b/atlasdb-cli/versions.lock @@ -223,8 +223,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.dagger:dagger": { @@ -553,11 +552,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -590,7 +589,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -600,7 +599,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -609,32 +608,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -824,8 +823,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.jboss.marshalling:jboss-marshalling": { @@ -1117,8 +1115,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1444,11 +1441,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1481,7 +1478,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1491,7 +1488,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1500,32 +1497,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1714,8 +1711,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.jboss.marshalling:jboss-marshalling": { diff --git a/atlasdb-client/build.gradle b/atlasdb-client/build.gradle index def2978f015..0f92316cb71 100644 --- a/atlasdb-client/build.gradle +++ b/atlasdb-client/build.gradle @@ -61,11 +61,12 @@ dependencies { compile group: 'com.palantir.remoting3', name: 'tracing' compile group: 'com.palantir.safe-logging', name: 'safe-logging' compile (group: 'com.palantir.tritium', name: 'tritium-lib') { + exclude (group: 'com.palantir.remoting3', module: 'tracing') exclude (group: 'io.dropwizard.metrics', module: 'metrics-core') exclude (group: 'org.hdrhistogram', module: 'HdrHistogram') } - compile group: 'org.hdrhistogram', name:'HdrHistogram' compile group: 'com.squareup', name:'javapoet', version: '1.9.0' + compile group: 'org.hdrhistogram', name: 'HdrHistogram' processor group: 'org.immutables', name: 'value' diff --git a/atlasdb-client/versions.lock b/atlasdb-client/versions.lock index c1d72b42656..1203eb404f6 100644 --- a/atlasdb-client/versions.lock +++ b/atlasdb-client/versions.lock @@ -86,8 +86,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -158,10 +157,7 @@ ] }, "com.palantir.remoting3:tracing": { - "locked": "3.5.1", - "transitive": [ - "com.palantir.tritium:tritium-tracing" - ] + "locked": "3.5.1" }, "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", @@ -177,7 +173,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -187,7 +183,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -196,28 +192,28 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3" + "locked": "0.8.4" }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -376,8 +372,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -448,10 +443,7 @@ ] }, "com.palantir.remoting3:tracing": { - "locked": "3.5.1", - "transitive": [ - "com.palantir.tritium:tritium-tracing" - ] + "locked": "3.5.1" }, "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", @@ -467,7 +459,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -477,7 +469,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -486,28 +478,28 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3" + "locked": "0.8.4" }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-config/build.gradle b/atlasdb-config/build.gradle index dd66e0ae39d..a5b2729678f 100644 --- a/atlasdb-config/build.gradle +++ b/atlasdb-config/build.gradle @@ -18,8 +18,12 @@ dependencies { compile group: 'com.palantir.remoting3', name: 'keystores' compile group: 'com.palantir.safe-logging', name: 'safe-logging' compile (group: 'com.palantir.tritium', name: 'tritium-lib') { + exclude (group: 'com.palantir.remoting3', module: 'tracing') exclude (group: 'io.dropwizard.metrics', module: 'metrics-core') + exclude (group: 'org.hdrhistogram', module: 'HdrHistogram') } + compile group: 'com.palantir.remoting3', name: 'tracing' + compile group: 'org.hdrhistogram', name: 'HdrHistogram' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind' compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml' diff --git a/atlasdb-config/versions.lock b/atlasdb-config/versions.lock index 53ac2e59f43..055e5f36580 100644 --- a/atlasdb-config/versions.lock +++ b/atlasdb-config/versions.lock @@ -192,8 +192,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -435,8 +434,7 @@ "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -466,7 +464,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -476,7 +474,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -485,31 +483,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -605,9 +603,7 @@ "org.hdrhistogram:HdrHistogram": { "locked": "2.1.10", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-client" ] }, "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir": { @@ -840,8 +836,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1083,8 +1078,7 @@ "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1114,7 +1108,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1124,7 +1118,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1133,31 +1127,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1253,9 +1247,7 @@ "org.hdrhistogram:HdrHistogram": { "locked": "2.1.10", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-client" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/atlasdb-console-distribution/versions.lock b/atlasdb-console-distribution/versions.lock index 6f36e4794a1..f6321b033fc 100644 --- a/atlasdb-console-distribution/versions.lock +++ b/atlasdb-console-distribution/versions.lock @@ -238,8 +238,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -618,11 +617,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -655,7 +654,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -665,7 +664,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -674,32 +673,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -929,8 +928,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.jboss.marshalling:jboss-marshalling": { diff --git a/atlasdb-console/versions.lock b/atlasdb-console/versions.lock index 1a95fe81a62..e5657eff936 100644 --- a/atlasdb-console/versions.lock +++ b/atlasdb-console/versions.lock @@ -199,8 +199,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -472,11 +471,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -507,7 +506,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -517,7 +516,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -526,32 +525,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -672,8 +671,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -919,8 +917,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1192,11 +1189,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1227,7 +1224,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1237,7 +1234,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1246,32 +1243,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1392,8 +1389,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/atlasdb-container-test-utils/versions.lock b/atlasdb-container-test-utils/versions.lock index f7899b03eef..be5da6f3710 100644 --- a/atlasdb-container-test-utils/versions.lock +++ b/atlasdb-container-test-utils/versions.lock @@ -135,8 +135,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -314,8 +313,7 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -335,7 +333,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -345,7 +343,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -354,31 +352,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -755,8 +753,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -934,8 +931,7 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -955,7 +951,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -965,7 +961,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -974,31 +970,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-dagger/versions.lock b/atlasdb-dagger/versions.lock index b086407177d..6d9cd8c35b6 100644 --- a/atlasdb-dagger/versions.lock +++ b/atlasdb-dagger/versions.lock @@ -199,8 +199,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.dagger:dagger": { @@ -475,11 +474,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -510,7 +509,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -520,7 +519,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -529,32 +528,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -662,8 +661,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -909,8 +907,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.dagger:dagger": { @@ -1185,11 +1182,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1220,7 +1217,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1230,7 +1227,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1239,32 +1236,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1372,8 +1369,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/atlasdb-dbkvs-tests/versions.lock b/atlasdb-dbkvs-tests/versions.lock index 6ccc7ef6622..b4814ffea18 100644 --- a/atlasdb-dbkvs-tests/versions.lock +++ b/atlasdb-dbkvs-tests/versions.lock @@ -150,8 +150,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -460,7 +459,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -470,7 +469,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -479,32 +478,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-tests-shared" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -556,8 +555,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:atlasdb-dbkvs-hikari", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "javax.validation:validation-api": { @@ -644,8 +642,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "org.mockito:mockito-core": { @@ -855,8 +852,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -1165,7 +1161,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1175,7 +1171,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1184,32 +1180,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-tests-shared" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1261,8 +1257,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:atlasdb-dbkvs-hikari", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "javax.validation:validation-api": { @@ -1349,8 +1344,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "org.mockito:mockito-core": { diff --git a/atlasdb-dbkvs/versions.lock b/atlasdb-dbkvs/versions.lock index 91ec9d83257..187d95d3e42 100644 --- a/atlasdb-dbkvs/versions.lock +++ b/atlasdb-dbkvs/versions.lock @@ -145,8 +145,7 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting-api:errors" ] }, "com.google.guava:guava": { @@ -382,8 +381,7 @@ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -407,7 +405,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -417,7 +415,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -426,31 +424,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -730,8 +728,7 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting-api:errors" ] }, "com.google.guava:guava": { @@ -967,8 +964,7 @@ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -992,7 +988,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1002,7 +998,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1011,31 +1007,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-dropwizard-bundle/versions.lock b/atlasdb-dropwizard-bundle/versions.lock index c49ac9257ff..6db4d847b00 100644 --- a/atlasdb-dropwizard-bundle/versions.lock +++ b/atlasdb-dropwizard-bundle/versions.lock @@ -243,8 +243,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -584,11 +583,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -621,7 +620,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -631,7 +630,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -640,32 +639,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1237,8 +1236,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.hibernate:hibernate-validator": { @@ -1582,8 +1580,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1923,11 +1920,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1960,7 +1957,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1970,7 +1967,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1979,32 +1976,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -2576,8 +2573,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.hibernate:hibernate-validator": { diff --git a/atlasdb-ete-tests/versions.lock b/atlasdb-ete-tests/versions.lock index e84fdf95984..a66c86b4137 100644 --- a/atlasdb-ete-tests/versions.lock +++ b/atlasdb-ete-tests/versions.lock @@ -246,8 +246,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -609,6 +608,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", @@ -646,7 +646,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -656,7 +656,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -665,32 +665,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -790,8 +790,7 @@ "io.dropwizard.metrics:metrics-servlets", "io.dropwizard:dropwizard-core", "io.dropwizard:dropwizard-metrics", - "io.dropwizard:dropwizard-servlets", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-servlets" ] }, "io.dropwizard.metrics:metrics-healthchecks": { @@ -1289,8 +1288,8 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.tritium:tritium-metrics" ] }, "org.hibernate:hibernate-validator": { @@ -1653,8 +1652,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -2064,6 +2062,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", @@ -2101,7 +2100,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -2111,7 +2110,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -2120,32 +2119,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -2254,8 +2253,7 @@ "io.dropwizard.metrics:metrics-servlets", "io.dropwizard:dropwizard-core", "io.dropwizard:dropwizard-metrics", - "io.dropwizard:dropwizard-servlets", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-servlets" ] }, "io.dropwizard.metrics:metrics-healthchecks": { @@ -2761,8 +2759,8 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.tritium:tritium-metrics" ] }, "org.hibernate:hibernate-validator": { diff --git a/atlasdb-hikari/versions.lock b/atlasdb-hikari/versions.lock index 699c49fc44b..3e5db2a23bc 100644 --- a/atlasdb-hikari/versions.lock +++ b/atlasdb-hikari/versions.lock @@ -95,8 +95,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -208,8 +207,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -228,7 +226,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -238,7 +236,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -247,31 +245,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -459,8 +457,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -572,8 +569,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -592,7 +588,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -602,7 +598,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -611,31 +607,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-impl-shared/versions.lock b/atlasdb-impl-shared/versions.lock index 14d04bee622..65045acc4bf 100644 --- a/atlasdb-impl-shared/versions.lock +++ b/atlasdb-impl-shared/versions.lock @@ -128,8 +128,7 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting-api:errors" ] }, "com.google.guava:guava": { @@ -278,8 +277,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -301,7 +299,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -311,7 +309,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -320,31 +318,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -571,8 +569,7 @@ "locked": "1.3.9", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting-api:errors" ] }, "com.google.guava:guava": { @@ -721,8 +718,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -744,7 +740,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -754,7 +750,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -763,31 +759,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-jdbc/versions.lock b/atlasdb-jdbc/versions.lock index 7eec42da73b..83d1c34b088 100644 --- a/atlasdb-jdbc/versions.lock +++ b/atlasdb-jdbc/versions.lock @@ -94,8 +94,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -199,8 +198,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -219,7 +217,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -229,7 +227,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -238,31 +236,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -442,8 +440,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -547,8 +544,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -567,7 +563,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -577,7 +573,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -586,31 +582,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/atlasdb-jepsen-tests/versions.lock b/atlasdb-jepsen-tests/versions.lock index 67b3eed33da..43a2f88f383 100644 --- a/atlasdb-jepsen-tests/versions.lock +++ b/atlasdb-jepsen-tests/versions.lock @@ -198,8 +198,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -463,11 +462,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -498,7 +497,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -508,7 +507,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -517,32 +516,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -649,8 +648,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -895,8 +893,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1160,11 +1157,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1195,7 +1192,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1205,7 +1202,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1214,32 +1211,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1346,8 +1343,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/atlasdb-perf/versions.lock b/atlasdb-perf/versions.lock index 4871de77d5e..cb187c8e6f1 100644 --- a/atlasdb-perf/versions.lock +++ b/atlasdb-perf/versions.lock @@ -246,8 +246,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.dagger:dagger": { @@ -644,11 +643,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -681,7 +680,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -691,7 +690,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -700,32 +699,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -977,8 +976,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.javassist:javassist": { @@ -1325,8 +1323,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.dagger:dagger": { @@ -1723,11 +1720,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1760,7 +1757,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1770,7 +1767,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1779,32 +1776,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -2056,8 +2053,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.javassist:javassist": { diff --git a/atlasdb-service-server/versions.lock b/atlasdb-service-server/versions.lock index 9470ffa273c..21ed7a6b59d 100644 --- a/atlasdb-service-server/versions.lock +++ b/atlasdb-service-server/versions.lock @@ -217,8 +217,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -491,11 +490,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -526,7 +525,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -536,7 +535,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -545,32 +544,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1043,8 +1042,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.hibernate:hibernate-validator": { @@ -1375,8 +1373,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1696,11 +1693,11 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1733,7 +1730,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1743,7 +1740,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1752,32 +1749,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -2327,8 +2324,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.hibernate:hibernate-validator": { diff --git a/atlasdb-service/versions.lock b/atlasdb-service/versions.lock index eaa4b96a425..ac6a839477d 100644 --- a/atlasdb-service/versions.lock +++ b/atlasdb-service/versions.lock @@ -198,8 +198,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -463,11 +462,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -498,7 +497,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -508,7 +507,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -517,32 +516,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -647,8 +646,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -893,8 +891,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1158,11 +1155,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1193,7 +1190,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1203,7 +1200,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1212,32 +1209,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1342,8 +1339,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/atlasdb-tests-shared/versions.lock b/atlasdb-tests-shared/versions.lock index 47f06d62a2c..ed72fc09c7f 100644 --- a/atlasdb-tests-shared/versions.lock +++ b/atlasdb-tests-shared/versions.lock @@ -141,8 +141,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -376,7 +375,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -386,7 +385,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -395,31 +394,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -452,8 +451,7 @@ "locked": "3.2.3", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "javax.validation:validation-api": { @@ -521,8 +519,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "org.mockito:mockito-core": { @@ -712,8 +709,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -947,7 +943,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -957,7 +953,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -966,31 +962,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1023,8 +1019,7 @@ "locked": "3.2.3", "transitive": [ "com.palantir.atlasdb:atlasdb-commons", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "javax.validation:validation-api": { @@ -1092,8 +1087,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.tritium:tritium-metrics" ] }, "org.mockito:mockito-core": { diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index 5891b87703f..05ce03d8fac 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -68,6 +68,10 @@ develop - ``SweeperServiceImpl`` now logs when it starts sweeping and makes it clear if it is running full sweep or not (`Pull Request `__) + * - |improved| + - AtlasDB now depends on Tritium 0.8.4, which depends on the same version of ``com.palantir.remoting3`` and ``HdrHistogram`` as AtlasDB. + (`Pull Request `__) + * - |fixed| - Check that immutable timestamp is locked on write transactions with no writes. This could cause long-running readers to read an incorrect empty value when using the ``Sweep.THOROUGH`` strategy. diff --git a/examples/profile-client/versions.lock b/examples/profile-client/versions.lock index 5c62495263b..f6fd869c36f 100644 --- a/examples/profile-client/versions.lock +++ b/examples/profile-client/versions.lock @@ -91,8 +91,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -184,8 +183,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -203,7 +201,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -213,7 +211,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -222,31 +220,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -419,8 +417,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -512,8 +509,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -531,7 +527,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -541,7 +537,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -550,31 +546,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/timelock-agent/versions.lock b/timelock-agent/versions.lock index 27c3cb91876..c699e1fdc84 100644 --- a/timelock-agent/versions.lock +++ b/timelock-agent/versions.lock @@ -171,8 +171,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -452,12 +451,12 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", "com.palantir.atlasdb:timelock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -490,7 +489,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -500,7 +499,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -509,32 +508,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -622,8 +621,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -840,8 +838,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -1121,12 +1118,12 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", "com.palantir.atlasdb:timelock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1159,7 +1156,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1169,7 +1166,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1178,32 +1175,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1291,8 +1288,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/timelock-impl/versions.lock b/timelock-impl/versions.lock index a01408833e4..957f38f12b6 100644 --- a/timelock-impl/versions.lock +++ b/timelock-impl/versions.lock @@ -170,8 +170,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -440,11 +439,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -476,7 +475,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -486,7 +485,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -495,32 +494,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -608,8 +607,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { @@ -825,8 +823,7 @@ "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", - "com.palantir.remoting3:refresh-utils", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { @@ -1095,11 +1092,11 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", - "com.palantir.remoting3:jersey-servers", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:jersey-servers" ] }, "com.palantir.safe-logging:safe-logging": { @@ -1131,7 +1128,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1141,7 +1138,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1150,32 +1147,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -1263,8 +1260,7 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config" ] }, "org.mortbay.jetty.alpn:jetty-alpn-agent": { diff --git a/timelock-server-distribution/versions.lock b/timelock-server-distribution/versions.lock index 9b7e8b7ac3b..4ba67c12813 100644 --- a/timelock-server-distribution/versions.lock +++ b/timelock-server-distribution/versions.lock @@ -267,8 +267,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -675,6 +674,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", @@ -715,7 +715,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -725,7 +725,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -734,7 +734,7 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config", @@ -742,25 +742,25 @@ ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -844,8 +844,7 @@ "io.dropwizard.metrics:metrics-servlets", "io.dropwizard:dropwizard-core", "io.dropwizard:dropwizard-metrics", - "io.dropwizard:dropwizard-servlets", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-servlets" ] }, "io.dropwizard.metrics:metrics-healthchecks": { @@ -1382,8 +1381,8 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.tritium:tritium-metrics" ] }, "org.hibernate:hibernate-validator": { diff --git a/timelock-server/versions.lock b/timelock-server/versions.lock index 2ba066b95ab..a10b7987583 100644 --- a/timelock-server/versions.lock +++ b/timelock-server/versions.lock @@ -228,8 +228,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -520,6 +519,7 @@ "locked": "3.5.1", "transitive": [ "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", @@ -558,7 +558,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -568,7 +568,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -577,32 +577,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -658,8 +658,7 @@ "io.dropwizard.metrics:metrics-servlets", "io.dropwizard:dropwizard-core", "io.dropwizard:dropwizard-metrics", - "io.dropwizard:dropwizard-servlets", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-servlets" ] }, "io.dropwizard.metrics:metrics-healthchecks": { @@ -1119,8 +1118,8 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.tritium:tritium-metrics" ] }, "org.hibernate:hibernate-validator": { @@ -1472,8 +1471,7 @@ "com.palantir.remoting-api:errors", "com.palantir.remoting2:error-handling", "com.palantir.remoting3:refresh-utils", - "io.dropwizard:dropwizard-util", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-util" ] }, "com.google.guava:guava": { @@ -1862,6 +1860,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-config", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:leader-election-impl", "com.palantir.atlasdb:lock-impl", @@ -1901,7 +1900,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -1911,7 +1910,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -1920,32 +1919,32 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:atlasdb-config" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -2029,8 +2028,7 @@ "io.dropwizard.metrics:metrics-servlets", "io.dropwizard:dropwizard-core", "io.dropwizard:dropwizard-metrics", - "io.dropwizard:dropwizard-servlets", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "io.dropwizard:dropwizard-servlets" ] }, "io.dropwizard.metrics:metrics-healthchecks": { @@ -2562,8 +2560,8 @@ "locked": "2.1.10", "transitive": [ "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-config", + "com.palantir.tritium:tritium-metrics" ] }, "org.hibernate:hibernate-validator": { diff --git a/timestamp-impl/versions.lock b/timestamp-impl/versions.lock index 26fe2a158d8..33a452fc03a 100644 --- a/timestamp-impl/versions.lock +++ b/timestamp-impl/versions.lock @@ -93,8 +93,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -187,8 +186,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -206,7 +204,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -216,7 +214,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -225,31 +223,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] @@ -424,8 +422,7 @@ "com.google.code.findbugs:jsr305": { "locked": "1.3.9", "transitive": [ - "com.palantir.atlasdb:atlasdb-commons", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:atlasdb-commons" ] }, "com.google.guava:guava": { @@ -518,8 +515,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.atlasdb:atlasdb-client", - "com.palantir.tritium:tritium-tracing" + "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.safe-logging:safe-logging": { @@ -537,7 +533,7 @@ ] }, "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-core", "com.palantir.tritium:tritium-lib", @@ -547,7 +543,7 @@ ] }, "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib", "com.palantir.tritium:tritium-metrics", @@ -556,31 +552,31 @@ ] }, "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.atlasdb:atlasdb-client" ] }, "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] }, "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", + "locked": "0.8.4", "transitive": [ "com.palantir.tritium:tritium-lib" ] diff --git a/versions.props b/versions.props index 5f6da12fb31..6d0ad329a15 100644 --- a/versions.props +++ b/versions.props @@ -18,7 +18,7 @@ com.palantir.remoting2:* = 2.3.0 com.palantir.remoting3:* = 3.5.1 com.palantir.remoting-api:* = 1.4.0 com.palantir.safe-logging:* = 0.1.3 -com.palantir.tritium:tritium-lib = 0.8.3 +com.palantir.tritium:* = 0.8.4 com.squareup:javapoet = 1.9.0 com.squareup.okhttp3:* = 3.8.1 commons-cli:commons-cli = 1.2 From 7fb4d174a60c255bad43782f1ad3724862302cca Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Tue, 14 Nov 2017 17:37:27 +0000 Subject: [PATCH 21/67] Correctly log Paxos events (#2674) * Log out Paxos values when recording Paxos events * Updated release notes * Checkstyle * Pull request number * Address comments * fix docs --- docs/source/release_notes/release-notes.rst | 4 ++++ .../main/java/com/palantir/leader/LeadershipEvents.java | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/source/release_notes/release-notes.rst b/docs/source/release_notes/release-notes.rst index 05ce03d8fac..3af7316f0e5 100644 --- a/docs/source/release_notes/release-notes.rst +++ b/docs/source/release_notes/release-notes.rst @@ -77,6 +77,10 @@ develop This could cause long-running readers to read an incorrect empty value when using the ``Sweep.THOROUGH`` strategy. (`Pull Request `__) + * - |fixed| + - Paxos value information is now correctly being logged when applicable leader events are happening. + (`Pull Request `__) + .. <<<<------------------------------------------------------------------------------------------------------------->>>> ======= diff --git a/leader-election-impl/src/main/java/com/palantir/leader/LeadershipEvents.java b/leader-election-impl/src/main/java/com/palantir/leader/LeadershipEvents.java index 8a935424b1c..c310d406fc9 100644 --- a/leader-election-impl/src/main/java/com/palantir/leader/LeadershipEvents.java +++ b/leader-election-impl/src/main/java/com/palantir/leader/LeadershipEvents.java @@ -54,22 +54,23 @@ public LeadershipEvents(MetricRegistry metrics) { } public void proposedLeadershipFor(long round) { - leaderLog.info("Proposing leadership", SafeArg.of("round", round)); + leaderLog.info("Proposing leadership for {}", SafeArg.of("round", round)); proposedLeadership.mark(); } public void gainedLeadershipFor(PaxosValue value) { - leaderLog.info("Gained leadership", SafeArg.of("value", value)); + leaderLog.info("Gained leadership for {}", SafeArg.of("value", value)); gainedLeadership.mark(); } public void lostLeadershipFor(PaxosValue value) { - leaderLog.info("Lost leadership", SafeArg.of("value", value)); + leaderLog.info("Lost leadership for {}", SafeArg.of("value", value)); lostLeadership.mark(); } public void noQuorum(PaxosValue value) { - leaderLog.warn("The most recent known information says this server is the leader, but there is no quorum right now", + leaderLog.warn("The most recent known information says this server is the leader," + + " but there is no quorum right now. The paxos value is {}", SafeArg.of("value", value)); noQuorum.mark(); } From 14216ac31059b9d321bdaae3f45378c4a372589d Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Tue, 14 Nov 2017 18:51:52 +0000 Subject: [PATCH 22/67] Slow log and tracing (#2673) * Trace and instrument the thrift client * Instrument CqlExecutor * Fix metric names of IntrumentedCassandraClient * Fix nit * Also log internal table references * Checkstyle * simplify metric names * Address comments * add slow logging to the cassandra thrift client * add slow logging to cqlExecutor * fix typos * Add tracing to the CassandraClient * trace cqlExecutor queries * Add slow-logging in the CassandraClient * Delete InstrumentedCC and InstrumentedCqlExec * Fix small nits * Checkstyle * Add kvs method names to slow logs * Fix wrapping of exception * Extract CqlQuery * Move kvs-slow-log and tracing of CqlExecutor to CCI * Propagate execute_cql3_query api breaks * checkstyle * delete unused string * checkstyle * fix number of mutations on batch_mutate * some refactors * fix compile --- .../SchemaMutationLockTestTools.java | 30 ++-- .../keyvalue/cassandra/CassandraClient.java | 2 +- .../cassandra/CassandraClientFactory.java | 7 +- .../cassandra/CassandraClientImpl.java | 119 ++++++++++++-- .../CassandraTimestampBackupRunner.java | 7 +- .../cassandra/CassandraTimestampUtils.java | 8 +- .../keyvalue/cassandra/CqlExecutorImpl.java | 149 ++++-------------- .../atlasdb/keyvalue/cassandra/CqlQuery.java | 70 ++++++++ .../keyvalue/cassandra/CqlQueryUtils.java | 90 +++++++++++ .../CassandraTimestampUtilsTest.java | 27 ++-- .../keyvalue/cassandra/CqlExecutorTest.java | 21 ++- .../logging/KeyValueServiceLogArbitrator.java | 7 + .../atlasdb/logging/KvsProfilingLogger.java | 29 ++-- .../palantir/atlasdb/logging/LoggingArgs.java | 15 +- .../atlasdb/logging/SafeLoggableData.java | 14 ++ 15 files changed, 409 insertions(+), 186 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQuery.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQueryUtils.java diff --git a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java index de51535bc7e..1f219570795 100644 --- a/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java +++ b/atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/SchemaMutationLockTestTools.java @@ -15,7 +15,6 @@ */ package com.palantir.atlasdb.keyvalue.cassandra; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.List; @@ -28,6 +27,7 @@ import com.google.common.collect.Iterables; import com.google.common.primitives.Longs; +import com.palantir.logsafe.SafeArg; public final class SchemaMutationLockTestTools { private final CassandraClientPool clientPool; @@ -50,8 +50,9 @@ public CqlResult setLocksTableValue(long lockId, int heartbeatCount) throws TExc public CqlResult truncateLocksTable() throws TException { return clientPool.run(client -> { - String truncateCql = String.format("TRUNCATE \"%s\";", lockTable.getOnlyTable().getQualifiedName()); - return runCqlQuery(truncateCql, client, ConsistencyLevel.ALL); + CqlQuery truncateQuery = new CqlQuery("TRUNCATE \"%s\";", + SafeArg.of("lockTable", lockTable.getOnlyTable().getQualifiedName())); + return runCqlQuery(truncateQuery, client, ConsistencyLevel.ALL); }); } @@ -59,11 +60,11 @@ public CqlResult readLocksTable() throws TException { return clientPool.run(client -> { String lockRowName = getHexEncodedBytes(CassandraConstants.GLOBAL_DDL_LOCK_ROW_NAME); String lockColName = getHexEncodedBytes(CassandraConstants.GLOBAL_DDL_LOCK_COLUMN_NAME); - String selectCql = String.format( + CqlQuery selectCql = new CqlQuery( "SELECT \"value\" FROM \"%s\" WHERE key = %s AND column1 = %s AND column2 = -1;", - lockTable.getOnlyTable().getQualifiedName(), - lockRowName, - lockColName); + SafeArg.of("lockTable", lockTable.getOnlyTable().getQualifiedName()), + SafeArg.of("lockRow", lockRowName), + SafeArg.of("lockColumn", lockColName)); return runCqlQuery(selectCql, client, ConsistencyLevel.LOCAL_QUORUM); }); } @@ -85,12 +86,12 @@ private CqlResult setLocksTableValueInternal(String hexLockValue) throws TExcept String lockRowName = getHexEncodedBytes(CassandraConstants.GLOBAL_DDL_LOCK_ROW_NAME); String lockColName = getHexEncodedBytes(CassandraConstants.GLOBAL_DDL_LOCK_COLUMN_NAME); String lockTableName = lockTable.getOnlyTable().getQualifiedName(); - String updateCql = String.format( + CqlQuery updateCql = new CqlQuery( "UPDATE \"%s\" SET value = %s WHERE key = %s AND column1 = %s AND column2 = -1;", - lockTableName, - hexLockValue, - lockRowName, - lockColName); + SafeArg.of("lockTable", lockTableName), + SafeArg.of("hexLockValue", hexLockValue), + SafeArg.of("lockRow", lockRowName), + SafeArg.of("lockCol", lockColName)); return runCqlQuery(updateCql, client, ConsistencyLevel.EACH_QUORUM); }); } @@ -105,9 +106,8 @@ private static String getHexEncodedBytes(String str) { return CassandraKeyValueServices.encodeAsHex(str.getBytes(StandardCharsets.UTF_8)); } - private static CqlResult runCqlQuery(String query, CassandraClient client, ConsistencyLevel consistency) + private static CqlResult runCqlQuery(CqlQuery cqlQuery, CassandraClient client, ConsistencyLevel consistency) throws TException { - ByteBuffer queryBuffer = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); - return client.execute_cql3_query(queryBuffer, Compression.NONE, consistency); + return client.execute_cql3_query(cqlQuery, Compression.NONE, consistency); } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 9308718d920..c81815bf73f 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -76,7 +76,7 @@ CASResult cas(TableReference tableReference, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, org.apache.thrift.TException; - CqlResult execute_cql3_query(ByteBuffer query, + CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index e477dc87665..d711932b0a2 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -78,7 +78,7 @@ public CassandraClientFactory(InetSocketAddress addr, CassandraKeyValueServiceCo @Override public CassandraClient create() throws Exception { try { - return instrumentClient(getClient(addr, config)); + return instrumentClient(getRawClient(addr, config)); } catch (Exception e) { String message = String.format("Failed to construct client for %s/%s", addr, config.getKeyspaceOrThrow()); if (config.usingSsl()) { @@ -93,8 +93,9 @@ private CassandraClient instrumentClient(Client client) { return AtlasDbMetrics.instrument(CassandraClient.class, new CassandraClientImpl(client)); } - private static Cassandra.Client getClient(InetSocketAddress addr, - CassandraKeyValueServiceConfig config) throws Exception { + private static Cassandra.Client getRawClient(InetSocketAddress addr, CassandraKeyValueServiceConfig config) + throws Exception { + Client ret = getClientInternal(addr, config); try { ret.set_keyspace(config.getKeyspaceOrThrow()); diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java index 3bcc3740e41..62719ef5408 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java @@ -17,6 +17,7 @@ package com.palantir.atlasdb.keyvalue.cassandra; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -42,10 +43,16 @@ import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.impl.AbstractKeyValueService; +import com.palantir.atlasdb.logging.KvsProfilingLogger; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.atlasdb.tracing.CloseableTrace; +import com.palantir.logsafe.SafeArg; @SuppressWarnings({"all"}) // thrift variable names. public class CassandraClientImpl implements CassandraClient { - private Cassandra.Client client; + private static final String SERVICE_NAME = "cassandra-thrift-client"; + + private final Cassandra.Client client; public CassandraClientImpl(Cassandra.Client client) { this.client = client; @@ -64,8 +71,25 @@ public Map> multiget_slice( SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - ColumnParent colFam = getColumnParent(tableRef); - return client.multiget_slice(keys, colFam, predicate, consistency_level); + int numberOfKeys = keys.size(); + int numberOfColumns = predicate.slice_range.count; + + try (CloseableTrace trace = startLocalTrace( + "client.multiget_slice(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", + LoggingArgs.safeTableOrPlaceholder(tableRef), + numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { + ColumnParent colFam = getColumnParent(tableRef); + + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException>, TException>) + () -> client.multiget_slice(keys, colFam, predicate, consistency_level), + (logger, timer) -> logger.log("client.multiget_slice({}, {}, {}, {}) on kvs.{}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("number of keys", numberOfKeys), + SafeArg.of("number of columns", numberOfColumns), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } } @Override @@ -75,8 +99,25 @@ public List get_range_slices(String kvsMethodName, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - ColumnParent colFam = getColumnParent(tableRef); - return client.get_range_slices(colFam, predicate, range, consistency_level); + int numberOfKeys = predicate.slice_range.count; + int numberOfColumns = range.count; + + try (CloseableTrace trace = startLocalTrace( + "client.get_range_slices(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", + LoggingArgs.safeTableOrPlaceholder(tableRef), + numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { + ColumnParent colFam = getColumnParent(tableRef); + + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException, TException>) + () -> client.get_range_slices(colFam, predicate, range, consistency_level), + (logger, timer) -> logger.log("client.get_range_slices({}, {}, {}, {}) on kvs.{}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("number of keys", numberOfKeys), + SafeArg.of("number of columns", numberOfColumns), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } } @Override @@ -84,7 +125,22 @@ public void batch_mutate(String kvsMethodName, Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - client.batch_mutate(mutation_map, consistency_level); + int numberOfRowsMutated = mutation_map.size(); + + try (CloseableTrace trace = startLocalTrace("client.batch_mutate(number of rows mutated {}, consistency {})" + + " on kvs.{}", + numberOfRowsMutated, consistency_level, kvsMethodName)) { + KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> { + client.batch_mutate(mutation_map, consistency_level); + return null; + }, + (logger, timer) -> logger.log("client.batch_mutate({}, {}) on kvs.{}", + SafeArg.of("number of mutations", numberOfRowsMutated), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } } @Override @@ -95,7 +151,17 @@ public ColumnOrSuperColumn get(TableReference tableReference, throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { ColumnPath columnPath = new ColumnPath(tableReference.getQualifiedName()); columnPath.setColumn(column); - return client.get(key, columnPath, consistency_level); + + try (CloseableTrace trace = startLocalTrace("client.get(table {}, consistency {})", + LoggingArgs.safeTableOrPlaceholder(tableReference), consistency_level)) { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.get(key, columnPath, consistency_level), + (logger, timer) -> logger.log("client.get({}, {}) took {} ms", + LoggingArgs.tableRef(tableReference), + SafeArg.of("consistency", consistency_level.toString()), + LoggingArgs.durationMillis(timer))); + } } @Override @@ -107,18 +173,49 @@ public CASResult cas(TableReference tableReference, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { String internalTableName = AbstractKeyValueService.internalTableName(tableReference); - return client.cas(key, internalTableName, expected, updates, serial_consistency_level, - commit_consistency_level); + + try (CloseableTrace trace = startLocalTrace("client.cas(table {})", + LoggingArgs.safeTableOrPlaceholder(tableReference))) { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.cas(key, internalTableName, expected, updates, serial_consistency_level, + commit_consistency_level), + (logger, timer) -> logger.log("client.cas({}) took {} ms", + LoggingArgs.tableRef(tableReference), + LoggingArgs.durationMillis(timer))); + } } @Override - public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) + public CqlResult execute_cql3_query(CqlQuery cqlQuery, + Compression compression, + ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { - return client.execute_cql3_query(query, compression, consistency); + ByteBuffer queryBytes = ByteBuffer.wrap(cqlQuery.toString().getBytes(StandardCharsets.UTF_8)); + + try (CloseableTrace trace = startLocalTrace("cqlExecutor.execute_cql3_query(query {})", + cqlQuery.getLazySafeLoggableObject())) { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.execute_cql3_query(queryBytes, compression, consistency), + (logger, timer) -> cqlQuery.logSlowResult(logger, timer), + this::logResultSize); + } + } + + + + private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { + log.log("and returned {} rows", + SafeArg.of("numRows", result.getRows().size())); } private ColumnParent getColumnParent(TableReference tableRef) { return new ColumnParent(AbstractKeyValueService.internalTableName(tableRef)); } + + private static CloseableTrace startLocalTrace(CharSequence operationFormat, Object... formatArguments) { + return CloseableTrace.startLocalTrace(SERVICE_NAME, operationFormat, formatArguments); + } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java index 18ee96a9de3..a234ad267d0 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampBackupRunner.java @@ -15,7 +15,6 @@ */ package com.palantir.atlasdb.keyvalue.cassandra; -import java.nio.ByteBuffer; import java.util.Map; import javax.annotation.Nullable; @@ -153,7 +152,7 @@ private BoundReadability getReadability(BoundData boundData) { private BoundData getCurrentBoundData(CassandraClient client) { checkTimestampTableExists(); - ByteBuffer selectQuery = CassandraTimestampUtils.constructSelectFromTimestampTableQuery(); + CqlQuery selectQuery = CassandraTimestampUtils.constructSelectFromTimestampTableQuery(); CqlResult existingData = executeQueryUnchecked(client, selectQuery); Map columnarResults = CassandraTimestampUtils.getValuesFromSelectionResult(existingData); @@ -171,13 +170,13 @@ private void checkTimestampTableExists() { } private void executeAndVerifyCas(CassandraClient client, Map> casMap) { - ByteBuffer casQueryBuffer = CassandraTimestampUtils.constructCheckAndSetMultipleQuery(casMap); + CqlQuery casQueryBuffer = CassandraTimestampUtils.constructCheckAndSetMultipleQuery(casMap); CqlResult casResult = executeQueryUnchecked(client, casQueryBuffer); CassandraTimestampUtils.verifyCompatible(casResult, casMap); } - private CqlResult executeQueryUnchecked(CassandraClient client, ByteBuffer query) { + private CqlResult executeQueryUnchecked(CassandraClient client, CqlQuery query) { try { return queryRunner().run(client, AtlasDbConstants.TIMESTAMP_TABLE, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtils.java index a8b69439e67..80d29110c9a 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtils.java @@ -81,7 +81,7 @@ private CassandraTimestampUtils() { // utility class } - public static ByteBuffer constructCheckAndSetMultipleQuery(Map> checkAndSetRequest) { + public static CqlQuery constructCheckAndSetMultipleQuery(Map> checkAndSetRequest) { StringBuilder builder = new StringBuilder(); builder.append("BEGIN UNLOGGED BATCH\n"); // Safe, because all updates are on the same partition key @@ -92,15 +92,15 @@ public static ByteBuffer constructCheckAndSetMultipleQuery(Map getTimestamps( TableReference tableRef, byte[] startRowInclusive, int limit) { + String selQuery = "SELECT key, column1, column2 FROM %s WHERE token(key) >= token(%s) LIMIT %s;"; CqlQuery query = new CqlQuery( - "SELECT key, column1, column2 FROM %s WHERE token(key) >= token(%s) LIMIT %s;", - quotedTableName(tableRef), - key(startRowInclusive), - limit(limit)); - return query.executeAndGetCells(startRowInclusive, this::getCellFromRow); + selQuery, + CqlQueryUtils.quotedTableName(tableRef), + CqlQueryUtils.key(startRowInclusive), + CqlQueryUtils.limit(limit)); + + return executeAndGetCells(query, startRowInclusive, CqlQueryUtils::getCellFromRow); } /** @@ -91,103 +82,25 @@ public List getTimestampsWithinRow( long startTimestampExclusive, int limit) { long invertedTimestamp = ~startTimestampExclusive; + String selQuery = "SELECT column1, column2 FROM %s WHERE key = %s AND (column1, column2) > (%s, %s) LIMIT %s;"; CqlQuery query = new CqlQuery( - "SELECT column1, column2 FROM %s WHERE key = %s AND (column1, column2) > (%s, %s) LIMIT %s;", - quotedTableName(tableRef), - key(row), - column1(startColumnInclusive), - column2(invertedTimestamp), - limit(limit)); - return query.executeAndGetCells(row, result -> getCellFromKeylessRow(result, row)); - } - - private CellWithTimestamp getCellFromKeylessRow(CqlRow row, byte[] key) { - byte[] rowName = key; - byte[] columnName = row.getColumns().get(0).getValue(); - long timestamp = extractTimestamp(row, 1); + selQuery, + CqlQueryUtils.quotedTableName(tableRef), + CqlQueryUtils.key(row), + CqlQueryUtils.column1(startColumnInclusive), + CqlQueryUtils.column2(invertedTimestamp), + CqlQueryUtils.limit(limit)); - return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); + return executeAndGetCells(query, row, + result -> CqlQueryUtils.getCellFromKeylessRow(result, row)); } - private CellWithTimestamp getCellFromRow(CqlRow row) { - byte[] rowName = row.getColumns().get(0).getValue(); - byte[] columnName = row.getColumns().get(1).getValue(); - long timestamp = extractTimestamp(row, 2); - - return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); - } - - private long extractTimestamp(CqlRow row, int columnIndex) { - byte[] flippedTimestampAsBytes = row.getColumns().get(columnIndex).getValue(); - return ~PtBytes.toLong(flippedTimestampAsBytes); - } - - private Arg key(byte[] row) { - return UnsafeArg.of("key", CassandraKeyValueServices.encodeAsHex(row)); - } - - private Arg column1(byte[] column) { - return UnsafeArg.of("column1", CassandraKeyValueServices.encodeAsHex(column)); - } - - private Arg column2(long invertedTimestamp) { - return SafeArg.of("column2", invertedTimestamp); - } - - private Arg limit(long limit) { - return SafeArg.of("limit", limit); - } - - private Arg quotedTableName(TableReference tableRef) { - String tableNameWithQuotes = "\"" + CassandraKeyValueServiceImpl.internalTableName(tableRef) + "\""; - return LoggingArgs.customTableName(tableRef, tableNameWithQuotes); - } - - private List getCells(Function cellTsExtractor, CqlResult cqlResult) { - return cqlResult.getRows() - .stream() - .map(cellTsExtractor) - .collect(Collectors.toList()); - } - - private final class CqlQuery { - private final String queryFormat; - private final Arg[] queryArgs; - - CqlQuery(String queryFormat, Arg... args) { - this.queryFormat = queryFormat; - this.queryArgs = args; - } - - public List executeAndGetCells( - byte[] rowHintForHostSelection, - Function cellTsExtractor) { - CqlResult cqlResult = KvsProfilingLogger.maybeLog( - () -> queryExecutor.execute(rowHintForHostSelection, toString()), - this::logSlowResult, - this::logResultSize); - return getCells(cellTsExtractor, cqlResult); - } - - private void logSlowResult(KvsProfilingLogger.LoggingFunction log, Stopwatch timer) { - Object[] allArgs = new Object[queryArgs.length + 3]; - allArgs[0] = SafeArg.of("queryFormat", queryFormat); - allArgs[1] = UnsafeArg.of("fullQuery", toString()); - allArgs[2] = LoggingArgs.durationMillis(timer); - System.arraycopy(queryArgs, 0, allArgs, 3, queryArgs.length); - - log.log("A CQL query was slow: queryFormat = [{}], fullQuery = [{}], durationMillis = {}", allArgs); - } - - private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { - log.log("and returned {} rows", - SafeArg.of("numRows", result.getRows().size())); - } - - @Override - public String toString() { - return String.format(queryFormat, (Object[]) queryArgs); - } + private List executeAndGetCells( + CqlQuery query, + byte[] rowHintForHostSelection, + Function cellTsExtractor) { + CqlResult cqlResult = queryExecutor.execute(query, rowHintForHostSelection); + return CqlQueryUtils.getCells(cellTsExtractor, cqlResult); } private static class QueryExecutorImpl implements QueryExecutor { @@ -200,17 +113,17 @@ private static class QueryExecutorImpl implements QueryExecutor { } @Override - public CqlResult execute(byte[] rowHintForHostSelection, String query) { - return executeQueryOnHost(query, getHostForRow(rowHintForHostSelection)); + public CqlResult execute(CqlQuery cqlQuery, byte[] rowHintForHostSelection) { + return executeQueryOnHost(cqlQuery, getHostForRow(rowHintForHostSelection)); } private InetSocketAddress getHostForRow(byte[] row) { return clientPool.getRandomHostForKey(row); } - private CqlResult executeQueryOnHost(String query, InetSocketAddress host) { + private CqlResult executeQueryOnHost(CqlQuery cqlQuery, InetSocketAddress host) { try { - return clientPool.runWithRetryOnHost(host, createCqlFunction(query)); + return clientPool.runWithRetryOnHost(host, createCqlFunction(cqlQuery)); } catch (UnavailableException e) { throw wrapIfConsistencyAll(e); } catch (TException e) { @@ -227,18 +140,16 @@ private RuntimeException wrapIfConsistencyAll(UnavailableException ex) { } } - private FunctionCheckedException createCqlFunction(String query) { - ByteBuffer queryBytes = ByteBuffer.wrap(query.getBytes(StandardCharsets.UTF_8)); - + private FunctionCheckedException createCqlFunction(CqlQuery cqlQuery) { return new FunctionCheckedException() { @Override public CqlResult apply(CassandraClient client) throws TException { - return client.execute_cql3_query(queryBytes, Compression.NONE, consistency); + return client.execute_cql3_query(cqlQuery, Compression.NONE, consistency); } @Override public String toString() { - return query; + return cqlQuery.toString(); } }; } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQuery.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQuery.java new file mode 100644 index 00000000000..74d2e61d56c --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQuery.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import com.google.common.base.Stopwatch; +import com.palantir.atlasdb.logging.KvsProfilingLogger; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.logsafe.Arg; +import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.UnsafeArg; + +public class CqlQuery { + final String queryFormat; + final Arg[] queryArgs; + + CqlQuery(String queryFormat, Arg... args) { + this.queryFormat = queryFormat; + this.queryArgs = args; + } + + @Override + public String toString() { + return String.format(queryFormat, (Object[]) queryArgs); + } + + public void logSlowResult(KvsProfilingLogger.LoggingFunction log, Stopwatch timer) { + Object[] allArgs = new Object[queryArgs.length + 3]; + allArgs[0] = SafeArg.of("queryFormat", queryFormat); + allArgs[1] = UnsafeArg.of("fullQuery", toString()); + allArgs[2] = LoggingArgs.durationMillis(timer); + System.arraycopy(queryArgs, 0, allArgs, 3, queryArgs.length); + + log.log("A CQL query was slow: queryFormat = [{}], fullQuery = [{}], durationMillis = {}", allArgs); + } + + /** + * Returns an object whose {@link #toString()} implementation returns a log-safe string, and does all formatting + * lazily. This is intended to be used for trace logging, where the {@link #toString()} method may never be called. + */ + public Object getLazySafeLoggableObject() { + return new Object() { + @Override + public String toString() { + String argsString = Arrays.stream(queryArgs) + .filter(Arg::isSafeForLogging) + .map(arg -> String.format("%s = %s", arg.getName(), arg.getValue())) + .collect(Collectors.joining(", ")); + + return queryFormat + ": " + argsString; + } + }; + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQueryUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQueryUtils.java new file mode 100644 index 00000000000..9522360b334 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CqlQueryUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlRow; + +import com.palantir.atlasdb.encoding.PtBytes; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.cassandra.sweep.CellWithTimestamp; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.logsafe.Arg; +import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.UnsafeArg; + +public final class CqlQueryUtils { + private CqlQueryUtils() { + // utility class + } + + public static CellWithTimestamp getCellFromKeylessRow(CqlRow row, byte[] key) { + byte[] rowName = key; + byte[] columnName = row.getColumns().get(0).getValue(); + long timestamp = extractTimestamp(row, 1); + + return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); + } + + public static CellWithTimestamp getCellFromRow(CqlRow row) { + byte[] rowName = row.getColumns().get(0).getValue(); + byte[] columnName = row.getColumns().get(1).getValue(); + long timestamp = extractTimestamp(row, 2); + + return CellWithTimestamp.of(Cell.create(rowName, columnName), timestamp); + } + + public static long extractTimestamp(CqlRow row, int columnIndex) { + byte[] flippedTimestampAsBytes = row.getColumns().get(columnIndex).getValue(); + return ~PtBytes.toLong(flippedTimestampAsBytes); + } + + public static Arg key(byte[] row) { + return UnsafeArg.of("key", CassandraKeyValueServices.encodeAsHex(row)); + } + + public static Arg column1(byte[] column) { + return UnsafeArg.of("column1", CassandraKeyValueServices.encodeAsHex(column)); + } + + public static Arg column2(long invertedTimestamp) { + return SafeArg.of("column2", invertedTimestamp); + } + + public static Arg limit(long limit) { + return SafeArg.of("limit", limit); + } + + public static Arg quotedTableName(TableReference tableRef) { + String tableNameWithQuotes = "\"" + CassandraKeyValueServiceImpl.internalTableName(tableRef) + "\""; + return LoggingArgs.customTableName(tableRef, tableNameWithQuotes); + } + + public static List getCells(Function cellTsExtractor, + CqlResult cqlResult) { + return cqlResult.getRows() + .stream() + .map(cellTsExtractor) + .collect(Collectors.toList()); + } +} diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtilsTest.java index 750d5b55015..0c3c8c63911 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraTimestampUtilsTest.java @@ -20,7 +20,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.nio.ByteBuffer; import java.util.List; import java.util.Map; @@ -71,8 +70,8 @@ public void canGetValuesFromSelectionResult() { @Test public void canGetSelectQuery() { - String query = queryBufferToString(CassandraTimestampUtils.constructSelectFromTimestampTableQuery()); - assertThat(query).isEqualTo("SELECT column1, value FROM \"_timestamp\" WHERE key=0x7473;"); + CqlQuery query = CassandraTimestampUtils.constructSelectFromTimestampTableQuery(); + assertThat(query.toString()).isEqualTo("SELECT column1, value FROM \"_timestamp\" WHERE key=0x7473;"); } @Test @@ -105,24 +104,24 @@ public void largeByteArrayIsNotValidTimestampData() { @Test public void checkAndSetIsInsertIfNotExistsIfExpectedIsNull() { - String query = queryBufferToString(CassandraTimestampUtils.constructCheckAndSetMultipleQuery( - ImmutableMap.of(COLUMN_NAME_1, Pair.create(null, VALUE_1)))); - assertThat(query).contains("INSERT").contains("IF NOT EXISTS;"); + CqlQuery query = CassandraTimestampUtils.constructCheckAndSetMultipleQuery( + ImmutableMap.of(COLUMN_NAME_1, Pair.create(null, VALUE_1))); + assertThat(query.toString()).contains("INSERT").contains("IF NOT EXISTS;"); } @Test public void checkAndSetIsUpdateIfEqualIfExpectedIsNotNull() { - String query = queryBufferToString(CassandraTimestampUtils.constructCheckAndSetMultipleQuery( - ImmutableMap.of(COLUMN_NAME_1, Pair.create(VALUE_1, VALUE_2)))); - assertThat(query).contains("UPDATE").contains("IF " + CassandraTimestampUtils.VALUE_COLUMN + "="); + CqlQuery query = CassandraTimestampUtils.constructCheckAndSetMultipleQuery( + ImmutableMap.of(COLUMN_NAME_1, Pair.create(VALUE_1, VALUE_2))); + assertThat(query.toString()).contains("UPDATE").contains("IF " + CassandraTimestampUtils.VALUE_COLUMN + "="); } @Test public void checkAndSetGeneratesBatchedStatements() { - String query = queryBufferToString(CassandraTimestampUtils.constructCheckAndSetMultipleQuery( + CqlQuery query = CassandraTimestampUtils.constructCheckAndSetMultipleQuery( ImmutableMap.of(COLUMN_NAME_1, Pair.create(VALUE_1, VALUE_2), - COLUMN_NAME_2, Pair.create(VALUE_2, VALUE_1)))); - assertThat(query).contains("BEGIN UNLOGGED BATCH").contains("APPLY BATCH;"); + COLUMN_NAME_2, Pair.create(VALUE_2, VALUE_1))); + assertThat(query.toString()).contains("BEGIN UNLOGGED BATCH").contains("APPLY BATCH;"); } @Test @@ -227,8 +226,4 @@ private static Column createColumn(String name, byte[] value) { column.setValue(value); return column; } - - private static String queryBufferToString(ByteBuffer query) { - return PtBytes.toString(query.array()); - } } diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java index d07a83ca4bc..6329292a021 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CqlExecutorTest.java @@ -17,6 +17,8 @@ package com.palantir.atlasdb.keyvalue.cassandra; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -26,6 +28,7 @@ import org.apache.cassandra.thrift.CqlResult; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatcher; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; @@ -61,7 +64,7 @@ public void getTimestamps() { executor.getTimestamps(TABLE_REF, ROW, LIMIT); - verify(queryExecutor).execute(ROW, expected); + verify(queryExecutor).execute(argThat(cqlQueryMatcher(expected)), eq(ROW)); } @Test @@ -71,7 +74,21 @@ public void getTimestampsWithinRow() { executor.getTimestampsWithinRow(TABLE_REF, ROW, COLUMN, TIMESTAMP, LIMIT); - verify(queryExecutor).execute(ROW, expected); + verify(queryExecutor).execute(argThat(cqlQueryMatcher(expected)), eq(ROW)); + } + + private ArgumentMatcher cqlQueryMatcher(String expected) { + return new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + if (!(argument instanceof CqlQuery)) { + return false; + } + + String actualQuery = ((CqlQuery) argument).toString(); + return expected.equals(actualQuery); + } + }; } } diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KeyValueServiceLogArbitrator.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KeyValueServiceLogArbitrator.java index eabef30baf6..c7223d2513a 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KeyValueServiceLogArbitrator.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KeyValueServiceLogArbitrator.java @@ -25,6 +25,11 @@ public boolean isTableReferenceSafe(TableReference tableReference) { return false; } + @Override + public boolean isInternalTableReferenceSafe(String internalTableReference) { + return false; + } + @Override public boolean isRowComponentNameSafe(TableReference tableReference, String rowComponentName) { return false; @@ -38,6 +43,8 @@ public boolean isColumnNameSafe(TableReference tableReference, String longNameFo boolean isTableReferenceSafe(TableReference tableReference); + boolean isInternalTableReferenceSafe(String internalTableReference); + boolean isRowComponentNameSafe( TableReference tableReference, String rowComponentName); diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KvsProfilingLogger.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KvsProfilingLogger.java index d3b61498e19..48069b95f0d 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KvsProfilingLogger.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/KvsProfilingLogger.java @@ -35,7 +35,7 @@ public class KvsProfilingLogger { public static final String SLOW_LOGGER_NAME = "kvs-slow-log"; - private static final Logger slowlogger = LoggerFactory.getLogger(SLOW_LOGGER_NAME); + public static final Logger slowlogger = LoggerFactory.getLogger(SLOW_LOGGER_NAME); private static final Logger log = LoggerFactory.getLogger(KvsProfilingLogger.class); public static final int DEFAULT_THRESHOLD_MILLIS = 1000; @@ -91,26 +91,32 @@ public static void setSlowLogThresholdMillis(long thresholdMillis) { slowLogPredicate = createLogPredicateForThresholdMillis(thresholdMillis); } - public static T maybeLog(Supplier action, BiConsumer logger) { - return maybeLog(action, logger, (loggingFunction, result) -> { }); - } - public static void maybeLog(Runnable runnable, BiConsumer logger) { - maybeLog(() -> { + maybeLog((Supplier) () -> { runnable.run(); return null; }, logger); } - public static T maybeLog(Supplier action, BiConsumer primaryLogger, - BiConsumer additonalLoggerWithAccessToResult) { + public static T maybeLog(Supplier action, BiConsumer logger) { + return maybeLog(action::get, logger, (loggingFunction, result) -> {}); + } + + public static T maybeLog(CallableCheckedException action, + BiConsumer primaryLogger) throws E { + return maybeLog(action, primaryLogger, ((loggingFunction, result) -> {})); + } + + public static T maybeLog(CallableCheckedException action, + BiConsumer primaryLogger, + BiConsumer additonalLoggerWithAccessToResult) throws E { if (log.isTraceEnabled() || slowlogger.isWarnEnabled()) { Monitor monitor = Monitor.createMonitor( primaryLogger, additonalLoggerWithAccessToResult, slowLogPredicate); try { - T res = action.get(); + T res = action.call(); monitor.registerResult(res); return res; } catch (Exception ex) { @@ -120,7 +126,7 @@ public static T maybeLog(Supplier action, BiConsumer createLogPredicateForThresholdMillis(long th return stopwatch -> stopwatch.elapsed(TimeUnit.MILLISECONDS) > thresholdMillis; } + public interface CallableCheckedException { + T call() throws E; + } } diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java index bd8ed1d9dcc..a1e5ec6cfe4 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/LoggingArgs.java @@ -44,9 +44,11 @@ */ public final class LoggingArgs { + private static final String PLACEHOLDER_TABLE_NAME = "{table}"; + @VisibleForTesting static final TableReference PLACEHOLDER_TABLE_REFERENCE = - TableReference.createWithEmptyNamespace("{table}"); + TableReference.createWithEmptyNamespace(PLACEHOLDER_TABLE_NAME); @Value.Immutable public interface SafeAndUnsafeTableReferences { @@ -110,6 +112,17 @@ public static TableReference safeTableOrPlaceholder(TableReference tableReferenc } } + /** + * If table is safe, returns the table. If unsafe, returns a placeholder name. + */ + public static String safeInternalTableNameOrPlaceholder(String internalTableReference) { + if (logArbitrator.isInternalTableReferenceSafe(internalTableReference)) { + return internalTableReference; + } else { + return PLACEHOLDER_TABLE_NAME; + } + } + public static Arg tableRef(String argName, TableReference tableReference) { return getArg(argName, tableReference.toString(), logArbitrator.isTableReferenceSafe(tableReference)); } diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/SafeLoggableData.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/SafeLoggableData.java index 4915c8221fe..01169442763 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/SafeLoggableData.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/logging/SafeLoggableData.java @@ -18,15 +18,24 @@ import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.immutables.value.Value; import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.impl.AbstractKeyValueService; @Value.Immutable public abstract class SafeLoggableData implements KeyValueServiceLogArbitrator { public abstract Set permittedTableReferences(); + @Value.Lazy + public Set permittedInternalTableReferences() { + return permittedTableReferences().stream() + .map(AbstractKeyValueService::internalTableName) + .collect(Collectors.toSet()); + } + public abstract Map> permittedRowComponents(); public abstract Map> permittedColumnNames(); @@ -40,6 +49,11 @@ public boolean isTableReferenceSafe(TableReference tableReference) { return permittedTableReferences().contains(tableReference); } + @Override + public boolean isInternalTableReferenceSafe(String internalTableReference) { + return permittedInternalTableReferences().contains(internalTableReference); + } + @Override public boolean isRowComponentNameSafe(TableReference tableReference, String rowComponentName) { return permittedRowComponents().containsKey(tableReference) From 78791cfa6170d8859241237c48cbf376c4d96fd8 Mon Sep 17 00:00:00 2001 From: Samuel Souza Date: Tue, 14 Nov 2017 19:49:28 +0000 Subject: [PATCH 23/67] Refactor cassandra client (#2676) * Extract TracingCassandraClient Extract ProfilingCassandraClient Move todos and some cleanup Cherry-pick QoS metrics to develop (#2679) * [QoS] Feature/qos meters (#2640) * Metrics for bytes and counts in each read/write * Refactors, dont throw if recordMetrics throws * Use meters instead of histograms * Multiget bytes * Batch mutate exact size * Cqlresult size * Calculate exact byte sizes for all thrift objects * tests and bugfixes - partial * More tests and bugs fixed * More tests and cr comments * byte buffer size * Remove register histogram * checkstyle * checkstyle * locks and license * Qos metrics CassandraClient * Exclude unused classes * fix cherry pick --- .../cassandra/CassandraClientFactory.java | 9 +- .../cassandra/CassandraClientImpl.java | 110 +------- .../cassandra/ProfilingCassandraClient.java | 168 +++++++++++++ .../cassandra/QosCassandraClient.java | 157 ++++++++++++ .../keyvalue/cassandra/QosMetrics.java | 53 ++++ .../cassandra/ThriftObjectSizeUtils.java | 237 ++++++++++++++++++ .../cassandra/TracingCassandraClient.java | 152 +++++++++++ .../atlasdb/ThriftObjectSizeUtilsTest.java | 201 +++++++++++++++ .../palantir/atlasdb/util/MetricsManager.java | 4 + 9 files changed, 990 insertions(+), 101 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ProfilingCassandraClient.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingCassandraClient.java create mode 100644 atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index d711932b0a2..e5b4e9490d8 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -88,9 +88,14 @@ public CassandraClient create() throws Exception { } } - private CassandraClient instrumentClient(Client client) { + private CassandraClient instrumentClient(Client rawClient) { + CassandraClient client = new CassandraClientImpl(rawClient); + client = new ProfilingCassandraClient(client); + client = new TracingCassandraClient(client); // TODO(ssouza): use the kvsMethodName to tag the timers. - return AtlasDbMetrics.instrument(CassandraClient.class, new CassandraClientImpl(client)); + client = AtlasDbMetrics.instrument(CassandraClient.class, client); + client = new QosCassandraClient(client); + return client; } private static Cassandra.Client getRawClient(InetSocketAddress addr, CassandraKeyValueServiceConfig config) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java index 62719ef5408..49d96d28491 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientImpl.java @@ -43,15 +43,9 @@ import com.palantir.atlasdb.keyvalue.api.TableReference; import com.palantir.atlasdb.keyvalue.impl.AbstractKeyValueService; -import com.palantir.atlasdb.logging.KvsProfilingLogger; -import com.palantir.atlasdb.logging.LoggingArgs; -import com.palantir.atlasdb.tracing.CloseableTrace; -import com.palantir.logsafe.SafeArg; @SuppressWarnings({"all"}) // thrift variable names. public class CassandraClientImpl implements CassandraClient { - private static final String SERVICE_NAME = "cassandra-thrift-client"; - private final Cassandra.Client client; public CassandraClientImpl(Cassandra.Client client) { @@ -71,25 +65,9 @@ public Map> multiget_slice( SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - int numberOfKeys = keys.size(); - int numberOfColumns = predicate.slice_range.count; - - try (CloseableTrace trace = startLocalTrace( - "client.multiget_slice(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", - LoggingArgs.safeTableOrPlaceholder(tableRef), - numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { - ColumnParent colFam = getColumnParent(tableRef); - - return KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException>, TException>) - () -> client.multiget_slice(keys, colFam, predicate, consistency_level), - (logger, timer) -> logger.log("client.multiget_slice({}, {}, {}, {}) on kvs.{}", - LoggingArgs.tableRef(tableRef), - SafeArg.of("number of keys", numberOfKeys), - SafeArg.of("number of columns", numberOfColumns), - SafeArg.of("consistency", consistency_level.toString()), - SafeArg.of("kvsMethodName", kvsMethodName))); - } + ColumnParent colFam = getColumnParent(tableRef); + + return client.multiget_slice(keys, colFam, predicate, consistency_level); } @Override @@ -99,25 +77,9 @@ public List get_range_slices(String kvsMethodName, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - int numberOfKeys = predicate.slice_range.count; - int numberOfColumns = range.count; - - try (CloseableTrace trace = startLocalTrace( - "client.get_range_slices(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", - LoggingArgs.safeTableOrPlaceholder(tableRef), - numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { - ColumnParent colFam = getColumnParent(tableRef); - - return KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException, TException>) - () -> client.get_range_slices(colFam, predicate, range, consistency_level), - (logger, timer) -> logger.log("client.get_range_slices({}, {}, {}, {}) on kvs.{}", - LoggingArgs.tableRef(tableRef), - SafeArg.of("number of keys", numberOfKeys), - SafeArg.of("number of columns", numberOfColumns), - SafeArg.of("consistency", consistency_level.toString()), - SafeArg.of("kvsMethodName", kvsMethodName))); - } + ColumnParent colFam = getColumnParent(tableRef); + + return client.get_range_slices(colFam, predicate, range, consistency_level); } @Override @@ -125,22 +87,7 @@ public void batch_mutate(String kvsMethodName, Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - int numberOfRowsMutated = mutation_map.size(); - - try (CloseableTrace trace = startLocalTrace("client.batch_mutate(number of rows mutated {}, consistency {})" - + " on kvs.{}", - numberOfRowsMutated, consistency_level, kvsMethodName)) { - KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException) - () -> { - client.batch_mutate(mutation_map, consistency_level); - return null; - }, - (logger, timer) -> logger.log("client.batch_mutate({}, {}) on kvs.{}", - SafeArg.of("number of mutations", numberOfRowsMutated), - SafeArg.of("consistency", consistency_level.toString()), - SafeArg.of("kvsMethodName", kvsMethodName))); - } + client.batch_mutate(mutation_map, consistency_level); } @Override @@ -152,16 +99,7 @@ public ColumnOrSuperColumn get(TableReference tableReference, ColumnPath columnPath = new ColumnPath(tableReference.getQualifiedName()); columnPath.setColumn(column); - try (CloseableTrace trace = startLocalTrace("client.get(table {}, consistency {})", - LoggingArgs.safeTableOrPlaceholder(tableReference), consistency_level)) { - return KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException) - () -> client.get(key, columnPath, consistency_level), - (logger, timer) -> logger.log("client.get({}, {}) took {} ms", - LoggingArgs.tableRef(tableReference), - SafeArg.of("consistency", consistency_level.toString()), - LoggingArgs.durationMillis(timer))); - } + return client.get(key, columnPath, consistency_level); } @Override @@ -174,16 +112,8 @@ public CASResult cas(TableReference tableReference, throws InvalidRequestException, UnavailableException, TimedOutException, TException { String internalTableName = AbstractKeyValueService.internalTableName(tableReference); - try (CloseableTrace trace = startLocalTrace("client.cas(table {})", - LoggingArgs.safeTableOrPlaceholder(tableReference))) { - return KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException) - () -> client.cas(key, internalTableName, expected, updates, serial_consistency_level, - commit_consistency_level), - (logger, timer) -> logger.log("client.cas({}) took {} ms", - LoggingArgs.tableRef(tableReference), - LoggingArgs.durationMillis(timer))); - } + return client.cas(key, internalTableName, expected, updates, serial_consistency_level, + commit_consistency_level); } @Override @@ -194,28 +124,10 @@ public CqlResult execute_cql3_query(CqlQuery cqlQuery, TException { ByteBuffer queryBytes = ByteBuffer.wrap(cqlQuery.toString().getBytes(StandardCharsets.UTF_8)); - try (CloseableTrace trace = startLocalTrace("cqlExecutor.execute_cql3_query(query {})", - cqlQuery.getLazySafeLoggableObject())) { - return KvsProfilingLogger.maybeLog( - (KvsProfilingLogger.CallableCheckedException) - () -> client.execute_cql3_query(queryBytes, compression, consistency), - (logger, timer) -> cqlQuery.logSlowResult(logger, timer), - this::logResultSize); - } - } - - - - private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { - log.log("and returned {} rows", - SafeArg.of("numRows", result.getRows().size())); + return client.execute_cql3_query(queryBytes, compression, consistency); } private ColumnParent getColumnParent(TableReference tableRef) { return new ColumnParent(AbstractKeyValueService.internalTableName(tableRef)); } - - private static CloseableTrace startLocalTrace(CharSequence operationFormat, Object... formatArguments) { - return CloseableTrace.startLocalTrace(SERVICE_NAME, operationFormat, formatArguments); - } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ProfilingCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ProfilingCassandraClient.java new file mode 100644 index 00000000000..04ef3014697 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ProfilingCassandraClient.java @@ -0,0 +1,168 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.thrift.CASResult; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.InvalidRequestException; +import org.apache.cassandra.thrift.KeyRange; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.NotFoundException; +import org.apache.cassandra.thrift.SchemaDisagreementException; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.TimedOutException; +import org.apache.cassandra.thrift.UnavailableException; +import org.apache.thrift.TException; + +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.logging.KvsProfilingLogger; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.logsafe.SafeArg; + +@SuppressWarnings({"all"}) // thrift variable names. +public class ProfilingCassandraClient implements CassandraClient { + private final CassandraClient client; + + public ProfilingCassandraClient(CassandraClient client) { + this.client = client; + } + + @Override + public Cassandra.Client rawClient() { + return client.rawClient(); + } + + @Override + public Map> multiget_slice(String kvsMethodName, + TableReference tableRef, + List keys, + SlicePredicate predicate, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + int numberOfKeys = keys.size(); + int numberOfColumns = predicate.slice_range.count; + + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException>, TException>) + () -> client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level), + (logger, timer) -> logger.log("client.multiget_slice({}, {}, {}, {}) on kvs.{}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("number of keys", numberOfKeys), + SafeArg.of("number of columns", numberOfColumns), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } + + @Override + public List get_range_slices(String kvsMethodName, + TableReference tableRef, + SlicePredicate predicate, + KeyRange range, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + int numberOfKeys = predicate.slice_range.count; + int numberOfColumns = range.count; + + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException, TException>) + () -> client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level), + (logger, timer) -> logger.log("client.get_range_slices({}, {}, {}, {}) on kvs.{}", + LoggingArgs.tableRef(tableRef), + SafeArg.of("number of keys", numberOfKeys), + SafeArg.of("number of columns", numberOfColumns), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } + + @Override + public void batch_mutate(String kvsMethodName, + Map>> mutation_map, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + // TODO(ssouza): log more info than just the numberOfRowsMutated. + int numberOfRowsMutated = mutation_map.size(); + + KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> { + client.batch_mutate(kvsMethodName, mutation_map, consistency_level); + return null; + }, + (logger, timer) -> logger.log("client.batch_mutate({}, {}) on kvs.{}", + SafeArg.of("number of mutations", numberOfRowsMutated), + SafeArg.of("consistency", consistency_level.toString()), + SafeArg.of("kvsMethodName", kvsMethodName))); + } + + @Override + public ColumnOrSuperColumn get(TableReference tableReference, + ByteBuffer key, + byte[] column, + ConsistencyLevel consistency_level) + throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.get(tableReference, key, column, consistency_level), + (logger, timer) -> logger.log("client.get({}, {}) took {} ms", + LoggingArgs.tableRef(tableReference), + SafeArg.of("consistency", consistency_level.toString()), + LoggingArgs.durationMillis(timer))); + } + + @Override + public CASResult cas(TableReference tableReference, + ByteBuffer key, + List expected, + List updates, + ConsistencyLevel serial_consistency_level, + ConsistencyLevel commit_consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.cas(tableReference, key, expected, updates, serial_consistency_level, + commit_consistency_level), + (logger, timer) -> logger.log("client.cas({}) took {} ms", + LoggingArgs.tableRef(tableReference), + LoggingArgs.durationMillis(timer))); + } + + @Override + public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) + throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, + TException { + return KvsProfilingLogger.maybeLog( + (KvsProfilingLogger.CallableCheckedException) + () -> client.execute_cql3_query(cqlQuery, compression, consistency), + (logger, timer) -> cqlQuery.logSlowResult(logger, timer), + this::logResultSize); + } + + private void logResultSize(KvsProfilingLogger.LoggingFunction log, CqlResult result) { + log.log("and returned {} rows", + SafeArg.of("numRows", result.getRows().size())); + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java new file mode 100644 index 00000000000..9b8212a5005 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java @@ -0,0 +1,157 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.cassandra.thrift.CASResult; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.InvalidRequestException; +import org.apache.cassandra.thrift.KeyRange; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.NotFoundException; +import org.apache.cassandra.thrift.SchemaDisagreementException; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.TimedOutException; +import org.apache.cassandra.thrift.UnavailableException; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.palantir.atlasdb.keyvalue.api.TableReference; + +@SuppressWarnings({"all"}) // thrift variable names. +public class QosCassandraClient implements CassandraClient { + private final Logger log = LoggerFactory.getLogger(CassandraClient.class); + + private final CassandraClient client; + private final QosMetrics qosMetrics; + + public QosCassandraClient(CassandraClient client) { + this.client = client; + qosMetrics = new QosMetrics(); + } + + @Override + public Cassandra.Client rawClient() { + return client.rawClient(); + } + + @Override + public Map> multiget_slice(String kvsMethodName, TableReference tableRef, + List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + Map> result = client.multiget_slice(kvsMethodName, tableRef, keys, + predicate, consistency_level); + recordBytesRead(getApproximateReadByteCount(result)); + return result; + } + + private long getApproximateReadByteCount(Map> result) { + return getCollectionSize(result.entrySet(), + rowResult -> ThriftObjectSizeUtils.getByteBufferSize(rowResult.getKey()) + + getCollectionSize(rowResult.getValue(), + ThriftObjectSizeUtils::getColumnOrSuperColumnSize)); + } + + @Override + public List get_range_slices(String kvsMethodName, TableReference tableRef, SlicePredicate predicate, + KeyRange range, ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + List result = client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); + recordBytesRead(getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); + return result; + } + + @Override + public void batch_mutate(String kvsMethodName, Map>> mutation_map, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + client.batch_mutate(kvsMethodName, mutation_map, consistency_level); + recordBytesWritten(getApproximateWriteByteCount(mutation_map)); + } + + private long getApproximateWriteByteCount(Map>> batchMutateMap) { + long approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); + long approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> + getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + + getCollectionSize(currentMap.values(), + mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); + return approxBytesForKeys + approxBytesForValues; + } + + @Override + public ColumnOrSuperColumn get(TableReference tableReference, ByteBuffer key, byte[] column, + ConsistencyLevel consistency_level) + throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { + ColumnOrSuperColumn result = client.get(tableReference, key, column, consistency_level); + recordBytesRead(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(result)); + return result; + } + + @Override + public CASResult cas(TableReference tableReference, ByteBuffer key, List expected, List updates, + ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + CASResult result = client.cas(tableReference, key, expected, updates, serial_consistency_level, + commit_consistency_level); + recordBytesWritten(updates.stream().mapToLong(ThriftObjectSizeUtils::getColumnSize).sum()); + recordBytesRead(expected.stream().mapToLong(ThriftObjectSizeUtils::getColumnSize).sum()); + return result; + } + + @Override + public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) + throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, + TException { + CqlResult cqlResult = client.execute_cql3_query(cqlQuery, compression, consistency); + recordBytesRead(ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); + return cqlResult; + } + + private void recordBytesRead(long numBytesRead) { + try { + qosMetrics.updateReadCount(); + qosMetrics.updateBytesRead(numBytesRead); + } catch (Exception e) { + log.warn("Encountered an exception when recording read metrics.", e); + } + } + + private void recordBytesWritten(long numBytesWritten) { + try { + qosMetrics.updateWriteCount(); + qosMetrics.updateBytesWritten(numBytesWritten); + } catch (Exception e) { + log.warn("Encountered an exception when recording write metrics.", e); + } + } + + private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { + return collection.stream().mapToLong(singleObjectSizeFunction::apply).sum(); + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java new file mode 100644 index 00000000000..23b3d9e58a8 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import com.codahale.metrics.Meter; +import com.palantir.atlasdb.util.MetricsManager; + +public class QosMetrics { + private final MetricsManager metricsManager = new MetricsManager(); + + private final Meter readRequestCount; + private final Meter writeRequestCount; + private final Meter bytesRead; + private final Meter bytesWritten; + + public QosMetrics() { + readRequestCount = metricsManager.registerMeter(QosMetrics.class, "numReadRequests"); + writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); + + bytesRead = metricsManager.registerMeter(QosMetrics.class, "bytesRead"); + bytesWritten = metricsManager.registerMeter(QosMetrics.class, "bytesWritten"); + } + + public void updateReadCount() { + readRequestCount.mark(); + } + + public void updateWriteCount() { + writeRequestCount.mark(); + } + + public void updateBytesRead(long numBytes) { + bytesRead.mark(numBytes); + } + + public void updateBytesWritten(long numBytes) { + bytesWritten.mark(numBytes); + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java new file mode 100644 index 00000000000..8917cfcc9b5 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -0,0 +1,237 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; + +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CounterColumn; +import org.apache.cassandra.thrift.CounterSuperColumn; +import org.apache.cassandra.thrift.CqlMetadata; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlRow; +import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SliceRange; +import org.apache.cassandra.thrift.SuperColumn; + +public final class ThriftObjectSizeUtils { + + private static final long ONE_BYTE = 1L; + + private ThriftObjectSizeUtils() { + // utility class + } + + public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { + if (columnOrSuperColumn == null) { + return getNullSize(); + } + return getColumnSize(columnOrSuperColumn.getColumn()) + + getSuperColumnSize(columnOrSuperColumn.getSuper_column()) + + getCounterColumnSize(columnOrSuperColumn.getCounter_column()) + + getCounterSuperColumnSize(columnOrSuperColumn.getCounter_super_column()); + } + + public static long getByteBufferSize(ByteBuffer byteBuffer) { + if (byteBuffer == null) { + return getNullSize(); + } + return byteBuffer.remaining(); + } + + public static long getMutationSize(Mutation mutation) { + if (mutation == null) { + return getNullSize(); + } + + return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize( + mutation.getDeletion()); + } + + public static long getCqlResultSize(CqlResult cqlResult) { + if (cqlResult == null) { + return getNullSize(); + } + return getThriftEnumSize() + + getCollectionSize(cqlResult.getRows(), ThriftObjectSizeUtils::getCqlRowSize) + + Integer.BYTES + + getCqlMetadataSize(cqlResult.getSchema()); + } + + public static long getKeySliceSize(KeySlice keySlice) { + if (keySlice == null) { + return getNullSize(); + } + + return getByteArraySize(keySlice.getKey()) + + getCollectionSize(keySlice.getColumns(), ThriftObjectSizeUtils::getColumnOrSuperColumnSize); + } + + public static long getStringSize(String string) { + if (string == null) { + return getNullSize(); + } + + return string.length() * Character.SIZE; + } + + public static long getColumnSize(Column column) { + if (column == null) { + return getNullSize(); + } + + return getByteArraySize(column.getValue()) + + getByteArraySize(column.getName()) + + getTtlSize() + + getTimestampSize(); + } + + private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { + if (counterSuperColumn == null) { + return getNullSize(); + } + + return getByteArraySize(counterSuperColumn.getName()) + + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); + } + + private static long getCounterColumnSize(CounterColumn counterColumn) { + if (counterColumn == null) { + return getNullSize(); + } + + return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); + } + + private static long getSuperColumnSize(SuperColumn superColumn) { + if (superColumn == null) { + return getNullSize(); + } + + return getByteArraySize(superColumn.getName()) + + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); + } + + private static long getDeletionSize(Deletion deletion) { + if (deletion == null) { + return getNullSize(); + } + + return getTimestampSize() + + getByteArraySize(deletion.getSuper_column()) + + getSlicePredicateSize(deletion.getPredicate()); + } + + private static long getSlicePredicateSize(SlicePredicate predicate) { + if (predicate == null) { + return getNullSize(); + } + + return getCollectionSize(predicate.getColumn_names(), ThriftObjectSizeUtils::getByteBufferSize) + + getSliceRangeSize(predicate.getSlice_range()); + } + + private static long getSliceRangeSize(SliceRange sliceRange) { + if (sliceRange == null) { + return getNullSize(); + } + + return getByteArraySize(sliceRange.getStart()) + + getByteArraySize(sliceRange.getFinish()) + + getReversedBooleanSize() + + getSliceRangeCountSize(); + } + + private static long getCqlMetadataSize(CqlMetadata schema) { + if (schema == null) { + return getNullSize(); + } + + return getByteBufferStringMapSize(schema.getName_types()) + + getByteBufferStringMapSize(schema.getValue_types()) + + getStringSize(schema.getDefault_name_type()) + + getStringSize(schema.getDefault_value_type()); + } + + private static long getByteBufferStringMapSize(Map nameTypes) { + return getCollectionSize(nameTypes.entrySet(), + entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + + ThriftObjectSizeUtils.getStringSize(entry.getValue())); + } + + private static Long getCqlRowSize(CqlRow cqlRow) { + if (cqlRow == null) { + return getNullSize(); + } + return getByteArraySize(cqlRow.getKey()) + + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); + } + + private static long getThriftEnumSize() { + return Integer.BYTES; + } + + private static long getByteArraySize(byte[] byteArray) { + if (byteArray == null) { + return getNullSize(); + } + return byteArray.length; + } + + private static long getTimestampSize() { + return Long.BYTES; + } + + private static long getTtlSize() { + return Integer.BYTES; + } + + private static long getCounterValueSize() { + return Long.BYTES; + } + + private static long getReversedBooleanSize() { + return ONE_BYTE; + } + + private static long getSliceRangeCountSize() { + return Integer.BYTES; + } + + private static long getNullSize() { + return Integer.BYTES; + } + + private static long getCollectionSize(Collection collection, Function sizeFunction) { + if (collection == null) { + return getNullSize(); + } + + long sum = 0; + for (T item : collection) { + sum += sizeFunction.apply(item); + } + return sum; + } +} diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingCassandraClient.java new file mode 100644 index 00000000000..add06f5f0ee --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/TracingCassandraClient.java @@ -0,0 +1,152 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.thrift.CASResult; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.InvalidRequestException; +import org.apache.cassandra.thrift.KeyRange; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.NotFoundException; +import org.apache.cassandra.thrift.SchemaDisagreementException; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.TimedOutException; +import org.apache.cassandra.thrift.UnavailableException; +import org.apache.thrift.TException; + +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.logging.LoggingArgs; +import com.palantir.atlasdb.tracing.CloseableTrace; + +@SuppressWarnings({"all"}) // thrift variable names. +public class TracingCassandraClient implements CassandraClient { + private static final String SERVICE_NAME = "cassandra-thrift-client"; + + private final CassandraClient client; + + public TracingCassandraClient(CassandraClient client) { + this.client = client; + } + + @Override + public Cassandra.Client rawClient() { + return client.rawClient(); + } + + @Override + public Map> multiget_slice(String kvsMethodName, + TableReference tableRef, + List keys, + SlicePredicate predicate, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + int numberOfKeys = keys.size(); + int numberOfColumns = predicate.slice_range.count; + + try (CloseableTrace trace = startLocalTrace( + "client.multiget_slice(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", + LoggingArgs.safeTableOrPlaceholder(tableRef), + numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { + return client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level); + } + } + + @Override + public List get_range_slices(String kvsMethodName, + TableReference tableRef, + SlicePredicate predicate, + KeyRange range, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + int numberOfKeys = predicate.slice_range.count; + int numberOfColumns = range.count; + + try (CloseableTrace trace = startLocalTrace( + "client.get_range_slices(table {}, number of keys {}, number of columns {}, consistency {}) on kvs.{}", + LoggingArgs.safeTableOrPlaceholder(tableRef), + numberOfKeys, numberOfColumns, consistency_level, kvsMethodName)) { + return client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); + } + } + + @Override + public void batch_mutate(String kvsMethodName, + Map>> mutation_map, + ConsistencyLevel consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + int numberOfRowsMutated = mutation_map.size(); + + try (CloseableTrace trace = startLocalTrace("client.batch_mutate(number of mutations {}, consistency {})" + + " on kvs.{}", + numberOfRowsMutated, consistency_level, kvsMethodName)) { + client.batch_mutate(kvsMethodName, mutation_map, consistency_level); + } + } + + @Override + public ColumnOrSuperColumn get(TableReference tableReference, + ByteBuffer key, + byte[] column, + ConsistencyLevel consistency_level) + throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { + try (CloseableTrace trace = startLocalTrace("client.get(table {}, consistency {})", + LoggingArgs.safeTableOrPlaceholder(tableReference), consistency_level)) { + return client.get(tableReference, key, column, consistency_level); + } + } + + @Override + public CASResult cas(TableReference tableReference, + ByteBuffer key, + List expected, + List updates, + ConsistencyLevel serial_consistency_level, + ConsistencyLevel commit_consistency_level) + throws InvalidRequestException, UnavailableException, TimedOutException, TException { + try (CloseableTrace trace = startLocalTrace("client.cas(table {})", + LoggingArgs.safeTableOrPlaceholder(tableReference))) { + return client.cas(tableReference, key, expected, updates, serial_consistency_level, + commit_consistency_level); + } + } + + @Override + public CqlResult execute_cql3_query(CqlQuery cqlQuery, + Compression compression, + ConsistencyLevel consistency) + throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, + TException { + try (CloseableTrace trace = startLocalTrace("cqlExecutor.execute_cql3_query(query {})", + cqlQuery.getLazySafeLoggableObject())) { + return client.execute_cql3_query(cqlQuery, compression, consistency); + } + } + + private static CloseableTrace startLocalTrace(CharSequence operationFormat, Object... formatArguments) { + return CloseableTrace.startLocalTrace(SERVICE_NAME, operationFormat, formatArguments); + } +} diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java new file mode 100644 index 00000000000..a9f623361e0 --- /dev/null +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -0,0 +1,201 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlResultType; +import org.apache.cassandra.thrift.CqlRow; +import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SuperColumn; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.palantir.atlasdb.keyvalue.cassandra.ThriftObjectSizeUtils; + +public class ThriftObjectSizeUtilsTest { + + private static final String TEST_MAME = "test"; + private static final Column TEST_COLUMN = new Column(ByteBuffer.wrap(TEST_MAME.getBytes())); + + + private static final long TEST_COLUMN_SIZE = 4L + TEST_MAME.getBytes().length + 4L + 8L; + private static final ColumnOrSuperColumn EMPTY_COLUMN_OR_SUPERCOLUMN = new ColumnOrSuperColumn(); + private static final long EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE = Integer.BYTES * 4; + + @Test + public void returnEightForNullColumnOrSuperColumn() { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyColumnOrSuperColumn() { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(EMPTY_COLUMN_OR_SUPERCOLUMN)) + .isEqualTo(EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } + + @Test + public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn().setColumn(new Column()))) + .isEqualTo(Integer.BYTES * 8); + } + + @Test + public void getSizeForColumnOrSuperColumnWithANonEmptyColumn() { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn().setColumn(TEST_COLUMN))) + .isEqualTo(Integer.BYTES * 3 + TEST_COLUMN_SIZE); + } + + @Test + public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn() + .setColumn(TEST_COLUMN) + .setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), + ImmutableList.of(TEST_COLUMN))))) + .isEqualTo(Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); + } + + @Test + public void getSizeForNullByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(new byte[] {}))) + .isEqualTo(0L); + } + + @Test + public void getSizeForNonEmptyByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(TEST_MAME.getBytes()))) + .isEqualTo(TEST_MAME.getBytes().length); + } + + @Test + public void getSizeForNullCqlResult() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForVoidCqlResult() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize(new CqlResult(CqlResultType.VOID))) + .isEqualTo(Integer.BYTES * 4); + } + + @Test + public void getSizeForCqlResultWithRows() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize( + new CqlResult(CqlResultType.ROWS).setRows(ImmutableList.of(new CqlRow())))) + .isEqualTo(Integer.BYTES * 5); + } + + @Test + public void getSizeForNullMutation() { + assertThat(ThriftObjectSizeUtils.getMutationSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyMutation() { + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation())).isEqualTo(Integer.BYTES * 2); + } + + @Test + public void getSizeForMutationWithColumnOrSuperColumn() { + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setColumn_or_supercolumn(EMPTY_COLUMN_OR_SUPERCOLUMN))) + .isEqualTo(Integer.BYTES + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } + + @Test + public void getSizeForMutationWithEmptyDeletion() { + long emptyDeletionSize = Long.BYTES + 2 * Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion()))) + .isEqualTo(Integer.BYTES + emptyDeletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingSuperColumn() { + long nonEmptyDeletionSize = Long.BYTES + TEST_MAME.getBytes().length + Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion().setSuper_column(TEST_MAME.getBytes())))) + .isEqualTo(Integer.BYTES + nonEmptyDeletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingEmptySlicePredicate() { + long deletionSize = Long.BYTES + Integer.BYTES + Integer.BYTES * 2; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion().setPredicate(new SlicePredicate())))) + .isEqualTo(Integer.BYTES + deletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingNonEmptySlicePredicate() { + long deletionSize = (Long.BYTES) + (Integer.BYTES) + (TEST_MAME.getBytes().length + Integer.BYTES); + assertThat(ThriftObjectSizeUtils + .getMutationSize(new Mutation() + .setDeletion(new Deletion() + .setPredicate(new SlicePredicate() + .setColumn_names(ImmutableList.of(ByteBuffer.wrap(TEST_MAME.getBytes()))))))) + .isEqualTo(Integer.BYTES + deletionSize); + } + + @Test + public void getSizeForMutationWithColumnOrSuperColumnAndDeletion() { + long emptyDeletionSize = Long.BYTES + 2 * Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setColumn_or_supercolumn(EMPTY_COLUMN_OR_SUPERCOLUMN) + .setDeletion(new Deletion()))) + .isEqualTo(EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE + emptyDeletionSize); + } + + @Test + public void getSizeForNullKeySlice() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForKeySliceWithKeyNotSetButColumnsSet() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice() + .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)))) + .isEqualTo(Integer.BYTES + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } + + @Test + public void getSizeForKeySliceWithKeySetSetButColumnsNotSet() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice().setKey(TEST_MAME.getBytes()))) + .isEqualTo(Integer.BYTES + TEST_MAME.getBytes().length); + } + + @Test + public void getSizeForKeySliceWithKeyAndColumns() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice() + .setKey(TEST_MAME.getBytes()) + .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)))) + .isEqualTo(TEST_MAME.getBytes().length + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } +} diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java index 044b39c1bbf..23c7fa14e2a 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java @@ -83,6 +83,10 @@ private synchronized void registerMetricWithFqn(String fullyQualifiedMetricName, } } + public Meter registerMeter(Class clazz, String meterName) { + return registerMeter(MetricRegistry.name(clazz, "", meterName)); + } + public Meter registerMeter(Class clazz, String metricPrefix, String meterName) { return registerMeter(MetricRegistry.name(clazz, metricPrefix, meterName)); } From 22e129ab9b682922990ba2b3e50e6992642aa075 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Tue, 14 Nov 2017 20:22:53 +0000 Subject: [PATCH 24/67] use supplier for object size [no release notes] --- .../cassandra/QosCassandraClient.java | 25 ++++++++++--------- .../cassandra/ThriftObjectSizeUtils.java | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java index 9b8212a5005..59fb0e0dc54 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; import org.apache.cassandra.thrift.CASResult; import org.apache.cassandra.thrift.Cassandra; @@ -67,7 +68,7 @@ public Map> multiget_slice(String kvsMetho throws InvalidRequestException, UnavailableException, TimedOutException, TException { Map> result = client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level); - recordBytesRead(getApproximateReadByteCount(result)); + recordBytesRead(() -> getApproximateReadByteCount(result)); return result; } @@ -83,7 +84,7 @@ public List get_range_slices(String kvsMethodName, TableReference tabl KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { List result = client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); - recordBytesRead(getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); + recordBytesRead(() -> getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); return result; } @@ -92,7 +93,7 @@ public void batch_mutate(String kvsMethodName, Map getApproximateWriteByteCount(mutation_map)); } private long getApproximateWriteByteCount(Map>> batchMutateMap) { @@ -109,7 +110,7 @@ public ColumnOrSuperColumn get(TableReference tableReference, ByteBuffer key, by ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { ColumnOrSuperColumn result = client.get(tableReference, key, column, consistency_level); - recordBytesRead(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(result)); + recordBytesRead(() -> ThriftObjectSizeUtils.getColumnOrSuperColumnSize(result)); return result; } @@ -119,8 +120,8 @@ public CASResult cas(TableReference tableReference, ByteBuffer key, List throws InvalidRequestException, UnavailableException, TimedOutException, TException { CASResult result = client.cas(tableReference, key, expected, updates, serial_consistency_level, commit_consistency_level); - recordBytesWritten(updates.stream().mapToLong(ThriftObjectSizeUtils::getColumnSize).sum()); - recordBytesRead(expected.stream().mapToLong(ThriftObjectSizeUtils::getColumnSize).sum()); + recordBytesWritten(() -> getCollectionSize(updates, ThriftObjectSizeUtils::getColumnSize)); + recordBytesRead(() -> getCollectionSize(updates, ThriftObjectSizeUtils::getColumnSize)); return result; } @@ -129,29 +130,29 @@ public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { CqlResult cqlResult = client.execute_cql3_query(cqlQuery, compression, consistency); - recordBytesRead(ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); + recordBytesRead(() -> ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); return cqlResult; } - private void recordBytesRead(long numBytesRead) { + private void recordBytesRead(Supplier numBytesRead) { try { qosMetrics.updateReadCount(); - qosMetrics.updateBytesRead(numBytesRead); + qosMetrics.updateBytesRead(numBytesRead.get()); } catch (Exception e) { log.warn("Encountered an exception when recording read metrics.", e); } } - private void recordBytesWritten(long numBytesWritten) { + private void recordBytesWritten(Supplier numBytesWritten) { try { qosMetrics.updateWriteCount(); - qosMetrics.updateBytesWritten(numBytesWritten); + qosMetrics.updateBytesWritten(numBytesWritten.get()); } catch (Exception e) { log.warn("Encountered an exception when recording write metrics.", e); } } private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { - return collection.stream().mapToLong(singleObjectSizeFunction::apply).sum(); + return ThriftObjectSizeUtils.getCollectionSize(collection, singleObjectSizeFunction); } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 8917cfcc9b5..b973bfc707f 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -223,7 +223,7 @@ private static long getNullSize() { return Integer.BYTES; } - private static long getCollectionSize(Collection collection, Function sizeFunction) { + public static long getCollectionSize(Collection collection, Function sizeFunction) { if (collection == null) { return getNullSize(); } From a45809d3e1cc2ef5ccc0fd62f94c955170673952 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Tue, 14 Nov 2017 23:44:10 +0000 Subject: [PATCH 25/67] fix merge in AtlasDbConfig --- .../main/java/com/palantir/atlasdb/config/AtlasDbConfig.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java index 42db08ce61e..dd72ed9de69 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbConfig.java @@ -330,11 +330,6 @@ private String checkNamespaceConfigAndGetNamespace() { + " DO NOT ATTEMPT TO FIX THIS YOURSELF."); } return keyValueServiceNamespace; - } else if (timelock().isPresent()) { - // Special case - empty timelock and empty namespace/keyspace does not make sense - Preconditions.checkState(timelock().get().client().isPresent(), - "For InMemoryKVS, the TimeLock client should not be empty"); - return timelock().get().client().get(); } else { Preconditions.checkState(keyValueService() instanceof InMemoryAtlasDbConfig, "Expecting KeyvalueServiceConfig to be instance of InMemoryAtlasDbConfig, found %s", From e3bd685140b230fd24f602fc13e41ee66e28d86f Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Wed, 15 Nov 2017 10:36:41 +0000 Subject: [PATCH 26/67] rate limiting --- .../palantir/atlasdb/spi/AtlasDbFactory.java | 2 +- .../cassandra/CassandraClientPoolImpl.java | 4 +- .../CassandraExpiringKeyValueService.java | 2 +- .../CassandraKeyValueServiceImpl.java | 4 +- .../cassandra/QosCassandraClient.java | 88 ++-- .../cassandra/ThriftObjectSizeUtils.java | 54 +-- .../cassandra/QosCassandraClientTest.java | 9 +- atlasdb-cli-distribution/versions.lock | 1 + atlasdb-cli/versions.lock | 2 + .../ServiceDiscoveringAtlasSupplier.java | 4 +- .../atlasdb/factory/TransactionManagers.java | 6 +- atlasdb-config/versions.lock | 6 +- atlasdb-console-distribution/versions.lock | 1 + atlasdb-console/versions.lock | 6 +- .../atlasdb/services/ServicesConfig.java | 2 +- atlasdb-dagger/versions.lock | 6 +- atlasdb-dropwizard-bundle/versions.lock | 2 + atlasdb-ete-tests/versions.lock | 2 + atlasdb-jepsen-tests/versions.lock | 6 +- atlasdb-perf/versions.lock | 2 + atlasdb-service-server/versions.lock | 4 +- atlasdb-service/versions.lock | 6 +- .../palantir/atlasdb/qos/FakeQosClient.java | 16 +- .../com/palantir/atlasdb/qos/QosClient.java | 21 +- .../com/palantir/atlasdb/qos/QosService.java | 2 +- qos-service-impl/build.gradle | 4 + .../com/palantir/atlasdb/qos}/QosMetrics.java | 2 +- .../com/palantir/atlasdb/qos/QosResource.java | 4 +- .../atlasdb/qos/client/AtlasDbQosClient.java | 82 ++-- .../qos/config/QosServiceRuntimeConfig.java | 2 +- .../qos/ratelimit/SmoothRateLimiter.java | 4 +- .../atlasdb/qos/AtlasDbQosClientTest.java | 66 ++- .../QosRuntimeConfigDeserializationTest.java | 2 +- .../qos/QosServiceRuntimeConfigTest.java | 4 +- .../palantir/atlasdb/qos/QosServiceTest.java | 12 +- qos-service-impl/versions.lock | 430 +++++++++++++++++- timelock-agent/versions.lock | 2 + timelock-impl/versions.lock | 2 + timelock-server-distribution/versions.lock | 1 + timelock-server/versions.lock | 2 + 40 files changed, 679 insertions(+), 198 deletions(-) rename {atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra => qos-service-impl/src/main/java/com/palantir/atlasdb/qos}/QosMetrics.java (97%) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/spi/AtlasDbFactory.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/spi/AtlasDbFactory.java index c85ca3c1466..28942d771cb 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/spi/AtlasDbFactory.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/spi/AtlasDbFactory.java @@ -37,7 +37,7 @@ public interface AtlasDbFactory { default KeyValueService createRawKeyValueService( KeyValueServiceConfig config, Optional leaderConfig) { return createRawKeyValueService(config, leaderConfig, Optional.empty(), DEFAULT_INITIALIZE_ASYNC, - FakeQosClient.getDefault()); + FakeQosClient.INSTANCE); } /** diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java index 5125de764c1..c4f020280ff 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolImpl.java @@ -164,11 +164,11 @@ public void shutdown() { @VisibleForTesting static CassandraClientPoolImpl createImplForTest(CassandraKeyValueServiceConfig config, StartupChecks startupChecks) { - return create(config, startupChecks, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, FakeQosClient.getDefault()); + return create(config, startupChecks, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, FakeQosClient.INSTANCE); } public static CassandraClientPool create(CassandraKeyValueServiceConfig config) { - return create(config, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, FakeQosClient.getDefault()); + return create(config, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, FakeQosClient.INSTANCE); } public static CassandraClientPool create(CassandraKeyValueServiceConfig config, boolean initializeAsync, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java index dd66214bce2..54ab0a5b292 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraExpiringKeyValueService.java @@ -69,7 +69,7 @@ private CassandraExpiringKeyValueService( Optional leaderConfig, boolean initializeAsync) { super(LoggerFactory.getLogger(CassandraKeyValueService.class), configManager, compactionManager, leaderConfig, - initializeAsync, FakeQosClient.getDefault()); + initializeAsync, FakeQosClient.INSTANCE); } @Override diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java index c4bece52c64..9c5fe033a11 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraKeyValueServiceImpl.java @@ -224,7 +224,7 @@ public static CassandraKeyValueService create( CassandraKeyValueServiceConfigManager configManager, Optional leaderConfig, boolean initializeAsync) { - return create(configManager, leaderConfig, initializeAsync, FakeQosClient.getDefault()); + return create(configManager, leaderConfig, initializeAsync, FakeQosClient.INSTANCE); } public static CassandraKeyValueService create( @@ -245,7 +245,7 @@ static CassandraKeyValueService create( Optional leaderConfig, Logger log) { return create(configManager, leaderConfig, log, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, - FakeQosClient.getDefault()); + FakeQosClient.INSTANCE); } private static CassandraKeyValueService create( diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java index bfaec55afda..18d60ca8f13 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.function.Supplier; import org.apache.cassandra.thrift.CASResult; import org.apache.cassandra.thrift.Cassandra; @@ -48,16 +47,17 @@ @SuppressWarnings({"all"}) // thrift variable names. public class QosCassandraClient implements CassandraClient { - private final Logger log = LoggerFactory.getLogger(CassandraClient.class); + + private static final int DEFAULT_ESTIMATED_READ_BYTES = 100; + + private static final Logger log = LoggerFactory.getLogger(CassandraClient.class); private final CassandraClient client; - private final QosMetrics qosMetrics; private final QosClient qosClient; public QosCassandraClient(CassandraClient client, QosClient qosClient) { this.client = client; this.qosClient = qosClient; - qosMetrics = new QosMetrics(); } @Override @@ -69,15 +69,14 @@ public Cassandra.Client rawClient() { public Map> multiget_slice(String kvsMethodName, TableReference tableRef, List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - qosClient.checkLimit(); - - Map> result = client.multiget_slice(kvsMethodName, tableRef, keys, - predicate, consistency_level); - recordBytesRead(() -> getApproximateReadByteCount(result)); - return result; + return qosClient.executeRead( + () -> DEFAULT_ESTIMATED_READ_BYTES, + () -> client.multiget_slice(kvsMethodName, tableRef, keys, + predicate, consistency_level), + this::getApproximateReadByteCount); } - private long getApproximateReadByteCount(Map> result) { + private int getApproximateReadByteCount(Map> result) { return getCollectionSize(result.entrySet(), rowResult -> ThriftObjectSizeUtils.getByteBufferSize(rowResult.getKey()) + getCollectionSize(rowResult.getValue(), @@ -88,26 +87,24 @@ private long getApproximateReadByteCount(Map get_range_slices(String kvsMethodName, TableReference tableRef, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - qosClient.checkLimit(); - - List result = client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); - recordBytesRead(() -> getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); - return result; + return qosClient.executeRead( + () -> DEFAULT_ESTIMATED_READ_BYTES, + () -> client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level), + result -> getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); } @Override public void batch_mutate(String kvsMethodName, Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - qosClient.checkLimit(); - - client.batch_mutate(kvsMethodName, mutation_map, consistency_level); - recordBytesWritten(() -> getApproximateWriteByteCount(mutation_map)); + qosClient.executeWrite( + () -> getApproximateWriteByteCount(mutation_map), + () -> client.batch_mutate(kvsMethodName, mutation_map, consistency_level)); } - private long getApproximateWriteByteCount(Map>> batchMutateMap) { - long approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); - long approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> + private int getApproximateWriteByteCount(Map>> batchMutateMap) { + int approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); + int approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + getCollectionSize(currentMap.values(), mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); @@ -118,56 +115,33 @@ private long getApproximateWriteByteCount(Map ThriftObjectSizeUtils.getColumnOrSuperColumnSize(result)); - return result; + return qosClient.executeRead( + () -> DEFAULT_ESTIMATED_READ_BYTES, + () -> client.get(tableReference, key, column, consistency_level), + ThriftObjectSizeUtils::getColumnOrSuperColumnSize); } @Override public CASResult cas(TableReference tableReference, ByteBuffer key, List expected, List updates, ConsistencyLevel serial_consistency_level, ConsistencyLevel commit_consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - qosClient.checkLimit(); - - CASResult result = client.cas(tableReference, key, expected, updates, serial_consistency_level, + // TODO(nziebart): should this be considered as a write or do we need to treat is as both read and write? + return client.cas(tableReference, key, expected, updates, serial_consistency_level, commit_consistency_level); - recordBytesWritten(() -> getCollectionSize(updates, ThriftObjectSizeUtils::getColumnSize)); - recordBytesRead(() -> getCollectionSize(updates, ThriftObjectSizeUtils::getColumnSize)); - return result; } @Override public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { - qosClient.checkLimit(); - - CqlResult cqlResult = client.execute_cql3_query(cqlQuery, compression, consistency); - recordBytesRead(() -> ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); - return cqlResult; - } - - private void recordBytesRead(Supplier numBytesRead) { - try { - qosMetrics.updateReadCount(); - qosMetrics.updateBytesRead(numBytesRead.get()); - } catch (Exception e) { - log.warn("Encountered an exception when recording read metrics.", e); - } - } - private void recordBytesWritten(Supplier numBytesWritten) { - try { - qosMetrics.updateWriteCount(); - qosMetrics.updateBytesWritten(numBytesWritten.get()); - } catch (Exception e) { - log.warn("Encountered an exception when recording write metrics.", e); - } + return qosClient.executeRead( + () -> DEFAULT_ESTIMATED_READ_BYTES, + () -> client.execute_cql3_query(cqlQuery, compression, consistency), + ThriftObjectSizeUtils::getCqlResultSize); } - private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { + private int getCollectionSize(Collection collection, Function singleObjectSizeFunction) { return ThriftObjectSizeUtils.getCollectionSize(collection, singleObjectSizeFunction); } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index b973bfc707f..f5f95e0a2a0 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -37,13 +37,13 @@ public final class ThriftObjectSizeUtils { - private static final long ONE_BYTE = 1L; + private static final int ONE_BYTE = 1; private ThriftObjectSizeUtils() { // utility class } - public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { + public static int getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { if (columnOrSuperColumn == null) { return getNullSize(); } @@ -53,14 +53,14 @@ public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperC + getCounterSuperColumnSize(columnOrSuperColumn.getCounter_super_column()); } - public static long getByteBufferSize(ByteBuffer byteBuffer) { + public static int getByteBufferSize(ByteBuffer byteBuffer) { if (byteBuffer == null) { return getNullSize(); } return byteBuffer.remaining(); } - public static long getMutationSize(Mutation mutation) { + public static int getMutationSize(Mutation mutation) { if (mutation == null) { return getNullSize(); } @@ -69,7 +69,7 @@ public static long getMutationSize(Mutation mutation) { mutation.getDeletion()); } - public static long getCqlResultSize(CqlResult cqlResult) { + public static int getCqlResultSize(CqlResult cqlResult) { if (cqlResult == null) { return getNullSize(); } @@ -79,7 +79,7 @@ public static long getCqlResultSize(CqlResult cqlResult) { + getCqlMetadataSize(cqlResult.getSchema()); } - public static long getKeySliceSize(KeySlice keySlice) { + public static int getKeySliceSize(KeySlice keySlice) { if (keySlice == null) { return getNullSize(); } @@ -88,7 +88,7 @@ public static long getKeySliceSize(KeySlice keySlice) { + getCollectionSize(keySlice.getColumns(), ThriftObjectSizeUtils::getColumnOrSuperColumnSize); } - public static long getStringSize(String string) { + public static int getStringSize(String string) { if (string == null) { return getNullSize(); } @@ -96,7 +96,7 @@ public static long getStringSize(String string) { return string.length() * Character.SIZE; } - public static long getColumnSize(Column column) { + public static int getColumnSize(Column column) { if (column == null) { return getNullSize(); } @@ -107,7 +107,7 @@ public static long getColumnSize(Column column) { + getTimestampSize(); } - private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { + private static int getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { if (counterSuperColumn == null) { return getNullSize(); } @@ -116,7 +116,7 @@ private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperCol + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); } - private static long getCounterColumnSize(CounterColumn counterColumn) { + private static int getCounterColumnSize(CounterColumn counterColumn) { if (counterColumn == null) { return getNullSize(); } @@ -124,7 +124,7 @@ private static long getCounterColumnSize(CounterColumn counterColumn) { return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); } - private static long getSuperColumnSize(SuperColumn superColumn) { + private static int getSuperColumnSize(SuperColumn superColumn) { if (superColumn == null) { return getNullSize(); } @@ -133,7 +133,7 @@ private static long getSuperColumnSize(SuperColumn superColumn) { + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - private static long getDeletionSize(Deletion deletion) { + private static int getDeletionSize(Deletion deletion) { if (deletion == null) { return getNullSize(); } @@ -143,7 +143,7 @@ private static long getDeletionSize(Deletion deletion) { + getSlicePredicateSize(deletion.getPredicate()); } - private static long getSlicePredicateSize(SlicePredicate predicate) { + private static int getSlicePredicateSize(SlicePredicate predicate) { if (predicate == null) { return getNullSize(); } @@ -152,7 +152,7 @@ private static long getSlicePredicateSize(SlicePredicate predicate) { + getSliceRangeSize(predicate.getSlice_range()); } - private static long getSliceRangeSize(SliceRange sliceRange) { + private static int getSliceRangeSize(SliceRange sliceRange) { if (sliceRange == null) { return getNullSize(); } @@ -163,7 +163,7 @@ private static long getSliceRangeSize(SliceRange sliceRange) { + getSliceRangeCountSize(); } - private static long getCqlMetadataSize(CqlMetadata schema) { + private static int getCqlMetadataSize(CqlMetadata schema) { if (schema == null) { return getNullSize(); } @@ -174,13 +174,13 @@ private static long getCqlMetadataSize(CqlMetadata schema) { + getStringSize(schema.getDefault_value_type()); } - private static long getByteBufferStringMapSize(Map nameTypes) { + private static int getByteBufferStringMapSize(Map nameTypes) { return getCollectionSize(nameTypes.entrySet(), entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + ThriftObjectSizeUtils.getStringSize(entry.getValue())); } - private static Long getCqlRowSize(CqlRow cqlRow) { + private static int getCqlRowSize(CqlRow cqlRow) { if (cqlRow == null) { return getNullSize(); } @@ -188,47 +188,47 @@ private static Long getCqlRowSize(CqlRow cqlRow) { + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - private static long getThriftEnumSize() { + private static int getThriftEnumSize() { return Integer.BYTES; } - private static long getByteArraySize(byte[] byteArray) { + private static int getByteArraySize(byte[] byteArray) { if (byteArray == null) { return getNullSize(); } return byteArray.length; } - private static long getTimestampSize() { + private static int getTimestampSize() { return Long.BYTES; } - private static long getTtlSize() { + private static int getTtlSize() { return Integer.BYTES; } - private static long getCounterValueSize() { + private static int getCounterValueSize() { return Long.BYTES; } - private static long getReversedBooleanSize() { + private static int getReversedBooleanSize() { return ONE_BYTE; } - private static long getSliceRangeCountSize() { + private static int getSliceRangeCountSize() { return Integer.BYTES; } - private static long getNullSize() { + private static int getNullSize() { return Integer.BYTES; } - public static long getCollectionSize(Collection collection, Function sizeFunction) { + public static int getCollectionSize(Collection collection, Function sizeFunction) { if (collection == null) { return getNullSize(); } - long sum = 0; + int sum = 0; for (T item : collection) { sum += sizeFunction.apply(item); } diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java index 3a0280c793e..38d762b3563 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.keyvalue.cassandra; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -61,7 +62,7 @@ public void setUp() { public void multigetSliceChecksLimit() throws TException, LimitExceededException { client.multiget_slice("get", TEST_TABLE, ImmutableList.of(ROW_KEY), SLICE_PREDICATE, ConsistencyLevel.ANY); - verify(qosClient, times(1)).checkLimit(); + verify(qosClient, times(1)).executeRead(any(), any(), any()); verifyNoMoreInteractions(qosClient); } @@ -69,7 +70,7 @@ public void multigetSliceChecksLimit() throws TException, LimitExceededException public void batchMutateChecksLimit() throws TException, LimitExceededException { client.batch_mutate("put", ImmutableMap.of(), ConsistencyLevel.ANY); - verify(qosClient, times(1)).checkLimit(); + verify(qosClient, times(1)).executeWrite(any(), any()); verifyNoMoreInteractions(qosClient); } @@ -78,7 +79,7 @@ public void executeCqlQueryChecksLimit() throws TException, LimitExceededExcepti CqlQuery query = new CqlQuery("SELECT * FROM test_table LIMIT 1"); client.execute_cql3_query(query, Compression.NONE, ConsistencyLevel.ANY); - verify(qosClient, times(1)).checkLimit(); + verify(qosClient, times(1)).executeRead(any(), any(), any()); verifyNoMoreInteractions(qosClient); } @@ -86,7 +87,7 @@ public void executeCqlQueryChecksLimit() throws TException, LimitExceededExcepti public void getRangeSlicesChecksLimit() throws TException, LimitExceededException { client.get_range_slices("get", TEST_TABLE, SLICE_PREDICATE, new KeyRange(), ConsistencyLevel.ANY); - verify(qosClient, times(1)).checkLimit(); + verify(qosClient, times(1)).executeRead(any(), any(), any()); verifyNoMoreInteractions(qosClient); } } diff --git a/atlasdb-cli-distribution/versions.lock b/atlasdb-cli-distribution/versions.lock index 828f6c32d38..a74e62d2a0a 100644 --- a/atlasdb-cli-distribution/versions.lock +++ b/atlasdb-cli-distribution/versions.lock @@ -380,6 +380,7 @@ "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-cli/versions.lock b/atlasdb-cli/versions.lock index bff4b7e79a3..ce547d2201e 100644 --- a/atlasdb-cli/versions.lock +++ b/atlasdb-cli/versions.lock @@ -343,6 +343,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1344,6 +1345,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceDiscoveringAtlasSupplier.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceDiscoveringAtlasSupplier.java index 180551f6e89..bbb63f61056 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceDiscoveringAtlasSupplier.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/ServiceDiscoveringAtlasSupplier.java @@ -59,7 +59,7 @@ public class ServiceDiscoveringAtlasSupplier { public ServiceDiscoveringAtlasSupplier(KeyValueServiceConfig config, Optional leaderConfig) { this(config, leaderConfig, Optional.empty(), AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, - FakeQosClient.getDefault()); + FakeQosClient.INSTANCE); } public ServiceDiscoveringAtlasSupplier( @@ -68,7 +68,7 @@ public ServiceDiscoveringAtlasSupplier( Optional namespace, Optional timestampTable) { this(config, leaderConfig, namespace, timestampTable, AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC, - FakeQosClient.getDefault()); + FakeQosClient.INSTANCE); } public ServiceDiscoveringAtlasSupplier( diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java index c1dd7003aaf..68e2b8ecf66 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java @@ -74,6 +74,7 @@ import com.palantir.atlasdb.persistentlock.PersistentLockService; import com.palantir.atlasdb.qos.FakeQosClient; import com.palantir.atlasdb.qos.QosClient; +import com.palantir.atlasdb.qos.QosMetrics; import com.palantir.atlasdb.qos.QosService; import com.palantir.atlasdb.qos.client.AtlasDbQosClient; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; @@ -412,18 +413,19 @@ SerializableTransactionManager serializable() { } private QosClient getQosClient(Optional serviceConfiguration) { - return serviceConfiguration.map(this::createAtlasDbQosClient).orElseGet(FakeQosClient::getDefault); + return serviceConfiguration.map(this::createAtlasDbQosClient).orElse(FakeQosClient.INSTANCE); } private QosClient createAtlasDbQosClient(ServiceConfiguration serviceConfiguration) { QosService qosService = JaxRsClient.create(QosService.class, userAgent(), ClientConfigurations.of(serviceConfiguration)); + // TODO(nziebart): create a RefreshingRateLimiter ScheduledExecutorService scheduler = new InstrumentedScheduledExecutorService( Executors.newSingleThreadScheduledExecutor(), AtlasDbMetrics.getMetricRegistry(), "qos-client-executor"); - return new AtlasDbQosClient(qosService, scheduler, config().getNamespaceString(), QosRateLimiter.create()); + return new AtlasDbQosClient(QosRateLimiter.create(), new QosMetrics()); } private static boolean areTransactionManagerInitializationPrerequisitesSatisfied( diff --git a/atlasdb-config/versions.lock b/atlasdb-config/versions.lock index a6177b13f9c..b383fa0f8a6 100644 --- a/atlasdb-config/versions.lock +++ b/atlasdb-config/versions.lock @@ -292,7 +292,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1047,7 +1048,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { diff --git a/atlasdb-console-distribution/versions.lock b/atlasdb-console-distribution/versions.lock index 16d9e08488b..a32f59f977b 100644 --- a/atlasdb-console-distribution/versions.lock +++ b/atlasdb-console-distribution/versions.lock @@ -371,6 +371,7 @@ "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-console/versions.lock b/atlasdb-console/versions.lock index 914f308086e..0fb9da34bf4 100644 --- a/atlasdb-console/versions.lock +++ b/atlasdb-console/versions.lock @@ -300,7 +300,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1130,7 +1131,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { diff --git a/atlasdb-dagger/src/main/java/com/palantir/atlasdb/services/ServicesConfig.java b/atlasdb-dagger/src/main/java/com/palantir/atlasdb/services/ServicesConfig.java index f25bc8dd431..eeafa07a2dc 100644 --- a/atlasdb-dagger/src/main/java/com/palantir/atlasdb/services/ServicesConfig.java +++ b/atlasdb-dagger/src/main/java/com/palantir/atlasdb/services/ServicesConfig.java @@ -43,7 +43,7 @@ public ServiceDiscoveringAtlasSupplier atlasDbSupplier() { atlasDbConfig().leader(), atlasDbConfig().namespace(), atlasDbConfig().initializeAsync(), - FakeQosClient.getDefault()); + FakeQosClient.INSTANCE); } @Value.Default diff --git a/atlasdb-dagger/versions.lock b/atlasdb-dagger/versions.lock index 7134077d838..478aea18939 100644 --- a/atlasdb-dagger/versions.lock +++ b/atlasdb-dagger/versions.lock @@ -303,7 +303,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1123,7 +1124,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { diff --git a/atlasdb-dropwizard-bundle/versions.lock b/atlasdb-dropwizard-bundle/versions.lock index 58dd032a0ab..672160c0a0d 100644 --- a/atlasdb-dropwizard-bundle/versions.lock +++ b/atlasdb-dropwizard-bundle/versions.lock @@ -367,6 +367,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1816,6 +1817,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-ete-tests/versions.lock b/atlasdb-ete-tests/versions.lock index 00bb8f05c9f..5ff819ae459 100644 --- a/atlasdb-ete-tests/versions.lock +++ b/atlasdb-ete-tests/versions.lock @@ -375,6 +375,7 @@ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1908,6 +1909,7 @@ "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-jepsen-tests/versions.lock b/atlasdb-jepsen-tests/versions.lock index 53dd704b203..37d043c5f17 100644 --- a/atlasdb-jepsen-tests/versions.lock +++ b/atlasdb-jepsen-tests/versions.lock @@ -299,7 +299,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1106,7 +1107,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { diff --git a/atlasdb-perf/versions.lock b/atlasdb-perf/versions.lock index 823e560e1c9..5e918b5c7b8 100644 --- a/atlasdb-perf/versions.lock +++ b/atlasdb-perf/versions.lock @@ -392,6 +392,7 @@ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1581,6 +1582,7 @@ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-service-server/versions.lock b/atlasdb-service-server/versions.lock index ff82b5dba67..0545fbde7c8 100644 --- a/atlasdb-service-server/versions.lock +++ b/atlasdb-service-server/versions.lock @@ -319,7 +319,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1602,6 +1603,7 @@ "transitive": [ "com.palantir.atlasdb:atlasdb-cassandra", "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/atlasdb-service/versions.lock b/atlasdb-service/versions.lock index 2fa0cb5920f..b1271c749cd 100644 --- a/atlasdb-service/versions.lock +++ b/atlasdb-service/versions.lock @@ -299,7 +299,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { @@ -1104,7 +1105,8 @@ "com.palantir.atlasdb:atlasdb-client": { "project": true, "transitive": [ - "com.palantir.atlasdb:atlasdb-impl-shared" + "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl" ] }, "com.palantir.atlasdb:atlasdb-client-protobufs": { diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java index 584b4422b1d..4b24852a266 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java @@ -16,16 +16,22 @@ package com.palantir.atlasdb.qos; +import java.util.function.Function; +import java.util.function.Supplier; + public class FakeQosClient implements QosClient { - private static final FakeQosClient DEFAULT = new FakeQosClient(); + public static final FakeQosClient INSTANCE = new FakeQosClient(); - public static FakeQosClient getDefault() { - return DEFAULT; + @Override + public T executeRead(Supplier estimatedWeight, ReadQuery query, + Function weigher) throws E { + return query.execute(); } @Override - public void checkLimit() { - // no op + public void executeWrite(Supplier weight, WriteQuery query) + throws E { + query.execute(); } } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java index c31757ad665..26d85e9d4c0 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java @@ -16,6 +16,25 @@ package com.palantir.atlasdb.qos; +import java.util.function.Function; +import java.util.function.Supplier; + public interface QosClient { - void checkLimit(); + + interface ReadQuery { + T execute() throws E; + } + + interface WriteQuery { + void execute() throws E; + } + + T executeRead( + Supplier estimatedWeight, + ReadQuery query, + Function weigher) throws E; + + void executeWrite( + Supplier weight, + WriteQuery query) throws E; } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java index 1dced841626..469caed1427 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java @@ -30,5 +30,5 @@ public interface QosService { @POST @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - long getLimit(@Safe @PathParam("client") String client); + int getLimit(@Safe @PathParam("client") String client); } diff --git a/qos-service-impl/build.gradle b/qos-service-impl/build.gradle index 014b66a60d8..91fe725b45c 100644 --- a/qos-service-impl/build.gradle +++ b/qos-service-impl/build.gradle @@ -16,6 +16,10 @@ jacocoTestReport { check.dependsOn integTest dependencies { compile (project(":qos-service-api")); + compile (project(":atlasdb-client")) { + exclude group: 'com.squareup.okhttp3' + exclude group: 'com.google.guava' + } processor group: 'org.immutables', name: 'value' diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java similarity index 97% rename from atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java rename to qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java index 23b3d9e58a8..1da9ae591ab 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.palantir.atlasdb.keyvalue.cassandra; +package com.palantir.atlasdb.qos; import com.codahale.metrics.Meter; import com.palantir.atlasdb.util.MetricsManager; diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java index ed199b69e4b..b7443421cfb 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java @@ -29,7 +29,7 @@ public QosResource(Supplier config) { } @Override - public long getLimit(String client) { - return config.get().clientLimits().getOrDefault(client, Long.MAX_VALUE); + public int getLimit(String client) { + return config.get().clientLimits().getOrDefault(client, Integer.MAX_VALUE); } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index 67c77542761..cedf7253329 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -14,48 +14,68 @@ * limitations under the License. */ package com.palantir.atlasdb.qos.client; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; + +import java.util.function.Function; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.palantir.atlasdb.qos.QosClient; -import com.palantir.atlasdb.qos.QosService; +import com.palantir.atlasdb.qos.QosMetrics; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; public class AtlasDbQosClient implements QosClient { - private final QosService qosService; - private final String clientName; - private final QosRateLimiter rateLimiter; - private volatile long credits; + private static final Logger log = LoggerFactory.getLogger(AtlasDbQosClient.class); + + private final QosRateLimiter rateLimiter; + private final QosMetrics metrics; - public AtlasDbQosClient(QosService qosService, - ScheduledExecutorService limitRefresher, - String clientName, - QosRateLimiter rateLimiter) { - this.qosService = qosService; - this.clientName = clientName; + public AtlasDbQosClient(QosRateLimiter rateLimiter, QosMetrics metrics) { + this.metrics = metrics; this.rateLimiter = rateLimiter; - limitRefresher.scheduleAtFixedRate(() -> { - try { - credits = qosService.getLimit(clientName); - } catch (Exception e) { - // do nothing - } - }, 0L, 60L, TimeUnit.SECONDS); } - // The KVS layer should call this before every read/write operation - // Currently all operations are treated equally; each uses up a unit of credits @Override - public void checkLimit() { - // always return immediately - i.e. no backoff - // TODO if soft-limited, pause - // if hard-limited, throw exception - if (credits > 0) { - credits--; - } else { - // TODO This should be a ThrottleException? - throw new RuntimeException("Rate limit exceeded"); + public T executeRead( + Supplier estimatedWeigher, + ReadQuery query, + Function weigher) throws E { + int estimatedWeight = getWeight(estimatedWeigher, 1); + rateLimiter.consumeWithBackoff(estimatedWeight); + + // TODO(nziebart): decide what to do if we encounter a timeout exception + T result = query.execute(); + + int actualWeight = getWeight(() -> weigher.apply(result), estimatedWeight); + metrics.updateReadCount(); + metrics.updateBytesRead(actualWeight); + rateLimiter.recordAdjustment(actualWeight - estimatedWeight); + + return result; + } + + @Override + public void executeWrite(Supplier weigher, WriteQuery query) throws E { + int weight = getWeight(weigher, 1); + rateLimiter.consumeWithBackoff(weight); + + // TODO(nziebart): decide what to do if we encounter a timeout exception + query.execute(); + + metrics.updateWriteCount(); + metrics.updateBytesWritten(weight); + } + + // TODO(nziebart): error handling in the weight calculation should be responsibility of the caller + private Integer getWeight(Supplier weigher, int fallback) { + try { + return weigher.get(); + } catch (Exception e) { + log.warn("Exception while calculating response weight", e); + return fallback; } } + } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java index 8ce8305589d..0faa8222f34 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java @@ -27,5 +27,5 @@ @JsonSerialize(as = ImmutableQosServiceRuntimeConfig.class) @Value.Immutable public abstract class QosServiceRuntimeConfig { - public abstract Map clientLimits(); + public abstract Map clientLimits(); } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java index 465b0141152..c80f20f1de5 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java @@ -20,8 +20,6 @@ import static java.lang.Math.min; import static java.util.concurrent.TimeUnit.SECONDS; -import com.google.common.math.LongMath; - /** * Copied from Guava, because {@link SmoothBursty} is a package private class. **/ @@ -123,7 +121,7 @@ final long reserveEarliestAvailable(int requiredPermits, long nowMicros) { storedPermitsToWaitTime(this.storedPermits, storedPermitsToSpend) + (long) (freshPermits * stableIntervalMicros); - this.nextFreeTicketMicros = LongMath.saturatedAdd(nextFreeTicketMicros, waitMicros); + this.nextFreeTicketMicros = nextFreeTicketMicros + waitMicros; this.storedPermits -= storedPermitsToSpend; return returnValue; } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java index 1c5b08ad46d..e7b7b3ef8c1 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java @@ -18,11 +18,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import java.util.concurrent.TimeUnit; - -import org.jmock.lib.concurrent.DeterministicScheduler; import org.junit.Before; import org.junit.Test; @@ -30,42 +29,65 @@ import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; public class AtlasDbQosClientTest { + + private static final int ESTIMATED_BYTES = 10; + private static final int ACTUAL_BYTES = 51; + private QosService qosService = mock(QosService.class); - private DeterministicScheduler scheduler = new DeterministicScheduler(); private QosRateLimiter rateLimiter = mock(QosRateLimiter.class); + private QosMetrics metrics = mock(QosMetrics.class); + + private AtlasDbQosClient qosClient = new AtlasDbQosClient(rateLimiter, metrics); @Before public void setUp() { - when(qosService.getLimit("test-client")).thenReturn(1L); + when(qosService.getLimit("test-client")).thenReturn(100); + } + + @Test + public void consumesSpecifiedNumUnitsForReads() { + qosClient.executeRead(() -> ESTIMATED_BYTES, () -> "foo", ignored -> ACTUAL_BYTES); + + verify(rateLimiter).consumeWithBackoff(ESTIMATED_BYTES); + verify(rateLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); + verifyNoMoreInteractions(rateLimiter); } @Test - public void doesNotBackOff() { - AtlasDbQosClient qosClient = new AtlasDbQosClient(qosService, scheduler, "test-client", rateLimiter); - scheduler.tick(1L, TimeUnit.MILLISECONDS); - qosClient.checkLimit(); + public void recordsReadMetrics() throws TestCheckedException { + qosClient.executeRead(() -> ESTIMATED_BYTES, () -> "foo", ignored -> ACTUAL_BYTES); + + verify(metrics).updateReadCount(); + verify(metrics).updateBytesRead(ACTUAL_BYTES); + verifyNoMoreInteractions(metrics); } @Test - public void throwsAfterLimitExceeded() { - AtlasDbQosClient qosClient = new AtlasDbQosClient(qosService, scheduler, "test-client", rateLimiter); - scheduler.tick(1L, TimeUnit.MILLISECONDS); - qosClient.checkLimit(); + public void consumesSpecifiedNumUnitsForWrites() { + qosClient.executeWrite(() -> ACTUAL_BYTES, () -> { }); - assertThatThrownBy(qosClient::checkLimit).isInstanceOf(RuntimeException.class); + verify(rateLimiter).consumeWithBackoff(ACTUAL_BYTES); + verifyNoMoreInteractions(rateLimiter); } @Test - public void canCheckAgainAfterRefreshPeriod() { - AtlasDbQosClient qosClient = new AtlasDbQosClient(qosService, scheduler, "test-client", rateLimiter); - scheduler.tick(1L, TimeUnit.MILLISECONDS); - qosClient.checkLimit(); + public void recordsWriteMetrics() throws TestCheckedException { + qosClient.executeWrite(() -> ACTUAL_BYTES, () -> { }); - assertThatThrownBy(qosClient::checkLimit) - .isInstanceOf(RuntimeException.class).hasMessage("Rate limit exceeded"); + verify(metrics).updateWriteCount(); + verify(metrics).updateBytesWritten(ACTUAL_BYTES); + } - scheduler.tick(60L, TimeUnit.SECONDS); + @Test + public void propagatesCheckedExceptions() throws TestCheckedException { + assertThatThrownBy(() -> qosClient.executeRead(() -> 1, () -> { + throw new TestCheckedException(); + }, ignored -> 1)).isInstanceOf(TestCheckedException.class); - qosClient.checkLimit(); + assertThatThrownBy(() -> qosClient.executeWrite(() -> 1, () -> { + throw new TestCheckedException(); + })).isInstanceOf(TestCheckedException.class); } + + static class TestCheckedException extends Exception { } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java index ada189ff4b4..2b7bb0449ae 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java @@ -41,7 +41,7 @@ public void canDeserializeQosServerConfiguration() throws IOException { QosServiceRuntimeConfig configuration = OBJECT_MAPPER.readValue(testConfigFile, QosServiceRuntimeConfig.class); assertThat(configuration).isEqualTo(ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test", 10L, "test2", 20L)) + .clientLimits(ImmutableMap.of("test", 10, "test2", 20)) .build()); } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java index c5c81c11e23..129ac04db62 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java @@ -30,14 +30,14 @@ public void canBuildFromEmptyClientLimits() { @Test public void canBuildFromSingleClientLimit() { ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test_client", 10L)) + .clientLimits(ImmutableMap.of("test_client", 10)) .build(); } @Test public void canBuildFromMultipleClientLimits() { ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test_client", 10L, "test_client2", 100L)) + .clientLimits(ImmutableMap.of("test_client", 10, "test_client2", 100)) .build(); } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java index 006cafef984..15fdeb1b64b 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java @@ -39,20 +39,20 @@ public class QosServiceTest { public void defaultsToNoLimit() { when(config.get()).thenReturn(configWithLimits(ImmutableMap.of())); - assertThat(resource.getLimit("foo")).isEqualTo(Long.MAX_VALUE); + assertThat(resource.getLimit("foo")).isEqualTo(Integer.MAX_VALUE); } @Test public void canLiveReloadLimits() { when(config.get()) - .thenReturn(configWithLimits(ImmutableMap.of("foo", 10L))) - .thenReturn(configWithLimits(ImmutableMap.of("foo", 20L))); + .thenReturn(configWithLimits(ImmutableMap.of("foo", 10))) + .thenReturn(configWithLimits(ImmutableMap.of("foo", 20))); - assertEquals(10L, resource.getLimit("foo")); - assertEquals(20L, resource.getLimit("foo")); + assertEquals(10, resource.getLimit("foo")); + assertEquals(20, resource.getLimit("foo")); } - private QosServiceRuntimeConfig configWithLimits(Map limits) { + private QosServiceRuntimeConfig configWithLimits(Map limits) { return ImmutableQosServiceRuntimeConfig.builder().clientLimits(limits).build(); } } diff --git a/qos-service-impl/versions.lock b/qos-service-impl/versions.lock index 664dc4edbce..a419e2f686b 100644 --- a/qos-service-impl/versions.lock +++ b/qos-service-impl/versions.lock @@ -3,7 +3,10 @@ "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "com.fasterxml.jackson.core:jackson-databind", + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.atlasdb:timestamp-api" ] }, "com.fasterxml.jackson.core:jackson-core": { @@ -14,7 +17,8 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.module:jackson-module-afterburner" + "com.fasterxml.jackson.module:jackson-module-afterburner", + "com.palantir.atlasdb:atlasdb-client" ] }, "com.fasterxml.jackson.core:jackson-databind": { @@ -25,6 +29,8 @@ "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", "com.fasterxml.jackson.module:jackson-module-afterburner", "com.netflix.feign:feign-jackson", + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", "com.palantir.remoting-api:ssl-config", @@ -44,6 +50,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava": { "locked": "2.6.7", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", "com.palantir.remoting3:jackson-support", "com.palantir.remoting3:tracing" ] @@ -72,11 +79,32 @@ "com.google.code.findbugs:annotations": { "locked": "2.0.3", "transitive": [ - "com.palantir.atlasdb:qos-service-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-client-protobufs", + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.atlasdb:commons-executors", + "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", + "com.palantir.tritium:tritium-api", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.remoting-api:errors", + "com.palantir.remoting3:jaxrs-clients", + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { - "locked": "21.0", + "locked": "18.0", "transitive": [ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.palantir.remoting3:error-handling", @@ -87,6 +115,25 @@ "com.palantir.remoting3:tracing" ] }, + "com.google.protobuf:protobuf-java": { + "locked": "2.6.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-client-protobufs" + ] + }, + "com.googlecode.json-simple:json-simple": { + "locked": "1.1.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.googlecode.protobuf-java-format:protobuf-java-format": { + "locked": "1.2", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "com.netflix.feign:feign-core": { "locked": "8.17.0", "transitive": [ @@ -120,9 +167,47 @@ "com.palantir.remoting3:jaxrs-clients" ] }, - "com.palantir.atlasdb:qos-service-api": { + "com.palantir.atlasdb:atlasdb-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:atlasdb-client": { "project": true }, + "com.palantir.atlasdb:atlasdb-client-protobufs": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:atlasdb-commons": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:commons-executors": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons" + ] + }, + "com.palantir.atlasdb:qos-service-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:timestamp-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, "com.palantir.remoting-api:errors": { "locked": "1.4.0", "transitive": [ @@ -138,6 +223,7 @@ "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", "com.palantir.remoting-api:service-config", "com.palantir.remoting3:keystores" ] @@ -173,6 +259,7 @@ "com.palantir.remoting3:jaxrs-clients": { "locked": "3.5.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:qos-service-api" ] }, @@ -198,6 +285,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", "com.palantir.remoting3:tracing-okhttp3" ] }, @@ -211,9 +299,17 @@ "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" ] }, "com.palantir.tokens:auth-tokens": { @@ -222,51 +318,161 @@ "com.palantir.remoting-api:service-config" ] }, + "com.palantir.tritium:tritium-api": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.palantir.tritium:tritium-core": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.palantir.tritium:tritium-lib": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.tritium:tritium-metrics": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-proxy": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-slf4j": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-tracing": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, "com.squareup.okhttp3:logging-interceptor": { "locked": "3.8.1", "transitive": [ "com.palantir.remoting3:okhttp-clients" ] }, + "com.squareup:javapoet": { + "locked": "1.9.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "commons-lang:commons-lang": { + "locked": "2.6", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "io.dropwizard.metrics:metrics-core": { "locked": "3.2.3", "transitive": [ + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting3:okhttp-clients" ] }, + "javax.validation:validation-api": { + "locked": "1.1.0.Final", + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, "javax.ws.rs:javax.ws.rs-api": { "locked": "2.0.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", "com.palantir.remoting-api:errors", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:jaxrs-clients" ] }, + "net.jpountz.lz4:lz4": { + "locked": "1.3.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, + "org.hdrhistogram:HdrHistogram": { + "locked": "2.1.10", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "org.jvnet:animal-sniffer-annotation": { "locked": "1.0", "transitive": [ "com.netflix.feign:feign-core" ] }, + "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir": { + "locked": "1.1.2", + "transitive": [ + "com.palantir.tritium:tritium-metrics" + ] + }, "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ "com.netflix.feign:feign-slf4j", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:jaxrs-clients", "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:tracing", "com.palantir.tokens:auth-tokens", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing", "io.dropwizard.metrics:metrics-core" ] + }, + "org.xerial.snappy:snappy-java": { + "locked": "1.1.1.7", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] } }, "runtime": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.6.7", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "com.fasterxml.jackson.core:jackson-databind", + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.atlasdb:timestamp-api" ] }, "com.fasterxml.jackson.core:jackson-core": { @@ -277,7 +483,8 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.module:jackson-module-afterburner" + "com.fasterxml.jackson.module:jackson-module-afterburner", + "com.palantir.atlasdb:atlasdb-client" ] }, "com.fasterxml.jackson.core:jackson-databind": { @@ -288,6 +495,8 @@ "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", "com.fasterxml.jackson.module:jackson-module-afterburner", "com.netflix.feign:feign-jackson", + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client", "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", "com.palantir.remoting-api:ssl-config", @@ -307,6 +516,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava": { "locked": "2.6.7", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", "com.palantir.remoting3:jackson-support", "com.palantir.remoting3:tracing" ] @@ -335,11 +545,32 @@ "com.google.code.findbugs:annotations": { "locked": "2.0.3", "transitive": [ - "com.palantir.atlasdb:qos-service-api" + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-client-protobufs", + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.atlasdb:commons-executors", + "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", + "com.palantir.tritium:tritium-api", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons", + "com.palantir.remoting-api:errors", + "com.palantir.remoting3:jaxrs-clients", + "com.palantir.remoting3:refresh-utils" ] }, "com.google.guava:guava": { - "locked": "21.0", + "locked": "18.0", "transitive": [ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.palantir.remoting3:error-handling", @@ -350,6 +581,25 @@ "com.palantir.remoting3:tracing" ] }, + "com.google.protobuf:protobuf-java": { + "locked": "2.6.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-client-protobufs" + ] + }, + "com.googlecode.json-simple:json-simple": { + "locked": "1.1.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.googlecode.protobuf-java-format:protobuf-java-format": { + "locked": "1.2", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "com.netflix.feign:feign-core": { "locked": "8.17.0", "transitive": [ @@ -383,9 +633,47 @@ "com.palantir.remoting3:jaxrs-clients" ] }, - "com.palantir.atlasdb:qos-service-api": { + "com.palantir.atlasdb:atlasdb-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:atlasdb-client": { "project": true }, + "com.palantir.atlasdb:atlasdb-client-protobufs": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:atlasdb-commons": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:commons-executors": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons" + ] + }, + "com.palantir.atlasdb:qos-service-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.atlasdb:timestamp-api": { + "project": true, + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, "com.palantir.remoting-api:errors": { "locked": "1.4.0", "transitive": [ @@ -401,6 +689,7 @@ "com.palantir.remoting-api:ssl-config": { "locked": "1.4.0", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", "com.palantir.remoting-api:service-config", "com.palantir.remoting3:keystores" ] @@ -436,6 +725,7 @@ "com.palantir.remoting3:jaxrs-clients": { "locked": "3.5.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", "com.palantir.atlasdb:qos-service-api" ] }, @@ -461,6 +751,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", "com.palantir.remoting3:tracing-okhttp3" ] }, @@ -474,9 +765,17 @@ "com.palantir.safe-logging:safe-logging": { "locked": "0.1.3", "transitive": [ + "com.palantir.atlasdb:atlasdb-client", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:tracing" + "com.palantir.remoting3:tracing", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" ] }, "com.palantir.tokens:auth-tokens": { @@ -485,44 +784,151 @@ "com.palantir.remoting-api:service-config" ] }, + "com.palantir.tritium:tritium-api": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.palantir.tritium:tritium-core": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing" + ] + }, + "com.palantir.tritium:tritium-lib": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "com.palantir.tritium:tritium-metrics": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-proxy": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-slf4j": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, + "com.palantir.tritium:tritium-tracing": { + "locked": "0.8.4", + "transitive": [ + "com.palantir.tritium:tritium-lib" + ] + }, "com.squareup.okhttp3:logging-interceptor": { "locked": "3.8.1", "transitive": [ "com.palantir.remoting3:okhttp-clients" ] }, + "com.squareup:javapoet": { + "locked": "1.9.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, + "commons-lang:commons-lang": { + "locked": "2.6", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "io.dropwizard.metrics:metrics-core": { "locked": "3.2.3", "transitive": [ + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting3:okhttp-clients" ] }, + "javax.validation:validation-api": { + "locked": "1.1.0.Final", + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, "javax.ws.rs:javax.ws.rs-api": { "locked": "2.0.1", "transitive": [ + "com.palantir.atlasdb:atlasdb-api", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.atlasdb:qos-service-api", + "com.palantir.atlasdb:timestamp-api", "com.palantir.remoting-api:errors", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:jaxrs-clients" ] }, + "net.jpountz.lz4:lz4": { + "locked": "1.3.0", + "transitive": [ + "com.palantir.atlasdb:atlasdb-commons" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "com.palantir.atlasdb:atlasdb-api" + ] + }, + "org.hdrhistogram:HdrHistogram": { + "locked": "2.1.10", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] + }, "org.jvnet:animal-sniffer-annotation": { "locked": "1.0", "transitive": [ "com.netflix.feign:feign-core" ] }, + "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir": { + "locked": "1.1.2", + "transitive": [ + "com.palantir.tritium:tritium-metrics" + ] + }, "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ "com.netflix.feign:feign-slf4j", + "com.palantir.atlasdb:atlasdb-commons", "com.palantir.remoting3:error-handling", "com.palantir.remoting3:jaxrs-clients", "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:tracing", "com.palantir.tokens:auth-tokens", + "com.palantir.tritium:tritium-core", + "com.palantir.tritium:tritium-lib", + "com.palantir.tritium:tritium-metrics", + "com.palantir.tritium:tritium-slf4j", + "com.palantir.tritium:tritium-tracing", "io.dropwizard.metrics:metrics-core" ] + }, + "org.xerial.snappy:snappy-java": { + "locked": "1.1.1.7", + "transitive": [ + "com.palantir.atlasdb:atlasdb-client" + ] } } } \ No newline at end of file diff --git a/timelock-agent/versions.lock b/timelock-agent/versions.lock index d4e51f093f8..0668e857b2f 100644 --- a/timelock-agent/versions.lock +++ b/timelock-agent/versions.lock @@ -271,6 +271,7 @@ "project": true, "transitive": [ "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1050,6 +1051,7 @@ "project": true, "transitive": [ "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/timelock-impl/versions.lock b/timelock-impl/versions.lock index 6c4f07be161..377f1de9443 100644 --- a/timelock-impl/versions.lock +++ b/timelock-impl/versions.lock @@ -270,6 +270,7 @@ "project": true, "transitive": [ "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1035,6 +1036,7 @@ "project": true, "transitive": [ "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/timelock-server-distribution/versions.lock b/timelock-server-distribution/versions.lock index 924287da028..927e1d516fc 100644 --- a/timelock-server-distribution/versions.lock +++ b/timelock-server-distribution/versions.lock @@ -405,6 +405,7 @@ "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, diff --git a/timelock-server/versions.lock b/timelock-server/versions.lock index 477cfc0f188..d8a4e791692 100644 --- a/timelock-server/versions.lock +++ b/timelock-server/versions.lock @@ -332,6 +332,7 @@ "project": true, "transitive": [ "com.palantir.atlasdb:atlasdb-impl-shared", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, @@ -1718,6 +1719,7 @@ "com.palantir.atlasdb:atlasdb-dbkvs", "com.palantir.atlasdb:atlasdb-impl-shared", "com.palantir.atlasdb:atlasdb-jdbc", + "com.palantir.atlasdb:qos-service-impl", "com.palantir.atlasdb:timestamp-impl" ] }, From dd55403cbda96c7470ed9936e605a244adca5036 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Wed, 15 Nov 2017 18:17:12 +0000 Subject: [PATCH 27/67] total-time --- .../atlasdb/factory/TransactionManagers.java | 3 +-- .../com/palantir/atlasdb/qos/QosMetrics.java | 19 ++++++++++++++++--- .../atlasdb/qos/client/AtlasDbQosClient.java | 18 +++++++++++++++++- .../atlasdb/qos/AtlasDbQosClientTest.java | 12 +++++++++++- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java index 68e2b8ecf66..e57ee08c3d9 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java @@ -74,7 +74,6 @@ import com.palantir.atlasdb.persistentlock.PersistentLockService; import com.palantir.atlasdb.qos.FakeQosClient; import com.palantir.atlasdb.qos.QosClient; -import com.palantir.atlasdb.qos.QosMetrics; import com.palantir.atlasdb.qos.QosService; import com.palantir.atlasdb.qos.client.AtlasDbQosClient; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; @@ -425,7 +424,7 @@ private QosClient createAtlasDbQosClient(ServiceConfiguration serviceConfigurati Executors.newSingleThreadScheduledExecutor(), AtlasDbMetrics.getMetricRegistry(), "qos-client-executor"); - return new AtlasDbQosClient(QosRateLimiter.create(), new QosMetrics()); + return AtlasDbQosClient.create(QosRateLimiter.create()); } private static boolean areTransactionManagerInitializationPrerequisitesSatisfied( diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java index 1da9ae591ab..d3d8f2f0573 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java @@ -23,16 +23,21 @@ public class QosMetrics { private final MetricsManager metricsManager = new MetricsManager(); private final Meter readRequestCount; - private final Meter writeRequestCount; private final Meter bytesRead; + private final Meter readTime; + + private final Meter writeRequestCount; private final Meter bytesWritten; + private final Meter writeTime; public QosMetrics() { readRequestCount = metricsManager.registerMeter(QosMetrics.class, "numReadRequests"); - writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); - bytesRead = metricsManager.registerMeter(QosMetrics.class, "bytesRead"); + readTime = metricsManager.registerMeter(QosMetrics.class, "readTime"); + + writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); bytesWritten = metricsManager.registerMeter(QosMetrics.class, "bytesWritten"); + writeTime = metricsManager.registerMeter(QosMetrics.class, "writeTime"); } public void updateReadCount() { @@ -50,4 +55,12 @@ public void updateBytesRead(long numBytes) { public void updateBytesWritten(long numBytes) { bytesWritten.mark(numBytes); } + + public void updateReadTimeMicros(long readTimeMicros) { + readTime.mark(readTimeMicros); + } + + public void updateWriteTimeMicros(long writeTimeMicros) { + readTime.mark(writeTimeMicros); + } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index cedf7253329..997c75cce36 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -15,12 +15,15 @@ */ package com.palantir.atlasdb.qos.client; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ticker; import com.palantir.atlasdb.qos.QosClient; import com.palantir.atlasdb.qos.QosMetrics; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; @@ -31,10 +34,17 @@ public class AtlasDbQosClient implements QosClient { private final QosRateLimiter rateLimiter; private final QosMetrics metrics; + private final Ticker ticker; - public AtlasDbQosClient(QosRateLimiter rateLimiter, QosMetrics metrics) { + public static AtlasDbQosClient create(QosRateLimiter rateLimiter) { + return new AtlasDbQosClient(rateLimiter, new QosMetrics(), Ticker.systemTicker()); + } + + @VisibleForTesting + AtlasDbQosClient(QosRateLimiter rateLimiter, QosMetrics metrics, Ticker ticker) { this.metrics = metrics; this.rateLimiter = rateLimiter; + this.ticker = ticker; } @Override @@ -46,11 +56,14 @@ public T executeRead( rateLimiter.consumeWithBackoff(estimatedWeight); // TODO(nziebart): decide what to do if we encounter a timeout exception + long startTimeNanos = ticker.read(); T result = query.execute(); + long totalTimeNanos = ticker.read() - startTimeNanos; int actualWeight = getWeight(() -> weigher.apply(result), estimatedWeight); metrics.updateReadCount(); metrics.updateBytesRead(actualWeight); + metrics.updateReadTimeMicros(TimeUnit.NANOSECONDS.toMicros(totalTimeNanos)); rateLimiter.recordAdjustment(actualWeight - estimatedWeight); return result; @@ -62,9 +75,12 @@ public void executeWrite(Supplier weigher, Wri rateLimiter.consumeWithBackoff(weight); // TODO(nziebart): decide what to do if we encounter a timeout exception + long startTimeNanos = ticker.read(); query.execute(); + long totalTimeNanos = ticker.read() - startTimeNanos; metrics.updateWriteCount(); + metrics.updateWriteTimeMicros(TimeUnit.NANOSECONDS.toMicros(totalTimeNanos)); metrics.updateBytesWritten(weight); } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java index e7b7b3ef8c1..f6f1ff61c74 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.Test; +import com.google.common.base.Ticker; import com.palantir.atlasdb.qos.client.AtlasDbQosClient; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; @@ -32,16 +33,22 @@ public class AtlasDbQosClientTest { private static final int ESTIMATED_BYTES = 10; private static final int ACTUAL_BYTES = 51; + private static final long START_NANOS = 1100L; + private static final long END_NANOS = 5500L; + private static final long TOTAL_TIME_MICROS = 4; private QosService qosService = mock(QosService.class); private QosRateLimiter rateLimiter = mock(QosRateLimiter.class); private QosMetrics metrics = mock(QosMetrics.class); + private Ticker ticker = mock(Ticker.class); - private AtlasDbQosClient qosClient = new AtlasDbQosClient(rateLimiter, metrics); + private AtlasDbQosClient qosClient = new AtlasDbQosClient(rateLimiter, metrics, ticker); @Before public void setUp() { when(qosService.getLimit("test-client")).thenReturn(100); + + when(ticker.read()).thenReturn(START_NANOS).thenReturn(END_NANOS); } @Test @@ -59,6 +66,7 @@ public void recordsReadMetrics() throws TestCheckedException { verify(metrics).updateReadCount(); verify(metrics).updateBytesRead(ACTUAL_BYTES); + verify(metrics).updateReadTimeMicros(TOTAL_TIME_MICROS); verifyNoMoreInteractions(metrics); } @@ -76,6 +84,8 @@ public void recordsWriteMetrics() throws TestCheckedException { verify(metrics).updateWriteCount(); verify(metrics).updateBytesWritten(ACTUAL_BYTES); + verify(metrics).updateWriteTimeMicros(TOTAL_TIME_MICROS); + verifyNoMoreInteractions(metrics); } @Test From 2318f6930e4e52dab96e9991012011ada45de0e9 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Wed, 15 Nov 2017 21:30:07 +0000 Subject: [PATCH 28/67] qos config --- .../atlasdb/config/AtlasDbRuntimeConfig.java | 7 +- .../atlasdb/factory/TransactionManagers.java | 30 ++------ .../factory/TransactionManagersTest.java | 3 +- .../qos/QosServiceIntegrationTest.java | 4 +- .../atlasdb/qos/client/AtlasDbQosClient.java | 18 ++--- .../atlasdb/qos/config/QosClientConfig.java | 47 +++++++++++++ .../atlasdb/qos/config/QosLimitsConfig.java | 41 +++++++++++ .../atlasdb/qos/ratelimit/QosRateLimiter.java | 2 +- .../qos/ratelimit/QosRateLimiters.java | 43 ++++++++++++ .../QosClientConfigDeserializationTest.java | 68 +++++++++++++++++++ .../QosRuntimeConfigDeserializationTest.java | 2 +- .../{ => client}/AtlasDbQosClientTest.java | 24 ++++--- .../src/test/resources/qos-client.yml | 10 +++ .../resources/{qos.yml => qos-server.yml} | 0 14 files changed, 250 insertions(+), 49 deletions(-) create mode 100644 qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosClientConfig.java create mode 100644 qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosLimitsConfig.java create mode 100644 qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java create mode 100644 qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosClientConfigDeserializationTest.java rename qos-service-impl/src/test/java/com/palantir/atlasdb/qos/{ => client}/AtlasDbQosClientTest.java (79%) create mode 100644 qos-service-impl/src/test/resources/qos-client.yml rename qos-service-impl/src/test/resources/{qos.yml => qos-server.yml} (100%) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java index 30ca96ce1cc..df5fbf01319 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/config/AtlasDbRuntimeConfig.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.palantir.atlasdb.AtlasDbConstants; -import com.palantir.remoting.api.config.service.ServiceConfiguration; +import com.palantir.atlasdb.qos.config.QosClientConfig; @JsonDeserialize(as = ImmutableAtlasDbRuntimeConfig.class) @JsonSerialize(as = ImmutableAtlasDbRuntimeConfig.class) @@ -61,7 +61,10 @@ public long getTimestampCacheSize() { return AtlasDbConstants.DEFAULT_TIMESTAMP_CACHE_SIZE; } - public abstract Optional getQosServiceConfiguration(); + @Value.Default + public QosClientConfig qos() { + return QosClientConfig.DEFAULT; + } /** * Runtime live-reloadable parameters for communicating with TimeLock. diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java index e57ee08c3d9..fd662a8fae3 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java @@ -18,8 +18,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Supplier; @@ -30,7 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.codahale.metrics.InstrumentedScheduledExecutorService; import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.annotations.VisibleForTesting; @@ -72,11 +69,10 @@ import com.palantir.atlasdb.persistentlock.KvsBackedPersistentLockService; import com.palantir.atlasdb.persistentlock.NoOpPersistentLockService; import com.palantir.atlasdb.persistentlock.PersistentLockService; -import com.palantir.atlasdb.qos.FakeQosClient; import com.palantir.atlasdb.qos.QosClient; -import com.palantir.atlasdb.qos.QosService; import com.palantir.atlasdb.qos.client.AtlasDbQosClient; -import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; +import com.palantir.atlasdb.qos.config.QosClientConfig; +import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters; import com.palantir.atlasdb.schema.generated.SweepTableFactory; import com.palantir.atlasdb.spi.AtlasDbFactory; import com.palantir.atlasdb.spi.KeyValueServiceConfig; @@ -117,9 +113,6 @@ import com.palantir.lock.impl.LockServiceImpl; import com.palantir.lock.v2.TimelockService; import com.palantir.logsafe.UnsafeArg; -import com.palantir.remoting.api.config.service.ServiceConfiguration; -import com.palantir.remoting3.clients.ClientConfigurations; -import com.palantir.remoting3.jaxrs.JaxRsClient; import com.palantir.timestamp.TimestampService; import com.palantir.timestamp.TimestampStoreInvalidator; import com.palantir.util.OptionalResolver; @@ -316,8 +309,7 @@ SerializableTransactionManager serializable() { java.util.function.Supplier runtimeConfigSupplier = () -> runtimeConfigSupplier().get().orElse(defaultRuntime); - - QosClient qosClient = getQosClient(runtimeConfigSupplier.get().getQosServiceConfiguration()); + QosClient qosClient = getQosClient(runtimeConfigSupplier.get().qos()); ServiceDiscoveringAtlasSupplier atlasFactory = new ServiceDiscoveringAtlasSupplier( @@ -411,20 +403,10 @@ SerializableTransactionManager serializable() { return transactionManager; } - private QosClient getQosClient(Optional serviceConfiguration) { - return serviceConfiguration.map(this::createAtlasDbQosClient).orElse(FakeQosClient.INSTANCE); - } - - private QosClient createAtlasDbQosClient(ServiceConfiguration serviceConfiguration) { - QosService qosService = JaxRsClient.create(QosService.class, - userAgent(), - ClientConfigurations.of(serviceConfiguration)); + private QosClient getQosClient(QosClientConfig config) { // TODO(nziebart): create a RefreshingRateLimiter - ScheduledExecutorService scheduler = new InstrumentedScheduledExecutorService( - Executors.newSingleThreadScheduledExecutor(), - AtlasDbMetrics.getMetricRegistry(), - "qos-client-executor"); - return AtlasDbQosClient.create(QosRateLimiter.create()); + QosRateLimiters rateLimiters = QosRateLimiters.create(config.limits()); + return AtlasDbQosClient.create(rateLimiters); } private static boolean areTransactionManagerInitializationPrerequisitesSatisfied( diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java index 729b4f0e69f..5a6258b43bd 100644 --- a/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/factory/TransactionManagersTest.java @@ -73,6 +73,7 @@ import com.palantir.atlasdb.config.TimeLockClientConfig; import com.palantir.atlasdb.factory.startup.TimeLockMigrator; import com.palantir.atlasdb.memory.InMemoryAtlasDbConfig; +import com.palantir.atlasdb.qos.config.QosClientConfig; import com.palantir.atlasdb.table.description.GenericTestSchema; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; import com.palantir.atlasdb.util.MetricsRule; @@ -193,7 +194,7 @@ public void setup() throws JsonProcessingException { runtimeConfig = mock(AtlasDbRuntimeConfig.class); when(runtimeConfig.timestampClient()).thenReturn(ImmutableTimestampClientConfig.of(false)); - when(runtimeConfig.getQosServiceConfiguration()).thenReturn(Optional.empty()); + when(runtimeConfig.qos()).thenReturn(QosClientConfig.DEFAULT); when(runtimeConfig.timelockRuntime()).thenReturn(Optional.empty()); environment = mock(Consumer.class); diff --git a/qos-service-impl/src/integTest/java/com/palantir/atlasdb/qos/QosServiceIntegrationTest.java b/qos-service-impl/src/integTest/java/com/palantir/atlasdb/qos/QosServiceIntegrationTest.java index 51642b0c450..4d355fa2978 100644 --- a/qos-service-impl/src/integTest/java/com/palantir/atlasdb/qos/QosServiceIntegrationTest.java +++ b/qos-service-impl/src/integTest/java/com/palantir/atlasdb/qos/QosServiceIntegrationTest.java @@ -50,8 +50,8 @@ public class QosServiceIntegrationTest { @Test public void returnsConfiguredLimits() { - assertThat(service.getLimit("test")).isEqualTo(10L); - assertThat(service.getLimit("test2")).isEqualTo(20L); + assertThat(service.getLimit("test")).isEqualTo(10); + assertThat(service.getLimit("test2")).isEqualTo(20); } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index 997c75cce36..09dc7ca64fc 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -26,24 +26,24 @@ import com.google.common.base.Ticker; import com.palantir.atlasdb.qos.QosClient; import com.palantir.atlasdb.qos.QosMetrics; -import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; +import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters; public class AtlasDbQosClient implements QosClient { private static final Logger log = LoggerFactory.getLogger(AtlasDbQosClient.class); - private final QosRateLimiter rateLimiter; + private final QosRateLimiters rateLimiters; private final QosMetrics metrics; private final Ticker ticker; - public static AtlasDbQosClient create(QosRateLimiter rateLimiter) { - return new AtlasDbQosClient(rateLimiter, new QosMetrics(), Ticker.systemTicker()); + public static AtlasDbQosClient create(QosRateLimiters rateLimiters) { + return new AtlasDbQosClient(rateLimiters, new QosMetrics(), Ticker.systemTicker()); } @VisibleForTesting - AtlasDbQosClient(QosRateLimiter rateLimiter, QosMetrics metrics, Ticker ticker) { + AtlasDbQosClient(QosRateLimiters rateLimiters, QosMetrics metrics, Ticker ticker) { this.metrics = metrics; - this.rateLimiter = rateLimiter; + this.rateLimiters = rateLimiters; this.ticker = ticker; } @@ -53,7 +53,7 @@ public T executeRead( ReadQuery query, Function weigher) throws E { int estimatedWeight = getWeight(estimatedWeigher, 1); - rateLimiter.consumeWithBackoff(estimatedWeight); + rateLimiters.read().consumeWithBackoff(estimatedWeight); // TODO(nziebart): decide what to do if we encounter a timeout exception long startTimeNanos = ticker.read(); @@ -64,7 +64,7 @@ public T executeRead( metrics.updateReadCount(); metrics.updateBytesRead(actualWeight); metrics.updateReadTimeMicros(TimeUnit.NANOSECONDS.toMicros(totalTimeNanos)); - rateLimiter.recordAdjustment(actualWeight - estimatedWeight); + rateLimiters.read().recordAdjustment(actualWeight - estimatedWeight); return result; } @@ -72,7 +72,7 @@ public T executeRead( @Override public void executeWrite(Supplier weigher, WriteQuery query) throws E { int weight = getWeight(weigher, 1); - rateLimiter.consumeWithBackoff(weight); + rateLimiters.write().consumeWithBackoff(weight); // TODO(nziebart): decide what to do if we encounter a timeout exception long startTimeNanos = ticker.read(); diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosClientConfig.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosClientConfig.java new file mode 100644 index 00000000000..313a02f3d7d --- /dev/null +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosClientConfig.java @@ -0,0 +1,47 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.qos.config; + +import java.util.Optional; + +import org.immutables.value.Value; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.palantir.remoting.api.config.service.HumanReadableDuration; +import com.palantir.remoting.api.config.service.ServiceConfiguration; + +@Value.Immutable +@JsonDeserialize(as = ImmutableQosClientConfig.class) +@JsonSerialize(as = ImmutableQosClientConfig.class) +public abstract class QosClientConfig { + + public static final QosClientConfig DEFAULT = ImmutableQosClientConfig.builder().build(); + + public abstract Optional qosService(); + + @Value.Default + public HumanReadableDuration maxBackoffSleepTime() { + return HumanReadableDuration.seconds(10); + } + + @Value.Default + public QosLimitsConfig limits() { + return QosLimitsConfig.DEFAULT_NO_LIMITS; + } + +} diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosLimitsConfig.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosLimitsConfig.java new file mode 100644 index 00000000000..e932632ddaa --- /dev/null +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosLimitsConfig.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.qos.config; + +import org.immutables.value.Value; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@Value.Immutable +@JsonDeserialize(as = ImmutableQosLimitsConfig.class) +@JsonSerialize(as = ImmutableQosLimitsConfig.class) +public abstract class QosLimitsConfig { + + public static final QosLimitsConfig DEFAULT_NO_LIMITS = ImmutableQosLimitsConfig.builder().build(); + + @Value.Default + public int readBytesPerSecond() { + return Integer.MAX_VALUE; + } + + @Value.Default + public int writeBytesPerSecond() { + return Integer.MAX_VALUE; + } + +} diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java index 4112e03d9e1..4ea8fc5e227 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java @@ -80,7 +80,7 @@ public Duration consumeWithBackoff(int estimatedNumUnits) { /** * Records an adjustment to the original estimate of units consumed passed to {@link #consumeWithBackoff(int)}. This * should be called after a query returns, when the exact number of units consumed is known. This value may be - * positive (if the original estimate was too small) or negative (if the original estimate was too large. + * positive (if the original estimate was too small) or negative (if the original estimate was too large). */ public void recordAdjustment(int adjustmentUnits) { if (adjustmentUnits > 0) { diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java new file mode 100644 index 00000000000..503b5b79d39 --- /dev/null +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.qos.ratelimit; + +import org.immutables.value.Value; + +import com.palantir.atlasdb.qos.config.QosLimitsConfig; + +@Value.Immutable +public interface QosRateLimiters { + + static QosRateLimiters create(QosLimitsConfig config) { + QosRateLimiter readLimiter = QosRateLimiter.create(); + readLimiter.updateRate(config.readBytesPerSecond()); + + QosRateLimiter writeLimiter = QosRateLimiter.create(); + writeLimiter.updateRate(config.writeBytesPerSecond()); + + return ImmutableQosRateLimiters.builder() + .read(readLimiter) + .write(writeLimiter) + .build(); + } + + QosRateLimiter read(); + + QosRateLimiter write(); + +} diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosClientConfigDeserializationTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosClientConfigDeserializationTest.java new file mode 100644 index 00000000000..1edfe1e9b16 --- /dev/null +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosClientConfigDeserializationTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.qos; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; +import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; +import com.palantir.atlasdb.qos.config.QosClientConfig; +import com.palantir.atlasdb.qos.config.QosServiceRuntimeConfig; +import com.palantir.remoting.api.config.service.HumanReadableDuration; +import com.palantir.remoting.api.config.service.ServiceConfiguration; +import com.palantir.remoting.api.config.ssl.SslConfiguration; +import com.palantir.remoting3.ext.jackson.ShimJdk7Module; + +public class QosClientConfigDeserializationTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()) + .registerModule(new GuavaModule()) + .registerModule(new ShimJdk7Module()) + .registerModule(new Jdk8Module()); + + @Test + public void canDeserializeFromYaml() throws IOException { + QosClientConfig expected = ImmutableQosClientConfig.builder() + .qosService( + ServiceConfiguration.builder() + .addUris("http://localhost:8080") + .security(SslConfiguration.of(Paths.get("trustStore.jks"))) + .build()) + .maxBackoffSleepTime(HumanReadableDuration.seconds(20)) + .limits(ImmutableQosLimitsConfig.builder() + .readBytesPerSecond(123) + .writeBytesPerSecond(456) + .build()) + .build(); + + File configFile = new File(QosServiceRuntimeConfig.class.getResource("/qos-client.yml").getPath()); + QosClientConfig config = OBJECT_MAPPER.readValue(configFile, QosClientConfig.class); + + assertThat(config).isEqualTo(expected); + } + +} diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java index 2b7bb0449ae..55a662bb1cb 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java @@ -37,7 +37,7 @@ public class QosRuntimeConfigDeserializationTest { @Test public void canDeserializeQosServerConfiguration() throws IOException { - File testConfigFile = new File(QosServiceRuntimeConfig.class.getResource("/qos.yml").getPath()); + File testConfigFile = new File(QosServiceRuntimeConfig.class.getResource("/qos-server.yml").getPath()); QosServiceRuntimeConfig configuration = OBJECT_MAPPER.readValue(testConfigFile, QosServiceRuntimeConfig.class); assertThat(configuration).isEqualTo(ImmutableQosServiceRuntimeConfig.builder() diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java similarity index 79% rename from qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java rename to qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java index f6f1ff61c74..27a8ddbda4a 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/AtlasDbQosClientTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.palantir.atlasdb.qos; +package com.palantir.atlasdb.qos.client; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; @@ -26,8 +26,11 @@ import org.junit.Test; import com.google.common.base.Ticker; -import com.palantir.atlasdb.qos.client.AtlasDbQosClient; +import com.palantir.atlasdb.qos.QosMetrics; +import com.palantir.atlasdb.qos.QosService; +import com.palantir.atlasdb.qos.ratelimit.ImmutableQosRateLimiters; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; +import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters; public class AtlasDbQosClientTest { @@ -38,11 +41,14 @@ public class AtlasDbQosClientTest { private static final long TOTAL_TIME_MICROS = 4; private QosService qosService = mock(QosService.class); - private QosRateLimiter rateLimiter = mock(QosRateLimiter.class); + private QosRateLimiter readLimiter = mock(QosRateLimiter.class); + private QosRateLimiter writeLimiter = mock(QosRateLimiter.class); + private QosRateLimiters rateLimiters = ImmutableQosRateLimiters.builder() + .read(readLimiter).write(writeLimiter).build(); private QosMetrics metrics = mock(QosMetrics.class); private Ticker ticker = mock(Ticker.class); - private AtlasDbQosClient qosClient = new AtlasDbQosClient(rateLimiter, metrics, ticker); + private AtlasDbQosClient qosClient = new AtlasDbQosClient(rateLimiters, metrics, ticker); @Before public void setUp() { @@ -55,9 +61,9 @@ public void setUp() { public void consumesSpecifiedNumUnitsForReads() { qosClient.executeRead(() -> ESTIMATED_BYTES, () -> "foo", ignored -> ACTUAL_BYTES); - verify(rateLimiter).consumeWithBackoff(ESTIMATED_BYTES); - verify(rateLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); - verifyNoMoreInteractions(rateLimiter); + verify(readLimiter).consumeWithBackoff(ESTIMATED_BYTES); + verify(readLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); + verifyNoMoreInteractions(readLimiter, writeLimiter); } @Test @@ -74,8 +80,8 @@ public void recordsReadMetrics() throws TestCheckedException { public void consumesSpecifiedNumUnitsForWrites() { qosClient.executeWrite(() -> ACTUAL_BYTES, () -> { }); - verify(rateLimiter).consumeWithBackoff(ACTUAL_BYTES); - verifyNoMoreInteractions(rateLimiter); + verify(writeLimiter).consumeWithBackoff(ACTUAL_BYTES); + verifyNoMoreInteractions(readLimiter, writeLimiter); } @Test diff --git a/qos-service-impl/src/test/resources/qos-client.yml b/qos-service-impl/src/test/resources/qos-client.yml new file mode 100644 index 00000000000..163da250c04 --- /dev/null +++ b/qos-service-impl/src/test/resources/qos-client.yml @@ -0,0 +1,10 @@ +qosService: + uris: + - http://localhost:8080 + security: + trustStorePath: trustStore.jks +maxBackoffSleepTime: 20 seconds +limits: + readBytesPerSecond: 123 + writeBytesPerSecond: 456 + diff --git a/qos-service-impl/src/test/resources/qos.yml b/qos-service-impl/src/test/resources/qos-server.yml similarity index 100% rename from qos-service-impl/src/test/resources/qos.yml rename to qos-service-impl/src/test/resources/qos-server.yml From c7dff29470a9247a61a1ed952f153c7f84dae3e4 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Wed, 15 Nov 2017 21:38:02 +0000 Subject: [PATCH 29/67] respect max backoff itme --- .../atlasdb/factory/TransactionManagers.java | 4 +++- .../atlasdb/qos/ratelimit/QosRateLimiter.java | 13 +++++++------ .../atlasdb/qos/ratelimit/QosRateLimiters.java | 6 +++--- .../atlasdb/qos/ratelimit/QosRateLimiterTest.java | 12 +++++++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java index fd662a8fae3..cf1450fb8eb 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java @@ -405,7 +405,9 @@ SerializableTransactionManager serializable() { private QosClient getQosClient(QosClientConfig config) { // TODO(nziebart): create a RefreshingRateLimiter - QosRateLimiters rateLimiters = QosRateLimiters.create(config.limits()); + QosRateLimiters rateLimiters = QosRateLimiters.create( + config.limits(), + config.maxBackoffSleepTime().toMilliseconds()); return AtlasDbQosClient.create(rateLimiters); } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java index 4ea8fc5e227..7bb5773349b 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java @@ -34,21 +34,22 @@ public class QosRateLimiter { private static final double MAX_BURST_SECONDS = 5; private static final double UNLIMITED_RATE = Double.MAX_VALUE; - private static final int MAX_WAIT_TIME_SECONDS = 10; + private final long maxBackoffTimeMillis; private RateLimiter rateLimiter; - public static QosRateLimiter create() { - return new QosRateLimiter(RateLimiter.SleepingStopwatch.createFromSystemTimer()); + public static QosRateLimiter create(long maxBackoffTimeMillis) { + return new QosRateLimiter(RateLimiter.SleepingStopwatch.createFromSystemTimer(), maxBackoffTimeMillis); } @VisibleForTesting - QosRateLimiter(RateLimiter.SleepingStopwatch stopwatch) { + QosRateLimiter(RateLimiter.SleepingStopwatch stopwatch, long maxBackoffTimeMillis) { rateLimiter = new SmoothRateLimiter.SmoothBursty( stopwatch, MAX_BURST_SECONDS); rateLimiter.setRate(UNLIMITED_RATE); + this.maxBackoffTimeMillis = maxBackoffTimeMillis; } /** @@ -67,8 +68,8 @@ public void updateRate(int unitsPerSecond) { public Duration consumeWithBackoff(int estimatedNumUnits) { Optional waitTime = rateLimiter.tryAcquire( estimatedNumUnits, - MAX_WAIT_TIME_SECONDS, - TimeUnit.SECONDS); + maxBackoffTimeMillis, + TimeUnit.MILLISECONDS); if (!waitTime.isPresent()) { throw new RuntimeException("rate limited"); diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java index 503b5b79d39..a60696dc649 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiters.java @@ -23,11 +23,11 @@ @Value.Immutable public interface QosRateLimiters { - static QosRateLimiters create(QosLimitsConfig config) { - QosRateLimiter readLimiter = QosRateLimiter.create(); + static QosRateLimiters create(QosLimitsConfig config, long maxBackoffSleepTimeMillis) { + QosRateLimiter readLimiter = QosRateLimiter.create(maxBackoffSleepTimeMillis); readLimiter.updateRate(config.readBytesPerSecond()); - QosRateLimiter writeLimiter = QosRateLimiter.create(); + QosRateLimiter writeLimiter = QosRateLimiter.create(maxBackoffSleepTimeMillis); writeLimiter.updateRate(config.writeBytesPerSecond()); return ImmutableQosRateLimiters.builder() diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiterTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiterTest.java index 8d88ad58b21..c68278e43d3 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiterTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiterTest.java @@ -30,9 +30,10 @@ public class QosRateLimiterTest { private static final long START_TIME_MICROS = 0L; + private static final long MAX_BACKOFF_TIME_MILLIS = 10_000; RateLimiter.SleepingStopwatch stopwatch = mock(RateLimiter.SleepingStopwatch.class); - QosRateLimiter limiter = new QosRateLimiter(stopwatch); + QosRateLimiter limiter = new QosRateLimiter(stopwatch, MAX_BACKOFF_TIME_MILLIS); @Before public void before() { @@ -63,6 +64,15 @@ public void limitsByThrowingIfSleepTimeIsTooGreat() { .hasMessageContaining("rate limited"); } + @Test + public void doesNotThrowIfMaxBackoffTimeIsVeryLarge() { + QosRateLimiter limiterWithLargeBackoffLimit = new QosRateLimiter(stopwatch, Long.MAX_VALUE); + limiterWithLargeBackoffLimit.updateRate(10); + + limiterWithLargeBackoffLimit.consumeWithBackoff(1_000_000_000); + limiterWithLargeBackoffLimit.consumeWithBackoff(1_000_000_000); + } + @Test public void consumingAdditionalUnitsPenalizesFutureCallers() { limiter.updateRate(10); From 76fd66488095efc2dfdc6e1aa81d6eee59d31332 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Thu, 16 Nov 2017 22:20:18 +0000 Subject: [PATCH 30/67] query weights --- .../cassandra/CassandraClientFactory.java | 1 + .../{ => qos}/QosCassandraClient.java | 48 ++----- .../{ => qos}/ThriftObjectSizeUtils.java | 77 +++++++----- .../cassandra/qos/ThriftQueryWeighers.java | 118 ++++++++++++++++++ .../atlasdb/ThriftObjectSizeUtilsTest.java | 2 +- .../cassandra/QosCassandraClientTest.java | 7 +- qos-service-api/build.gradle | 3 + .../palantir/atlasdb/qos/FakeQosClient.java | 11 +- .../com/palantir/atlasdb/qos/QosClient.java | 18 +-- .../com/palantir/atlasdb/qos/QosService.java | 2 +- .../com/palantir/atlasdb/qos/QueryWeight.java | 37 ++++++ .../com/palantir/atlasdb/qos/QosResource.java | 4 +- .../atlasdb/qos/client/AtlasDbQosClient.java | 48 +++---- .../qos/config/QosServiceRuntimeConfig.java | 2 +- .../atlasdb/qos/{ => metrics}/QosMetrics.java | 34 +++-- .../atlasdb/qos/ratelimit/QosRateLimiter.java | 13 +- .../QosRuntimeConfigDeserializationTest.java | 2 +- .../qos/QosServiceRuntimeConfigTest.java | 4 +- .../palantir/atlasdb/qos/QosServiceTest.java | 12 +- .../qos/client/AtlasDbQosClientTest.java | 66 +++++++--- 20 files changed, 338 insertions(+), 171 deletions(-) rename atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/{ => qos}/QosCassandraClient.java (70%) rename atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/{ => qos}/ThriftObjectSizeUtils.java (67%) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java create mode 100644 qos-service-api/src/main/java/com/palantir/atlasdb/qos/QueryWeight.java rename qos-service-impl/src/main/java/com/palantir/atlasdb/qos/{ => metrics}/QosMetrics.java (69%) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java index 3d8a3a5e7d8..a29facdd13a 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientFactory.java @@ -46,6 +46,7 @@ import com.google.common.collect.Maps; import com.palantir.atlasdb.cassandra.CassandraCredentialsConfig; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; +import com.palantir.atlasdb.keyvalue.cassandra.qos.QosCassandraClient; import com.palantir.atlasdb.qos.QosClient; import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.common.exception.AtlasDbDependencyException; diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java similarity index 70% rename from atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java rename to atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java index 18d60ca8f13..be0a912be7f 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java @@ -14,13 +14,11 @@ * limitations under the License. */ -package com.palantir.atlasdb.keyvalue.cassandra; +package com.palantir.atlasdb.keyvalue.cassandra.qos; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.function.Function; import org.apache.cassandra.thrift.CASResult; import org.apache.cassandra.thrift.Cassandra; @@ -43,13 +41,13 @@ import org.slf4j.LoggerFactory; import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.cassandra.CassandraClient; +import com.palantir.atlasdb.keyvalue.cassandra.CqlQuery; import com.palantir.atlasdb.qos.QosClient; @SuppressWarnings({"all"}) // thrift variable names. public class QosCassandraClient implements CassandraClient { - private static final int DEFAULT_ESTIMATED_READ_BYTES = 100; - private static final Logger log = LoggerFactory.getLogger(CassandraClient.class); private final CassandraClient client; @@ -70,17 +68,8 @@ public Map> multiget_slice(String kvsMetho List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { return qosClient.executeRead( - () -> DEFAULT_ESTIMATED_READ_BYTES, - () -> client.multiget_slice(kvsMethodName, tableRef, keys, - predicate, consistency_level), - this::getApproximateReadByteCount); - } - - private int getApproximateReadByteCount(Map> result) { - return getCollectionSize(result.entrySet(), - rowResult -> ThriftObjectSizeUtils.getByteBufferSize(rowResult.getKey()) - + getCollectionSize(rowResult.getValue(), - ThriftObjectSizeUtils::getColumnOrSuperColumnSize)); + () -> client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level), + ThriftQueryWeighers.MULTIGET_SLICE); } @Override @@ -88,9 +77,8 @@ public List get_range_slices(String kvsMethodName, TableReference tabl KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { return qosClient.executeRead( - () -> DEFAULT_ESTIMATED_READ_BYTES, () -> client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level), - result -> getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); + ThriftQueryWeighers.GET_RANGE_SLICES); } @Override @@ -98,17 +86,8 @@ public void batch_mutate(String kvsMethodName, Map getApproximateWriteByteCount(mutation_map), - () -> client.batch_mutate(kvsMethodName, mutation_map, consistency_level)); - } - - private int getApproximateWriteByteCount(Map>> batchMutateMap) { - int approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); - int approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> - getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) - + getCollectionSize(currentMap.values(), - mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); - return approxBytesForKeys + approxBytesForValues; + () -> client.batch_mutate(kvsMethodName, mutation_map, consistency_level), + ThriftQueryWeighers.batchMutate(mutation_map)); } @Override @@ -116,9 +95,8 @@ public ColumnOrSuperColumn get(TableReference tableReference, ByteBuffer key, by ConsistencyLevel consistency_level) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException { return qosClient.executeRead( - () -> DEFAULT_ESTIMATED_READ_BYTES, () -> client.get(tableReference, key, column, consistency_level), - ThriftObjectSizeUtils::getColumnOrSuperColumnSize); + ThriftQueryWeighers.GET); } @Override @@ -134,14 +112,10 @@ public CASResult cas(TableReference tableReference, ByteBuffer key, List public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { - return qosClient.executeRead( - () -> DEFAULT_ESTIMATED_READ_BYTES, () -> client.execute_cql3_query(cqlQuery, compression, consistency), - ThriftObjectSizeUtils::getCqlResultSize); + ThriftQueryWeighers.EXECUTE_CQL3_QUERY); } - private int getCollectionSize(Collection collection, Function singleObjectSizeFunction) { - return ThriftObjectSizeUtils.getCollectionSize(collection, singleObjectSizeFunction); - } + } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java similarity index 67% rename from atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java rename to atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java index f5f95e0a2a0..986caaf06b0 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java @@ -14,10 +14,11 @@ * limitations under the License. */ -package com.palantir.atlasdb.keyvalue.cassandra; +package com.palantir.atlasdb.keyvalue.cassandra.qos; import java.nio.ByteBuffer; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -37,13 +38,33 @@ public final class ThriftObjectSizeUtils { - private static final int ONE_BYTE = 1; + private static final long ONE_BYTE = 1; private ThriftObjectSizeUtils() { // utility class } - public static int getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { + public static long getApproximateWriteByteCount(Map>> batchMutateMap) { + long approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); + long approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> + getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + + getCollectionSize(currentMap.values(), + mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); + return approxBytesForKeys + approxBytesForValues; + } + + public static long getApproximateReadByteCount(Map> result) { + return getCollectionSize(result.entrySet(), + rowResult -> ThriftObjectSizeUtils.getByteBufferSize(rowResult.getKey()) + + getCollectionSize(rowResult.getValue(), + ThriftObjectSizeUtils::getColumnOrSuperColumnSize)); + } + + public static long getApproximateReadByteCount(List slices) { + return getCollectionSize(slices, ThriftObjectSizeUtils::getKeySliceSize); + } + + public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { if (columnOrSuperColumn == null) { return getNullSize(); } @@ -53,14 +74,14 @@ public static int getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperCo + getCounterSuperColumnSize(columnOrSuperColumn.getCounter_super_column()); } - public static int getByteBufferSize(ByteBuffer byteBuffer) { + public static long getByteBufferSize(ByteBuffer byteBuffer) { if (byteBuffer == null) { return getNullSize(); } return byteBuffer.remaining(); } - public static int getMutationSize(Mutation mutation) { + public static long getMutationSize(Mutation mutation) { if (mutation == null) { return getNullSize(); } @@ -69,7 +90,7 @@ public static int getMutationSize(Mutation mutation) { mutation.getDeletion()); } - public static int getCqlResultSize(CqlResult cqlResult) { + public static long getCqlResultSize(CqlResult cqlResult) { if (cqlResult == null) { return getNullSize(); } @@ -79,7 +100,7 @@ public static int getCqlResultSize(CqlResult cqlResult) { + getCqlMetadataSize(cqlResult.getSchema()); } - public static int getKeySliceSize(KeySlice keySlice) { + public static long getKeySliceSize(KeySlice keySlice) { if (keySlice == null) { return getNullSize(); } @@ -88,7 +109,7 @@ public static int getKeySliceSize(KeySlice keySlice) { + getCollectionSize(keySlice.getColumns(), ThriftObjectSizeUtils::getColumnOrSuperColumnSize); } - public static int getStringSize(String string) { + public static long getStringSize(String string) { if (string == null) { return getNullSize(); } @@ -96,7 +117,7 @@ public static int getStringSize(String string) { return string.length() * Character.SIZE; } - public static int getColumnSize(Column column) { + public static long getColumnSize(Column column) { if (column == null) { return getNullSize(); } @@ -107,7 +128,7 @@ public static int getColumnSize(Column column) { + getTimestampSize(); } - private static int getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { + private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { if (counterSuperColumn == null) { return getNullSize(); } @@ -116,7 +137,7 @@ private static int getCounterSuperColumnSize(CounterSuperColumn counterSuperColu + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); } - private static int getCounterColumnSize(CounterColumn counterColumn) { + private static long getCounterColumnSize(CounterColumn counterColumn) { if (counterColumn == null) { return getNullSize(); } @@ -124,7 +145,7 @@ private static int getCounterColumnSize(CounterColumn counterColumn) { return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); } - private static int getSuperColumnSize(SuperColumn superColumn) { + private static long getSuperColumnSize(SuperColumn superColumn) { if (superColumn == null) { return getNullSize(); } @@ -133,7 +154,7 @@ private static int getSuperColumnSize(SuperColumn superColumn) { + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - private static int getDeletionSize(Deletion deletion) { + private static long getDeletionSize(Deletion deletion) { if (deletion == null) { return getNullSize(); } @@ -143,7 +164,7 @@ private static int getDeletionSize(Deletion deletion) { + getSlicePredicateSize(deletion.getPredicate()); } - private static int getSlicePredicateSize(SlicePredicate predicate) { + private static long getSlicePredicateSize(SlicePredicate predicate) { if (predicate == null) { return getNullSize(); } @@ -152,7 +173,7 @@ private static int getSlicePredicateSize(SlicePredicate predicate) { + getSliceRangeSize(predicate.getSlice_range()); } - private static int getSliceRangeSize(SliceRange sliceRange) { + private static long getSliceRangeSize(SliceRange sliceRange) { if (sliceRange == null) { return getNullSize(); } @@ -163,7 +184,7 @@ private static int getSliceRangeSize(SliceRange sliceRange) { + getSliceRangeCountSize(); } - private static int getCqlMetadataSize(CqlMetadata schema) { + private static long getCqlMetadataSize(CqlMetadata schema) { if (schema == null) { return getNullSize(); } @@ -174,13 +195,13 @@ private static int getCqlMetadataSize(CqlMetadata schema) { + getStringSize(schema.getDefault_value_type()); } - private static int getByteBufferStringMapSize(Map nameTypes) { + private static long getByteBufferStringMapSize(Map nameTypes) { return getCollectionSize(nameTypes.entrySet(), entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + ThriftObjectSizeUtils.getStringSize(entry.getValue())); } - private static int getCqlRowSize(CqlRow cqlRow) { + private static long getCqlRowSize(CqlRow cqlRow) { if (cqlRow == null) { return getNullSize(); } @@ -188,47 +209,47 @@ private static int getCqlRowSize(CqlRow cqlRow) { + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - private static int getThriftEnumSize() { + private static long getThriftEnumSize() { return Integer.BYTES; } - private static int getByteArraySize(byte[] byteArray) { + private static long getByteArraySize(byte[] byteArray) { if (byteArray == null) { return getNullSize(); } return byteArray.length; } - private static int getTimestampSize() { + private static long getTimestampSize() { return Long.BYTES; } - private static int getTtlSize() { + private static long getTtlSize() { return Integer.BYTES; } - private static int getCounterValueSize() { + private static long getCounterValueSize() { return Long.BYTES; } - private static int getReversedBooleanSize() { + private static long getReversedBooleanSize() { return ONE_BYTE; } - private static int getSliceRangeCountSize() { + private static long getSliceRangeCountSize() { return Integer.BYTES; } - private static int getNullSize() { + private static long getNullSize() { return Integer.BYTES; } - public static int getCollectionSize(Collection collection, Function sizeFunction) { + public static long getCollectionSize(Collection collection, Function sizeFunction) { if (collection == null) { return getNullSize(); } - int sum = 0; + long sum = 0; for (T item : collection) { sum += sizeFunction.apply(item); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java new file mode 100644 index 00000000000..45a4e79ae0c --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java @@ -0,0 +1,118 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra.qos; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Suppliers; +import com.palantir.atlasdb.keyvalue.cassandra.CassandraClient; +import com.palantir.atlasdb.qos.ImmutableQueryWeight; +import com.palantir.atlasdb.qos.QosClient; +import com.palantir.atlasdb.qos.QueryWeight; + +public final class ThriftQueryWeighers { + + private static final Logger log = LoggerFactory.getLogger(CassandraClient.class); + + public static final QueryWeight DEFAULT_ESTIMATED_WEIGHT = ImmutableQueryWeight.builder() + .numBytes(100) + .numDistinctRows(1) + .timeTakenNanos(TimeUnit.MILLISECONDS.toNanos(2)) + .build(); + + private ThriftQueryWeighers() { } + + public static final QosClient.QueryWeigher>> MULTIGET_SLICE = + readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount); + + public static final QosClient.QueryWeigher> GET_RANGE_SLICES = + readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount); + + public static final QosClient.QueryWeigher GET = + readWeigher(ThriftObjectSizeUtils::getColumnOrSuperColumnSize); + + public static final QosClient.QueryWeigher EXECUTE_CQL3_QUERY = + readWeigher(ThriftObjectSizeUtils::getCqlResultSize); + + public static final QosClient.QueryWeigher batchMutate(Map>> mutationMap) { + return writeWeigher(() -> ThriftObjectSizeUtils.getApproximateWriteByteCount(mutationMap)); + } + + public static QosClient.QueryWeigher readWeigher(Function bytesRead) { + return new QosClient.QueryWeigher() { + @Override + public QueryWeight estimate() { + return DEFAULT_ESTIMATED_WEIGHT; + } + + @Override + public QueryWeight weigh(T result, long timeTakenNanos) { + return ImmutableQueryWeight.builder() + .numBytes(safeGetNumBytesOrDefault(() -> bytesRead.apply(result))) + .timeTakenNanos((int)timeTakenNanos) + .numDistinctRows(1) + .build(); + } + }; + } + + public static QosClient.QueryWeigher writeWeigher(Supplier bytesWritten) { + + Supplier weight = Suppliers.memoize(() -> safeGetNumBytesOrDefault(bytesWritten))::get; + + return new QosClient.QueryWeigher() { + @Override + public QueryWeight estimate() { + return ImmutableQueryWeight.builder() + .from(DEFAULT_ESTIMATED_WEIGHT) + .numBytes(weight.get()) + .build(); + } + + @Override + public QueryWeight weigh(T result, long timeTakenNanos) { + return ImmutableQueryWeight.builder() + .from(estimate()) + .timeTakenNanos((int)timeTakenNanos) + .build(); + } + }; + } + + // TODO(nziebart): we really shouldn't be needing to catch exceptions here + private static long safeGetNumBytesOrDefault(Supplier numBytes) { + try { + return numBytes.get(); + } catch (Exception e) { + log.warn("Error calculating number of bytes", e); + return DEFAULT_ESTIMATED_WEIGHT.numBytes(); + } + } + +} diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index a9f623361e0..25d26d25828 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -33,7 +33,7 @@ import org.junit.Test; import com.google.common.collect.ImmutableList; -import com.palantir.atlasdb.keyvalue.cassandra.ThriftObjectSizeUtils; +import com.palantir.atlasdb.keyvalue.cassandra.qos.ThriftObjectSizeUtils; public class ThriftObjectSizeUtilsTest { diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java index 38d762b3563..3283aab1bcc 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/QosCassandraClientTest.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableMap; import com.palantir.atlasdb.encoding.PtBytes; import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.cassandra.qos.QosCassandraClient; import com.palantir.atlasdb.keyvalue.cassandra.thrift.SlicePredicates; import com.palantir.atlasdb.qos.QosClient; @@ -62,7 +63,7 @@ public void setUp() { public void multigetSliceChecksLimit() throws TException, LimitExceededException { client.multiget_slice("get", TEST_TABLE, ImmutableList.of(ROW_KEY), SLICE_PREDICATE, ConsistencyLevel.ANY); - verify(qosClient, times(1)).executeRead(any(), any(), any()); + verify(qosClient, times(1)).executeRead(any(), any()); verifyNoMoreInteractions(qosClient); } @@ -79,7 +80,7 @@ public void executeCqlQueryChecksLimit() throws TException, LimitExceededExcepti CqlQuery query = new CqlQuery("SELECT * FROM test_table LIMIT 1"); client.execute_cql3_query(query, Compression.NONE, ConsistencyLevel.ANY); - verify(qosClient, times(1)).executeRead(any(), any(), any()); + verify(qosClient, times(1)).executeRead(any(), any()); verifyNoMoreInteractions(qosClient); } @@ -87,7 +88,7 @@ public void executeCqlQueryChecksLimit() throws TException, LimitExceededExcepti public void getRangeSlicesChecksLimit() throws TException, LimitExceededException { client.get_range_slices("get", TEST_TABLE, SLICE_PREDICATE, new KeyRange(), ConsistencyLevel.ANY); - verify(qosClient, times(1)).executeRead(any(), any(), any()); + verify(qosClient, times(1)).executeRead(any(), any()); verifyNoMoreInteractions(qosClient); } } diff --git a/qos-service-api/build.gradle b/qos-service-api/build.gradle index 60b3ef8dbf4..421a6862cd3 100644 --- a/qos-service-api/build.gradle +++ b/qos-service-api/build.gradle @@ -17,5 +17,8 @@ dependencies { exclude (module:'okhttp') exclude (module:'jsr305') } + + processor group: 'org.immutables', name: 'value' + } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java index 4b24852a266..7aaa05293e0 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/FakeQosClient.java @@ -16,22 +16,19 @@ package com.palantir.atlasdb.qos; -import java.util.function.Function; -import java.util.function.Supplier; - public class FakeQosClient implements QosClient { public static final FakeQosClient INSTANCE = new FakeQosClient(); @Override - public T executeRead(Supplier estimatedWeight, ReadQuery query, - Function weigher) throws E { + public T executeRead(ReadQuery query, QueryWeigher weigher) + throws E { return query.execute(); } @Override - public void executeWrite(Supplier weight, WriteQuery query) - throws E { + public void executeWrite(WriteQuery query, + QueryWeigher weigher) throws E { query.execute(); } } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java index 26d85e9d4c0..2eb70896a3f 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java @@ -16,9 +16,6 @@ package com.palantir.atlasdb.qos; -import java.util.function.Function; -import java.util.function.Supplier; - public interface QosClient { interface ReadQuery { @@ -29,12 +26,19 @@ interface WriteQuery { void execute() throws E; } + interface QueryWeigher { + + QueryWeight estimate(); + + QueryWeight weigh(T result, long timeTakenNanos); + + } + T executeRead( - Supplier estimatedWeight, ReadQuery query, - Function weigher) throws E; + QueryWeigher weigher) throws E; void executeWrite( - Supplier weight, - WriteQuery query) throws E; + WriteQuery query, + QueryWeigher weigher) throws E; } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java index 469caed1427..1dced841626 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosService.java @@ -30,5 +30,5 @@ public interface QosService { @POST @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - int getLimit(@Safe @PathParam("client") String client); + long getLimit(@Safe @PathParam("client") String client); } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QueryWeight.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QueryWeight.java new file mode 100644 index 00000000000..6e36a73c4dc --- /dev/null +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QueryWeight.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.qos; + +import java.util.concurrent.TimeUnit; + +import org.immutables.value.Value; + +@Value.Immutable +public interface QueryWeight { + + long numBytes(); + + long numDistinctRows(); + + // TODO(nziebart): need to standardize everyhting to longs, and handle casting to int in QosRateLimiter + long timeTakenNanos(); + + default long timeTakenMicros() { + return TimeUnit.NANOSECONDS.toMicros(timeTakenNanos()); + } + +} diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java index b7443421cfb..ed199b69e4b 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosResource.java @@ -29,7 +29,7 @@ public QosResource(Supplier config) { } @Override - public int getLimit(String client) { - return config.get().clientLimits().getOrDefault(client, Integer.MAX_VALUE); + public long getLimit(String client) { + return config.get().clientLimits().getOrDefault(client, Long.MAX_VALUE); } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index 09dc7ca64fc..f1832ba2199 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -15,23 +15,22 @@ */ package com.palantir.atlasdb.qos.client; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Supplier; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; import com.palantir.atlasdb.qos.QosClient; -import com.palantir.atlasdb.qos.QosMetrics; +import com.palantir.atlasdb.qos.QueryWeight; +import com.palantir.atlasdb.qos.metrics.QosMetrics; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters; public class AtlasDbQosClient implements QosClient { private static final Logger log = LoggerFactory.getLogger(AtlasDbQosClient.class); + private static final Void NO_RESULT = null; + private final QosRateLimiters rateLimiters; private final QosMetrics metrics; private final Ticker ticker; @@ -48,50 +47,35 @@ public static AtlasDbQosClient create(QosRateLimiters rateLimiters) { } @Override - public T executeRead( - Supplier estimatedWeigher, - ReadQuery query, - Function weigher) throws E { - int estimatedWeight = getWeight(estimatedWeigher, 1); - rateLimiters.read().consumeWithBackoff(estimatedWeight); + public T executeRead(ReadQuery query, QueryWeigher weigher) throws E { + long estimatedNumBytes = weigher.estimate().numBytes(); + rateLimiters.read().consumeWithBackoff(estimatedNumBytes); // TODO(nziebart): decide what to do if we encounter a timeout exception long startTimeNanos = ticker.read(); T result = query.execute(); long totalTimeNanos = ticker.read() - startTimeNanos; - int actualWeight = getWeight(() -> weigher.apply(result), estimatedWeight); - metrics.updateReadCount(); - metrics.updateBytesRead(actualWeight); - metrics.updateReadTimeMicros(TimeUnit.NANOSECONDS.toMicros(totalTimeNanos)); - rateLimiters.read().recordAdjustment(actualWeight - estimatedWeight); + QueryWeight actualWeight = weigher.weigh(result, totalTimeNanos); + metrics.recordRead(actualWeight); + rateLimiters.read().recordAdjustment(actualWeight.numBytes() - estimatedNumBytes); return result; } @Override - public void executeWrite(Supplier weigher, WriteQuery query) throws E { - int weight = getWeight(weigher, 1); - rateLimiters.write().consumeWithBackoff(weight); + public void executeWrite(WriteQuery query, QueryWeigher weigher) throws E { + long estimatedNumBytes = weigher.estimate().numBytes(); + rateLimiters.write().consumeWithBackoff(estimatedNumBytes); // TODO(nziebart): decide what to do if we encounter a timeout exception long startTimeNanos = ticker.read(); query.execute(); long totalTimeNanos = ticker.read() - startTimeNanos; - metrics.updateWriteCount(); - metrics.updateWriteTimeMicros(TimeUnit.NANOSECONDS.toMicros(totalTimeNanos)); - metrics.updateBytesWritten(weight); - } - - // TODO(nziebart): error handling in the weight calculation should be responsibility of the caller - private Integer getWeight(Supplier weigher, int fallback) { - try { - return weigher.get(); - } catch (Exception e) { - log.warn("Exception while calculating response weight", e); - return fallback; - } + QueryWeight actualWeight = weigher.weigh(NO_RESULT, totalTimeNanos); + metrics.recordWrite(actualWeight); + rateLimiters.write().recordAdjustment(actualWeight.numBytes() - estimatedNumBytes); } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java index 0faa8222f34..8ce8305589d 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/config/QosServiceRuntimeConfig.java @@ -27,5 +27,5 @@ @JsonSerialize(as = ImmutableQosServiceRuntimeConfig.class) @Value.Immutable public abstract class QosServiceRuntimeConfig { - public abstract Map clientLimits(); + public abstract Map clientLimits(); } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/metrics/QosMetrics.java similarity index 69% rename from qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java rename to qos-service-impl/src/main/java/com/palantir/atlasdb/qos/metrics/QosMetrics.java index d3d8f2f0573..d8ee536efb2 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/QosMetrics.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/metrics/QosMetrics.java @@ -14,53 +14,51 @@ * limitations under the License. */ -package com.palantir.atlasdb.qos; +package com.palantir.atlasdb.qos.metrics; import com.codahale.metrics.Meter; +import com.palantir.atlasdb.qos.QueryWeight; import com.palantir.atlasdb.util.MetricsManager; +// TODO(nziebart): needs tests public class QosMetrics { private final MetricsManager metricsManager = new MetricsManager(); private final Meter readRequestCount; private final Meter bytesRead; private final Meter readTime; + private final Meter rowsRead; private final Meter writeRequestCount; private final Meter bytesWritten; private final Meter writeTime; + private final Meter rowsWritten; + public QosMetrics() { readRequestCount = metricsManager.registerMeter(QosMetrics.class, "numReadRequests"); bytesRead = metricsManager.registerMeter(QosMetrics.class, "bytesRead"); readTime = metricsManager.registerMeter(QosMetrics.class, "readTime"); + rowsRead = metricsManager.registerMeter(QosMetrics.class, "rowsRead"); writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); bytesWritten = metricsManager.registerMeter(QosMetrics.class, "bytesWritten"); writeTime = metricsManager.registerMeter(QosMetrics.class, "writeTime"); + rowsWritten = metricsManager.registerMeter(QosMetrics.class, "rowsWritten"); } - public void updateReadCount() { + public void recordRead(QueryWeight weight) { readRequestCount.mark(); + bytesRead.mark(weight.numBytes()); + readTime.mark(weight.timeTakenMicros()); + rowsRead.mark(weight.numDistinctRows()); } - public void updateWriteCount() { + public void recordWrite(QueryWeight weight) { writeRequestCount.mark(); + bytesWritten.mark(weight.numBytes()); + writeTime.mark(weight.timeTakenMicros()); + rowsWritten.mark(weight.numDistinctRows()); } - public void updateBytesRead(long numBytes) { - bytesRead.mark(numBytes); - } - - public void updateBytesWritten(long numBytes) { - bytesWritten.mark(numBytes); - } - - public void updateReadTimeMicros(long readTimeMicros) { - readTime.mark(readTimeMicros); - } - - public void updateWriteTimeMicros(long writeTimeMicros) { - readTime.mark(writeTimeMicros); - } } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java index 7bb5773349b..299a417f371 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/QosRateLimiter.java @@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Ints; /** * A rate limiter for database queries, based on "units" of expense. This limiter strives to maintain an upper limit on @@ -55,7 +56,7 @@ public static QosRateLimiter create(long maxBackoffTimeMillis) { /** * Update the allowed rate, in units per second. */ - public void updateRate(int unitsPerSecond) { + public void updateRate(double unitsPerSecond) { rateLimiter.setRate(unitsPerSecond); } @@ -65,9 +66,9 @@ public void updateRate(int unitsPerSecond) { * * @return the amount of time slept for, if any */ - public Duration consumeWithBackoff(int estimatedNumUnits) { + public Duration consumeWithBackoff(long estimatedNumUnits) { Optional waitTime = rateLimiter.tryAcquire( - estimatedNumUnits, + Ints.saturatedCast(estimatedNumUnits), // TODO(nziebart): deal with longs maxBackoffTimeMillis, TimeUnit.MILLISECONDS); @@ -79,13 +80,13 @@ public Duration consumeWithBackoff(int estimatedNumUnits) { } /** - * Records an adjustment to the original estimate of units consumed passed to {@link #consumeWithBackoff(int)}. This + * Records an adjustment to the original estimate of units consumed passed to {@link #consumeWithBackoff}. This * should be called after a query returns, when the exact number of units consumed is known. This value may be * positive (if the original estimate was too small) or negative (if the original estimate was too large). */ - public void recordAdjustment(int adjustmentUnits) { + public void recordAdjustment(long adjustmentUnits) { if (adjustmentUnits > 0) { - rateLimiter.steal(adjustmentUnits); + rateLimiter.steal(Ints.saturatedCast(adjustmentUnits)); // TODO(nziebart): deal with longs } // TODO(nziebart): handle negative case } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java index 55a662bb1cb..9edcedafa2b 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosRuntimeConfigDeserializationTest.java @@ -41,7 +41,7 @@ public void canDeserializeQosServerConfiguration() throws IOException { QosServiceRuntimeConfig configuration = OBJECT_MAPPER.readValue(testConfigFile, QosServiceRuntimeConfig.class); assertThat(configuration).isEqualTo(ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test", 10, "test2", 20)) + .clientLimits(ImmutableMap.of("test", 10L, "test2", 20L)) .build()); } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java index 129ac04db62..c5c81c11e23 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceRuntimeConfigTest.java @@ -30,14 +30,14 @@ public void canBuildFromEmptyClientLimits() { @Test public void canBuildFromSingleClientLimit() { ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test_client", 10)) + .clientLimits(ImmutableMap.of("test_client", 10L)) .build(); } @Test public void canBuildFromMultipleClientLimits() { ImmutableQosServiceRuntimeConfig.builder() - .clientLimits(ImmutableMap.of("test_client", 10, "test_client2", 100)) + .clientLimits(ImmutableMap.of("test_client", 10L, "test_client2", 100L)) .build(); } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java index 15fdeb1b64b..006cafef984 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/QosServiceTest.java @@ -39,20 +39,20 @@ public class QosServiceTest { public void defaultsToNoLimit() { when(config.get()).thenReturn(configWithLimits(ImmutableMap.of())); - assertThat(resource.getLimit("foo")).isEqualTo(Integer.MAX_VALUE); + assertThat(resource.getLimit("foo")).isEqualTo(Long.MAX_VALUE); } @Test public void canLiveReloadLimits() { when(config.get()) - .thenReturn(configWithLimits(ImmutableMap.of("foo", 10))) - .thenReturn(configWithLimits(ImmutableMap.of("foo", 20))); + .thenReturn(configWithLimits(ImmutableMap.of("foo", 10L))) + .thenReturn(configWithLimits(ImmutableMap.of("foo", 20L))); - assertEquals(10, resource.getLimit("foo")); - assertEquals(20, resource.getLimit("foo")); + assertEquals(10L, resource.getLimit("foo")); + assertEquals(20L, resource.getLimit("foo")); } - private QosServiceRuntimeConfig configWithLimits(Map limits) { + private QosServiceRuntimeConfig configWithLimits(Map limits) { return ImmutableQosServiceRuntimeConfig.builder().clientLimits(limits).build(); } } diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java index 27a8ddbda4a..4142408ce02 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java @@ -17,6 +17,8 @@ package com.palantir.atlasdb.qos.client; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -26,8 +28,11 @@ import org.junit.Test; import com.google.common.base.Ticker; -import com.palantir.atlasdb.qos.QosMetrics; +import com.palantir.atlasdb.qos.ImmutableQueryWeight; +import com.palantir.atlasdb.qos.QosClient; import com.palantir.atlasdb.qos.QosService; +import com.palantir.atlasdb.qos.QueryWeight; +import com.palantir.atlasdb.qos.metrics.QosMetrics; import com.palantir.atlasdb.qos.ratelimit.ImmutableQosRateLimiters; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiter; import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters; @@ -38,7 +43,21 @@ public class AtlasDbQosClientTest { private static final int ACTUAL_BYTES = 51; private static final long START_NANOS = 1100L; private static final long END_NANOS = 5500L; - private static final long TOTAL_TIME_MICROS = 4; + private static final long TOTAL_NANOS = END_NANOS - START_NANOS; + + private static final QueryWeight ESTIMATED_WEIGHT = ImmutableQueryWeight.builder() + .numBytes(ESTIMATED_BYTES) + .numDistinctRows(1) + .timeTakenNanos((int) TOTAL_NANOS) + .build(); + + private static final QueryWeight ACTUAL_WEIGHT = ImmutableQueryWeight.builder() + .numBytes(ACTUAL_BYTES) + .numDistinctRows(10) + .timeTakenNanos((int) TOTAL_NANOS) + .build(); + + private QosClient.QueryWeigher weigher = mock(QosClient.QueryWeigher.class); private QosService qosService = mock(QosService.class); private QosRateLimiter readLimiter = mock(QosRateLimiter.class); @@ -52,14 +71,17 @@ public class AtlasDbQosClientTest { @Before public void setUp() { - when(qosService.getLimit("test-client")).thenReturn(100); + when(qosService.getLimit("test-client")).thenReturn(100L); when(ticker.read()).thenReturn(START_NANOS).thenReturn(END_NANOS); + + when(weigher.estimate()).thenReturn(ESTIMATED_WEIGHT); + when(weigher.weigh(any(), anyLong())).thenReturn(ACTUAL_WEIGHT); } @Test public void consumesSpecifiedNumUnitsForReads() { - qosClient.executeRead(() -> ESTIMATED_BYTES, () -> "foo", ignored -> ACTUAL_BYTES); + qosClient.executeRead(() -> "foo", weigher); verify(readLimiter).consumeWithBackoff(ESTIMATED_BYTES); verify(readLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); @@ -68,42 +90,48 @@ public void consumesSpecifiedNumUnitsForReads() { @Test public void recordsReadMetrics() throws TestCheckedException { - qosClient.executeRead(() -> ESTIMATED_BYTES, () -> "foo", ignored -> ACTUAL_BYTES); + qosClient.executeRead(() -> "foo", weigher); - verify(metrics).updateReadCount(); - verify(metrics).updateBytesRead(ACTUAL_BYTES); - verify(metrics).updateReadTimeMicros(TOTAL_TIME_MICROS); + verify(metrics).recordRead(ACTUAL_WEIGHT); verifyNoMoreInteractions(metrics); } + @Test + public void passesResultAndTimeToReadWeigher() throws TestCheckedException { + qosClient.executeRead(() -> "foo", weigher); + + verify(weigher).weigh("foo", TOTAL_NANOS); + } + @Test public void consumesSpecifiedNumUnitsForWrites() { - qosClient.executeWrite(() -> ACTUAL_BYTES, () -> { }); + qosClient.executeWrite(() -> { + }, weigher); - verify(writeLimiter).consumeWithBackoff(ACTUAL_BYTES); + verify(writeLimiter).consumeWithBackoff(ESTIMATED_BYTES); + verify(writeLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); verifyNoMoreInteractions(readLimiter, writeLimiter); } @Test public void recordsWriteMetrics() throws TestCheckedException { - qosClient.executeWrite(() -> ACTUAL_BYTES, () -> { }); + qosClient.executeWrite(() -> { + }, weigher); - verify(metrics).updateWriteCount(); - verify(metrics).updateBytesWritten(ACTUAL_BYTES); - verify(metrics).updateWriteTimeMicros(TOTAL_TIME_MICROS); + verify(metrics).recordWrite(ACTUAL_WEIGHT); verifyNoMoreInteractions(metrics); } @Test public void propagatesCheckedExceptions() throws TestCheckedException { - assertThatThrownBy(() -> qosClient.executeRead(() -> 1, () -> { + assertThatThrownBy(() -> qosClient.executeRead(() -> { throw new TestCheckedException(); - }, ignored -> 1)).isInstanceOf(TestCheckedException.class); + }, weigher)).isInstanceOf(TestCheckedException.class); - assertThatThrownBy(() -> qosClient.executeWrite(() -> 1, () -> { + assertThatThrownBy(() -> qosClient.executeWrite(() -> { throw new TestCheckedException(); - })).isInstanceOf(TestCheckedException.class); + }, weigher)).isInstanceOf(TestCheckedException.class); } - static class TestCheckedException extends Exception { } + static class TestCheckedException extends Exception {} } From 710d2161fda5c677f9bea36a40882067c22f146e Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Thu, 16 Nov 2017 22:54:17 +0000 Subject: [PATCH 31/67] extra tests --- .../cassandra/qos/ThriftObjectSizeUtils.java | 2 +- .../cassandra/qos/ThriftQueryWeighers.java | 1 - .../atlasdb/ThriftObjectSizeUtilsTest.java | 62 +++++++++++++++++-- .../com/palantir/atlasdb/qos/QosClient.java | 3 - .../qos/client/AtlasDbQosClientTest.java | 10 +-- 5 files changed, 60 insertions(+), 18 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java index 986caaf06b0..c97473fccab 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java @@ -114,7 +114,7 @@ public static long getStringSize(String string) { return getNullSize(); } - return string.length() * Character.SIZE; + return string.length(); } public static long getColumnSize(Column column) { diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java index 45a4e79ae0c..2f31cf247cf 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java @@ -83,7 +83,6 @@ public QueryWeight weigh(T result, long timeTakenNanos) { } public static QosClient.QueryWeigher writeWeigher(Supplier bytesWritten) { - Supplier weight = Suppliers.memoize(() -> safeGetNumBytesOrDefault(bytesWritten))::get; return new QosClient.QueryWeigher() { diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index 25d26d25828..a55b655be25 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -19,6 +19,8 @@ import static org.assertj.core.api.Assertions.assertThat; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; @@ -33,15 +35,18 @@ import org.junit.Test; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.palantir.atlasdb.keyvalue.cassandra.qos.ThriftObjectSizeUtils; public class ThriftObjectSizeUtilsTest { - private static final String TEST_MAME = "test"; - private static final Column TEST_COLUMN = new Column(ByteBuffer.wrap(TEST_MAME.getBytes())); + private static final String TEST_MAME = "foo"; + private static final ByteBuffer TEST_NAME_BYTES = ByteBuffer.wrap(TEST_MAME.getBytes()); + private static final Column TEST_COLUMN = new Column(TEST_NAME_BYTES); - - private static final long TEST_COLUMN_SIZE = 4L + TEST_MAME.getBytes().length + 4L + 8L; + private static final long TEST_NAME_SIZE = 3L; + private static final long TEST_NAME_BYTES_SIZE = TEST_NAME_BYTES.remaining(); + private static final long TEST_COLUMN_SIZE = TEST_NAME_BYTES_SIZE + 4L + 4L + 8L; private static final ColumnOrSuperColumn EMPTY_COLUMN_OR_SUPERCOLUMN = new ColumnOrSuperColumn(); private static final long EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE = Integer.BYTES * 4; @@ -74,7 +79,7 @@ public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() { .setColumn(TEST_COLUMN) .setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), ImmutableList.of(TEST_COLUMN))))) - .isEqualTo(Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); + .isEqualTo(Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_NAME_BYTES_SIZE + TEST_COLUMN_SIZE); } @Test @@ -198,4 +203,51 @@ public void getSizeForKeySliceWithKeyAndColumns() { .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)))) .isEqualTo(TEST_MAME.getBytes().length + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); } + + @Test + public void getSizeForBatchMutate() { + Map>> batchMutateMap = ImmutableMap.of( + TEST_NAME_BYTES, + ImmutableMap.of( + TEST_MAME, + ImmutableList.of(new Mutation().setColumn_or_supercolumn(EMPTY_COLUMN_OR_SUPERCOLUMN)))); + + long expectedSize = TEST_NAME_BYTES_SIZE + + TEST_NAME_SIZE + + Integer.BYTES + + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE; + + assertThat(ThriftObjectSizeUtils.getApproximateWriteByteCount(batchMutateMap)).isEqualTo(expectedSize); + } + + @Test + public void getStringSize() { + assertThat(ThriftObjectSizeUtils.getStringSize(TEST_MAME)).isEqualTo(TEST_NAME_SIZE); + } + + @Test + public void getMultigetResultSize() { + Map> result = ImmutableMap.of( + TEST_NAME_BYTES, ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)); + + long expectedSize = TEST_NAME_BYTES_SIZE + + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE; + + assertThat(ThriftObjectSizeUtils.getApproximateReadByteCount(result)).isEqualTo(expectedSize); + } + + @Test + public void getKeySlicesSize() { + List slices = ImmutableList.of( + new KeySlice() + .setKey(TEST_NAME_BYTES) + .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN))); + + long expectedSize = TEST_NAME_BYTES_SIZE + + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE; + + assertThat(ThriftObjectSizeUtils.getApproximateReadByteCount(slices)).isEqualTo(expectedSize); + + } + } diff --git a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java index 2eb70896a3f..656be55b66b 100644 --- a/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java +++ b/qos-service-api/src/main/java/com/palantir/atlasdb/qos/QosClient.java @@ -27,11 +27,8 @@ interface WriteQuery { } interface QueryWeigher { - QueryWeight estimate(); - QueryWeight weigh(T result, long timeTakenNanos); - } T executeRead( diff --git a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java index 4142408ce02..b59df46e598 100644 --- a/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java +++ b/qos-service-impl/src/test/java/com/palantir/atlasdb/qos/client/AtlasDbQosClientTest.java @@ -30,7 +30,6 @@ import com.google.common.base.Ticker; import com.palantir.atlasdb.qos.ImmutableQueryWeight; import com.palantir.atlasdb.qos.QosClient; -import com.palantir.atlasdb.qos.QosService; import com.palantir.atlasdb.qos.QueryWeight; import com.palantir.atlasdb.qos.metrics.QosMetrics; import com.palantir.atlasdb.qos.ratelimit.ImmutableQosRateLimiters; @@ -59,7 +58,6 @@ public class AtlasDbQosClientTest { private QosClient.QueryWeigher weigher = mock(QosClient.QueryWeigher.class); - private QosService qosService = mock(QosService.class); private QosRateLimiter readLimiter = mock(QosRateLimiter.class); private QosRateLimiter writeLimiter = mock(QosRateLimiter.class); private QosRateLimiters rateLimiters = ImmutableQosRateLimiters.builder() @@ -71,8 +69,6 @@ public class AtlasDbQosClientTest { @Before public void setUp() { - when(qosService.getLimit("test-client")).thenReturn(100L); - when(ticker.read()).thenReturn(START_NANOS).thenReturn(END_NANOS); when(weigher.estimate()).thenReturn(ESTIMATED_WEIGHT); @@ -105,8 +101,7 @@ public void passesResultAndTimeToReadWeigher() throws TestCheckedException { @Test public void consumesSpecifiedNumUnitsForWrites() { - qosClient.executeWrite(() -> { - }, weigher); + qosClient.executeWrite(() -> { }, weigher); verify(writeLimiter).consumeWithBackoff(ESTIMATED_BYTES); verify(writeLimiter).recordAdjustment(ACTUAL_BYTES - ESTIMATED_BYTES); @@ -115,8 +110,7 @@ public void consumesSpecifiedNumUnitsForWrites() { @Test public void recordsWriteMetrics() throws TestCheckedException { - qosClient.executeWrite(() -> { - }, weigher); + qosClient.executeWrite(() -> { }, weigher); verify(metrics).recordWrite(ACTUAL_WEIGHT); verifyNoMoreInteractions(metrics); From 575b583d274ed3ac0ff1f335b07061253aa78c35 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Thu, 16 Nov 2017 23:50:00 +0000 Subject: [PATCH 32/67] num rows --- .../cassandra/qos/ThriftQueryWeighers.java | 30 ++++--- .../qos/ThriftQueryWeighersTest.java | 82 +++++++++++++++++++ 2 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java index 2f31cf247cf..129b5299dcd 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java @@ -49,22 +49,27 @@ public final class ThriftQueryWeighers { private ThriftQueryWeighers() { } public static final QosClient.QueryWeigher>> MULTIGET_SLICE = - readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount); + readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount, Map::size); public static final QosClient.QueryWeigher> GET_RANGE_SLICES = - readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount); + readWeigher(ThriftObjectSizeUtils::getApproximateReadByteCount, List::size); public static final QosClient.QueryWeigher GET = - readWeigher(ThriftObjectSizeUtils::getColumnOrSuperColumnSize); + readWeigher(ThriftObjectSizeUtils::getColumnOrSuperColumnSize, ignored -> 1); public static final QosClient.QueryWeigher EXECUTE_CQL3_QUERY = - readWeigher(ThriftObjectSizeUtils::getCqlResultSize); - - public static final QosClient.QueryWeigher batchMutate(Map>> mutationMap) { - return writeWeigher(() -> ThriftObjectSizeUtils.getApproximateWriteByteCount(mutationMap)); + // TODO(nziebart): we need to inspect the schema to see how many rows there are - a CQL row is NOT a partition + // rows here will depend on the type of query executed in CqlExecutor: either (column, ts) pairs, or + // (key, column, ts) triplets + readWeigher(ThriftObjectSizeUtils::getCqlResultSize, ignored -> 1); + + public static final QosClient.QueryWeigher batchMutate( + Map>> mutationMap) { + long numRows = ThriftObjectSizeUtils.getCollectionSize(mutationMap.values(), map -> (long) map.size()); + return writeWeigher(numRows, () -> ThriftObjectSizeUtils.getApproximateWriteByteCount(mutationMap)); } - public static QosClient.QueryWeigher readWeigher(Function bytesRead) { + public static QosClient.QueryWeigher readWeigher(Function bytesRead, Function numRows) { return new QosClient.QueryWeigher() { @Override public QueryWeight estimate() { @@ -75,14 +80,14 @@ public QueryWeight estimate() { public QueryWeight weigh(T result, long timeTakenNanos) { return ImmutableQueryWeight.builder() .numBytes(safeGetNumBytesOrDefault(() -> bytesRead.apply(result))) - .timeTakenNanos((int)timeTakenNanos) - .numDistinctRows(1) + .timeTakenNanos((int) timeTakenNanos) + .numDistinctRows(numRows.apply(result)) .build(); } }; } - public static QosClient.QueryWeigher writeWeigher(Supplier bytesWritten) { + public static QosClient.QueryWeigher writeWeigher(long numRows, Supplier bytesWritten) { Supplier weight = Suppliers.memoize(() -> safeGetNumBytesOrDefault(bytesWritten))::get; return new QosClient.QueryWeigher() { @@ -91,6 +96,7 @@ public QueryWeight estimate() { return ImmutableQueryWeight.builder() .from(DEFAULT_ESTIMATED_WEIGHT) .numBytes(weight.get()) + .numDistinctRows(numRows) .build(); } @@ -98,7 +104,7 @@ public QueryWeight estimate() { public QueryWeight weigh(T result, long timeTakenNanos) { return ImmutableQueryWeight.builder() .from(estimate()) - .timeTakenNanos((int)timeTakenNanos) + .timeTakenNanos((int) timeTakenNanos) .build(); } }; diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java new file mode 100644 index 00000000000..e7993634b99 --- /dev/null +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra.qos; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class ThriftQueryWeighersTest { + + private static final ByteBuffer BYTES1 = ByteBuffer.allocate(1); + private static final ByteBuffer BYTES2 = ByteBuffer.allocate(1); + private static final ColumnOrSuperColumn COLUMN = new ColumnOrSuperColumn(); + private static final KeySlice KEY_SLICE = new KeySlice(); + private static final Mutation MUTATION = new Mutation(); + + private static final long UNIMPORTANT_ARG = 123L; + + @Test + public void multigetSliceWeigherReturnsCorrectNumRows() { + Map> result = ImmutableMap.of( + BYTES1, ImmutableList.of(COLUMN, COLUMN), + BYTES2, ImmutableList.of(COLUMN)); + + assertThat(ThriftQueryWeighers.MULTIGET_SLICE.weigh(result, UNIMPORTANT_ARG)).isEqualTo(2); + } + + @Test + public void rangeSlicesWeigherReturnsCorrectNumRows() { + List result = ImmutableList.of(KEY_SLICE, KEY_SLICE, KEY_SLICE); + + assertThat(ThriftQueryWeighers.GET_RANGE_SLICES.weigh(result, UNIMPORTANT_ARG)).isEqualTo(3); + } + + @Test + public void getWeigherReturnsCorrectNumRows() { + assertThat(ThriftQueryWeighers.GET.weigh(COLUMN, UNIMPORTANT_ARG)).isEqualTo(1); + } + + @Test + public void executeCql3QueryReturnsOneRowAlways() { + assertThat(ThriftQueryWeighers.EXECUTE_CQL3_QUERY.weigh(new CqlResult(), UNIMPORTANT_ARG)).isEqualTo(1); + } + + @Test + public void batchMutateWeigherReturnsCorrectNumRows() { + Map>> mutations = ImmutableMap.of( + BYTES1, ImmutableMap.of( + "foo", ImmutableList.of(MUTATION, MUTATION), + "bar", ImmutableList.of(MUTATION)), + BYTES2, ImmutableMap.of( + "baz", ImmutableList.of(MUTATION))); + + assertThat(ThriftQueryWeighers.batchMutate(mutations)).isEqualTo(3); + } + +} From c9b48396f23456759019a9ec5b294dd47f6edfd9 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Thu, 16 Nov 2017 23:57:50 +0000 Subject: [PATCH 33/67] checkstyle --- .../keyvalue/cassandra/qos/ThriftObjectSizeUtils.java | 6 +++--- .../keyvalue/cassandra/qos/ThriftQueryWeighers.java | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java index c97473fccab..42afbd151a6 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftObjectSizeUtils.java @@ -46,10 +46,10 @@ private ThriftObjectSizeUtils() { public static long getApproximateWriteByteCount(Map>> batchMutateMap) { long approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); - long approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> - getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + long approxBytesForValues = getCollectionSize(batchMutateMap.values(), + currentMap -> getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + getCollectionSize(currentMap.values(), - mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); + mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); return approxBytesForKeys + approxBytesForValues; } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java index 129b5299dcd..62f991d4eb8 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java @@ -58,12 +58,12 @@ private ThriftQueryWeighers() { } readWeigher(ThriftObjectSizeUtils::getColumnOrSuperColumnSize, ignored -> 1); public static final QosClient.QueryWeigher EXECUTE_CQL3_QUERY = - // TODO(nziebart): we need to inspect the schema to see how many rows there are - a CQL row is NOT a partition - // rows here will depend on the type of query executed in CqlExecutor: either (column, ts) pairs, or - // (key, column, ts) triplets + // TODO(nziebart): we need to inspect the schema to see how many rows there are - a CQL row is NOT a + // partition. rows here will depend on the type of query executed in CqlExecutor: either (column, ts) pairs, + // or (key, column, ts) triplets readWeigher(ThriftObjectSizeUtils::getCqlResultSize, ignored -> 1); - public static final QosClient.QueryWeigher batchMutate( + public static QosClient.QueryWeigher batchMutate( Map>> mutationMap) { long numRows = ThriftObjectSizeUtils.getCollectionSize(mutationMap.values(), map -> (long) map.size()); return writeWeigher(numRows, () -> ThriftObjectSizeUtils.getApproximateWriteByteCount(mutationMap)); From 29ca959cd6c19bb669308811fec4558e37b92fb2 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Fri, 17 Nov 2017 11:59:09 +0000 Subject: [PATCH 34/67] fix tests --- .../qos/ThriftQueryWeighersTest.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java index e7993634b99..86cbb642d51 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighersTest.java @@ -33,8 +33,8 @@ public class ThriftQueryWeighersTest { - private static final ByteBuffer BYTES1 = ByteBuffer.allocate(1); - private static final ByteBuffer BYTES2 = ByteBuffer.allocate(1); + private static final ByteBuffer BYTES1 = ByteBuffer.allocate(3); + private static final ByteBuffer BYTES2 = ByteBuffer.allocate(7); private static final ColumnOrSuperColumn COLUMN = new ColumnOrSuperColumn(); private static final KeySlice KEY_SLICE = new KeySlice(); private static final Mutation MUTATION = new Mutation(); @@ -47,24 +47,33 @@ public void multigetSliceWeigherReturnsCorrectNumRows() { BYTES1, ImmutableList.of(COLUMN, COLUMN), BYTES2, ImmutableList.of(COLUMN)); - assertThat(ThriftQueryWeighers.MULTIGET_SLICE.weigh(result, UNIMPORTANT_ARG)).isEqualTo(2); + long actualNumRows = ThriftQueryWeighers.MULTIGET_SLICE.weigh(result, UNIMPORTANT_ARG).numDistinctRows(); + + assertThat(actualNumRows).isEqualTo(2); } @Test public void rangeSlicesWeigherReturnsCorrectNumRows() { List result = ImmutableList.of(KEY_SLICE, KEY_SLICE, KEY_SLICE); - assertThat(ThriftQueryWeighers.GET_RANGE_SLICES.weigh(result, UNIMPORTANT_ARG)).isEqualTo(3); + long actualNumRows = ThriftQueryWeighers.GET_RANGE_SLICES.weigh(result, UNIMPORTANT_ARG).numDistinctRows(); + + assertThat(actualNumRows).isEqualTo(3); } @Test public void getWeigherReturnsCorrectNumRows() { - assertThat(ThriftQueryWeighers.GET.weigh(COLUMN, UNIMPORTANT_ARG)).isEqualTo(1); + long actualNumRows = ThriftQueryWeighers.GET.weigh(COLUMN, UNIMPORTANT_ARG).numDistinctRows(); + + assertThat(actualNumRows).isEqualTo(1); } @Test - public void executeCql3QueryReturnsOneRowAlways() { - assertThat(ThriftQueryWeighers.EXECUTE_CQL3_QUERY.weigh(new CqlResult(), UNIMPORTANT_ARG)).isEqualTo(1); + public void executeCql3QueryWeigherReturnsOneRowAlways() { + long actualNumRows = ThriftQueryWeighers.EXECUTE_CQL3_QUERY.weigh(new CqlResult(), + UNIMPORTANT_ARG).numDistinctRows(); + + assertThat(actualNumRows).isEqualTo(1); } @Test @@ -76,7 +85,10 @@ public void batchMutateWeigherReturnsCorrectNumRows() { BYTES2, ImmutableMap.of( "baz", ImmutableList.of(MUTATION))); - assertThat(ThriftQueryWeighers.batchMutate(mutations)).isEqualTo(3); + long actualNumRows = ThriftQueryWeighers.batchMutate(mutations).weigh(null, UNIMPORTANT_ARG) + .numDistinctRows(); + + assertThat(actualNumRows).isEqualTo(3); } } From b35bb7f44852b10395244c9b89507cddef80a052 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Fri, 17 Nov 2017 13:00:38 +0000 Subject: [PATCH 35/67] no int casting --- .../atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java index 62f991d4eb8..68c5abd7cf5 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/ThriftQueryWeighers.java @@ -80,7 +80,7 @@ public QueryWeight estimate() { public QueryWeight weigh(T result, long timeTakenNanos) { return ImmutableQueryWeight.builder() .numBytes(safeGetNumBytesOrDefault(() -> bytesRead.apply(result))) - .timeTakenNanos((int) timeTakenNanos) + .timeTakenNanos(timeTakenNanos) .numDistinctRows(numRows.apply(result)) .build(); } @@ -104,7 +104,7 @@ public QueryWeight estimate() { public QueryWeight weigh(T result, long timeTakenNanos) { return ImmutableQueryWeight.builder() .from(estimate()) - .timeTakenNanos((int) timeTakenNanos) + .timeTakenNanos(timeTakenNanos) .build(); } }; From f014f5668b40224de30184156cd8075e665f7ba4 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 15:47:21 +0000 Subject: [PATCH 36/67] Qos ete tests --- .../docker-compose.qos.cassandra.yml | 32 ++++++++++++++ atlasdb-ete-tests/docker-compose.yml | 32 ++++++++++++++ .../docker/conf/atlasdb-ete.qos.cassandra.yml | 35 ++++++++++++++++ .../atlasdb/AtlasDbEteConfiguration.java | 1 + .../atlasdb/ete/QosCassandraTestSuite.java | 42 +++++++++++++++++++ qos-ete-tests/build.gradle | 25 +++++++++++ .../src/main/java/QosServiceEteTest.java | 28 +++++++++++++ settings.gradle | 1 + 8 files changed, 196 insertions(+) create mode 100644 atlasdb-ete-tests/docker-compose.qos.cassandra.yml create mode 100644 atlasdb-ete-tests/docker-compose.yml create mode 100644 atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml create mode 100644 atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java create mode 100644 qos-ete-tests/build.gradle create mode 100644 qos-ete-tests/src/main/java/QosServiceEteTest.java diff --git a/atlasdb-ete-tests/docker-compose.qos.cassandra.yml b/atlasdb-ete-tests/docker-compose.qos.cassandra.yml new file mode 100644 index 00000000000..ceacb6f135a --- /dev/null +++ b/atlasdb-ete-tests/docker-compose.qos.cassandra.yml @@ -0,0 +1,32 @@ +version: '2' + +services: + cassandra: + image: palantirtechnologies/docker-cassandra-atlasdb:$CASSANDRA_VERSION + ports: + - "9160" + - "9042" + - "7199" + environment: + - MAX_HEAP_SIZE=512m + - HEAP_NEWSIZE=64m + - LOCAL_JMX=no + + ete1: + build: . + command: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && dockerize -timeout 120s -wait tcp://cassandra:9160 && service/bin/init.sh console'] + ports: + - "3828" + depends_on: + - cassandra + environment: + - ME=ete1 + + ete-cli: + build: . + entrypoint: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && "$$@"', --] + command: exit + depends_on: + - ete1 + environment: + - ME=ete1 diff --git a/atlasdb-ete-tests/docker-compose.yml b/atlasdb-ete-tests/docker-compose.yml new file mode 100644 index 00000000000..ceacb6f135a --- /dev/null +++ b/atlasdb-ete-tests/docker-compose.yml @@ -0,0 +1,32 @@ +version: '2' + +services: + cassandra: + image: palantirtechnologies/docker-cassandra-atlasdb:$CASSANDRA_VERSION + ports: + - "9160" + - "9042" + - "7199" + environment: + - MAX_HEAP_SIZE=512m + - HEAP_NEWSIZE=64m + - LOCAL_JMX=no + + ete1: + build: . + command: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && dockerize -timeout 120s -wait tcp://cassandra:9160 && service/bin/init.sh console'] + ports: + - "3828" + depends_on: + - cassandra + environment: + - ME=ete1 + + ete-cli: + build: . + entrypoint: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && "$$@"', --] + command: exit + depends_on: + - ete1 + environment: + - ME=ete1 diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml new file mode 100644 index 00000000000..0bdbc18750f --- /dev/null +++ b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml @@ -0,0 +1,35 @@ +server: + #rootPath: /api/* + #applicationContextPath: / + #adminContextPath: / + applicationConnectors: + - type: http + port: 3828 + adminConnectors: + - type: http + port: 3829 + +atlasdb: + keyValueService: + type: cassandra + servers: + - cassandra:9160 + poolSize: 20 + keyspace: atlasete + credentials: + username: cassandra + password: cassandra + ssl: false + replicationFactor: 1 + mutationBatchCount: 10000 + mutationBatchSizeBytes: 10000000 + fetchBatchCount: 1000 + autoRefreshNodes: false + +atlasdbRuntime: + qos: + limits: + readBytesPerSecond: 10 + writeBytesPerSecond: 100 + maxBackoffSleepTime: 1 + diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java index d35e8acebbb..ba9fbabeaca 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java @@ -32,4 +32,5 @@ public AtlasDbEteConfiguration(@JsonProperty("atlasdb") AtlasDbConfig atlasdb) { public AtlasDbConfig getAtlasDbConfig() { return atlasdb; } + } diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java new file mode 100644 index 00000000000..2da1850bdec --- /dev/null +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.palantir.atlasdb.ete; + +import java.util.List; + +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import com.google.common.collect.ImmutableList; +import com.palantir.atlasdb.containers.CassandraEnvironment; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + TodoEteTest.class, + CommandLineEteTest.class, + }) +public class QosCassandraTestSuite extends EteSetup { + private static final List CLIENTS = ImmutableList.of("ete1"); + + @ClassRule + public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( + QosCassandraTestSuite.class, + "docker-compose.qos.cassandra.yml", + CLIENTS, + CassandraEnvironment.get()); +} diff --git a/qos-ete-tests/build.gradle b/qos-ete-tests/build.gradle new file mode 100644 index 00000000000..ceb069aa550 --- /dev/null +++ b/qos-ete-tests/build.gradle @@ -0,0 +1,25 @@ +apply from: "../gradle/publish-jars.gradle" +apply from: "../gradle/shared.gradle" + +apply plugin: 'org.inferred.processors' + +repositories { + mavenCentral() +} + +libsDirName = file('build/artifacts') + +dependencies { + compile group: 'com.palantir.safe-logging', name: 'safe-logging' + compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-core' + compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-core' + compile project(":atlasdb-ete-test-utils") + + compile group: 'junit', name: 'junit' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + compile (group: 'com.palantir.remoting3', name: 'jaxrs-clients') { + exclude (module:'okhttp') + exclude (module:'jsr305') + } +} + diff --git a/qos-ete-tests/src/main/java/QosServiceEteTest.java b/qos-ete-tests/src/main/java/QosServiceEteTest.java new file mode 100644 index 00000000000..8b6c191ffa1 --- /dev/null +++ b/qos-ete-tests/src/main/java/QosServiceEteTest.java @@ -0,0 +1,28 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.ClassRule; +import org.junit.rules.RuleChain; + +import com.palantir.atlasdb.ete.Gradle; + +public class QosServiceEteTest { + private static final Gradle GRADLE_PREPARE_TASK = Gradle.ensureTaskHasRun(":atlasdb-ete-tests:prepareForEteTests"); + + @ClassRule + public static final RuleChain RULE_CHAIN = RuleChain.outerRule(GRADLE_PREPARE_TASK); + +} diff --git a/settings.gradle b/settings.gradle index f5a3fcc6ce7..ea373ae1251 100644 --- a/settings.gradle +++ b/settings.gradle @@ -41,6 +41,7 @@ include ":commons-proxy" include ":cassandra-partitioner" include ":qos-service-api" include ":qos-service-impl" +include ":qos-ete-tests" include ":timelock-agent" include ":timelock-impl" include ":timelock-server" From 46e1308bf38c3f4c9ec8037c3b7a232225b5746e Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 17:02:13 +0000 Subject: [PATCH 37/67] shouldFailIfWritingTooManyBytes --- .../AtlasDbConfigurationProvider.java | 5 ++++ .../commands/AtlasDbConsoleCommandTest.java | 7 +++++ .../docker/conf/atlasdb-ete.qos.cassandra.yml | 5 ++-- .../atlasdb/AtlasDbEteConfiguration.java | 13 ++++++++- .../palantir/atlasdb/AtlasDbEteServer.java | 16 +++++++---- .../atlasdb/ete/QosCassandraTestSuite.java | 23 ++++++++++----- qos-ete-tests/build.gradle | 25 ----------------- .../src/main/java/QosServiceEteTest.java | 28 ------------------- settings.gradle | 1 - 9 files changed, 53 insertions(+), 70 deletions(-) delete mode 100644 qos-ete-tests/build.gradle delete mode 100644 qos-ete-tests/src/main/java/QosServiceEteTest.java diff --git a/atlasdb-dropwizard-bundle/src/main/java/com/palantir/atlasdb/dropwizard/AtlasDbConfigurationProvider.java b/atlasdb-dropwizard-bundle/src/main/java/com/palantir/atlasdb/dropwizard/AtlasDbConfigurationProvider.java index 14de47696c1..3020c18668b 100644 --- a/atlasdb-dropwizard-bundle/src/main/java/com/palantir/atlasdb/dropwizard/AtlasDbConfigurationProvider.java +++ b/atlasdb-dropwizard-bundle/src/main/java/com/palantir/atlasdb/dropwizard/AtlasDbConfigurationProvider.java @@ -15,8 +15,13 @@ */ package com.palantir.atlasdb.dropwizard; +import java.util.Optional; + import com.palantir.atlasdb.config.AtlasDbConfig; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; public interface AtlasDbConfigurationProvider { AtlasDbConfig getAtlasDbConfig(); + + Optional getAtlasDbRuntimeConfig(); } diff --git a/atlasdb-dropwizard-bundle/src/test/java/com/palantir/atlasdb/dropwizard/commands/AtlasDbConsoleCommandTest.java b/atlasdb-dropwizard-bundle/src/test/java/com/palantir/atlasdb/dropwizard/commands/AtlasDbConsoleCommandTest.java index 635a642c3dc..2063efe73d5 100644 --- a/atlasdb-dropwizard-bundle/src/test/java/com/palantir/atlasdb/dropwizard/commands/AtlasDbConsoleCommandTest.java +++ b/atlasdb-dropwizard-bundle/src/test/java/com/palantir/atlasdb/dropwizard/commands/AtlasDbConsoleCommandTest.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; @@ -30,6 +31,7 @@ import com.google.common.base.Throwables; import com.palantir.atlasdb.config.AtlasDbConfig; import com.palantir.atlasdb.config.AtlasDbConfigs; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; import com.palantir.atlasdb.config.ImmutableLeaderConfig; import com.palantir.atlasdb.dropwizard.AtlasDbConfigurationProvider; @@ -100,5 +102,10 @@ private class AtlasDbDropwizardConfig extends Configuration implements AtlasDbCo public AtlasDbConfig getAtlasDbConfig() { return atlasDbConfig; } + + @Override + public Optional getAtlasDbRuntimeConfig() { + return Optional.empty(); + } } } diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml index 0bdbc18750f..de961db2036 100644 --- a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml +++ b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml @@ -29,7 +29,6 @@ atlasdb: atlasdbRuntime: qos: limits: - readBytesPerSecond: 10 - writeBytesPerSecond: 100 - maxBackoffSleepTime: 1 + readBytesPerSecond: 10000 + writeBytesPerSecond: 10000 diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java index ba9fbabeaca..2729b38d1e2 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java @@ -15,17 +15,23 @@ */ package com.palantir.atlasdb; +import java.util.Optional; + import com.fasterxml.jackson.annotation.JsonProperty; import com.palantir.atlasdb.config.AtlasDbConfig; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; import com.palantir.atlasdb.dropwizard.AtlasDbConfigurationProvider; import io.dropwizard.Configuration; public class AtlasDbEteConfiguration extends Configuration implements AtlasDbConfigurationProvider { private final AtlasDbConfig atlasdb; + private final Optional atlasdbRuntime; - public AtlasDbEteConfiguration(@JsonProperty("atlasdb") AtlasDbConfig atlasdb) { + public AtlasDbEteConfiguration(@JsonProperty("atlasdb") AtlasDbConfig atlasdb, + @JsonProperty("atlasdbRuntime") Optional atlasDbRuntimeConfig) { this.atlasdb = atlasdb; + this.atlasdbRuntime = atlasDbRuntimeConfig; } @Override @@ -33,4 +39,9 @@ public AtlasDbConfig getAtlasDbConfig() { return atlasdb; } + @Override + public Optional getAtlasDbRuntimeConfig() { + return atlasdbRuntime; + } + } diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java index fb01dc45baa..1298a7a8407 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java @@ -15,6 +15,7 @@ */ package com.palantir.atlasdb; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -28,6 +29,7 @@ import com.palantir.atlasdb.cas.CheckAndSetSchema; import com.palantir.atlasdb.cas.SimpleCheckAndSetResource; import com.palantir.atlasdb.config.AtlasDbConfig; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; import com.palantir.atlasdb.dropwizard.AtlasDbBundle; import com.palantir.atlasdb.factory.TransactionManagers; import com.palantir.atlasdb.http.NotInitializedExceptionMapper; @@ -77,18 +79,20 @@ public void run(AtlasDbEteConfiguration config, final Environment environment) t private TransactionManager tryToCreateTransactionManager(AtlasDbEteConfiguration config, Environment environment) throws InterruptedException { if (config.getAtlasDbConfig().initializeAsync()) { - return createTransactionManager(config.getAtlasDbConfig(), environment); + return createTransactionManager(config.getAtlasDbConfig(), environment, config.getAtlasDbRuntimeConfig()); } else { - return createTransactionManagerWithRetry(config.getAtlasDbConfig(), environment); + return createTransactionManagerWithRetry(config.getAtlasDbConfig(), environment, + config.getAtlasDbRuntimeConfig()); } } - private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig config, Environment environment) + private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig config, Environment environment, + Optional atlasDbRuntimeConfig) throws InterruptedException { Stopwatch sw = Stopwatch.createStarted(); while (sw.elapsed(TimeUnit.SECONDS) < CREATE_TRANSACTION_MANAGER_MAX_WAIT_TIME_SECS) { try { - return createTransactionManager(config, environment); + return createTransactionManager(config, environment, atlasDbRuntimeConfig); } catch (RuntimeException e) { log.warn("An error occurred while trying to create transaction manager. Retrying...", e); Thread.sleep(CREATE_TRANSACTION_MANAGER_POLL_INTERVAL_SECS); @@ -97,12 +101,14 @@ private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig confi throw new IllegalStateException("Timed-out because we were unable to create transaction manager"); } - private TransactionManager createTransactionManager(AtlasDbConfig config, Environment environment) { + private TransactionManager createTransactionManager(AtlasDbConfig config, Environment environment, + Optional atlasDbRuntimeConfigOptional) { return TransactionManagers.builder() .config(config) .schemas(ETE_SCHEMAS) .registrar(environment.jersey()::register) .userAgent("ete test") + .runtimeConfigSupplier(() -> atlasDbRuntimeConfigOptional) .buildSerializable(); } diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 2da1850bdec..70d6b2790dd 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -15,23 +15,24 @@ */ package com.palantir.atlasdb.ete; +import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; + +import java.util.Collections; import java.util.List; import org.junit.ClassRule; +import org.junit.Test; import org.junit.rules.RuleChain; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; import com.google.common.collect.ImmutableList; import com.palantir.atlasdb.containers.CassandraEnvironment; +import com.palantir.atlasdb.todo.ImmutableTodo; +import com.palantir.atlasdb.todo.Todo; +import com.palantir.atlasdb.todo.TodoResource; -@RunWith(Suite.class) -@Suite.SuiteClasses({ - TodoEteTest.class, - CommandLineEteTest.class, - }) public class QosCassandraTestSuite extends EteSetup { private static final List CLIENTS = ImmutableList.of("ete1"); + private static final Todo TODO = ImmutableTodo.of(String.join("", Collections.nCopies(10_000, "a"))); @ClassRule public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( @@ -39,4 +40,12 @@ public class QosCassandraTestSuite extends EteSetup { "docker-compose.qos.cassandra.yml", CLIENTS, CassandraEnvironment.get()); + + @Test + public void shouldFailIfWritingTooManyBytes() { + TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + assertThatThrownBy(() -> todoClient.addTodo(TODO)) + .isInstanceOf(RuntimeException.class) + .hasMessage("rate limited"); + } } diff --git a/qos-ete-tests/build.gradle b/qos-ete-tests/build.gradle deleted file mode 100644 index ceb069aa550..00000000000 --- a/qos-ete-tests/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -apply from: "../gradle/publish-jars.gradle" -apply from: "../gradle/shared.gradle" - -apply plugin: 'org.inferred.processors' - -repositories { - mavenCentral() -} - -libsDirName = file('build/artifacts') - -dependencies { - compile group: 'com.palantir.safe-logging', name: 'safe-logging' - compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-core' - compile group: 'com.palantir.docker.compose', name: 'docker-compose-rule-core' - compile project(":atlasdb-ete-test-utils") - - compile group: 'junit', name: 'junit' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind' - compile (group: 'com.palantir.remoting3', name: 'jaxrs-clients') { - exclude (module:'okhttp') - exclude (module:'jsr305') - } -} - diff --git a/qos-ete-tests/src/main/java/QosServiceEteTest.java b/qos-ete-tests/src/main/java/QosServiceEteTest.java deleted file mode 100644 index 8b6c191ffa1..00000000000 --- a/qos-ete-tests/src/main/java/QosServiceEteTest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2017 Palantir Technologies, Inc. All rights reserved. - * - * Licensed under the BSD-3 License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.junit.ClassRule; -import org.junit.rules.RuleChain; - -import com.palantir.atlasdb.ete.Gradle; - -public class QosServiceEteTest { - private static final Gradle GRADLE_PREPARE_TASK = Gradle.ensureTaskHasRun(":atlasdb-ete-tests:prepareForEteTests"); - - @ClassRule - public static final RuleChain RULE_CHAIN = RuleChain.outerRule(GRADLE_PREPARE_TASK); - -} diff --git a/settings.gradle b/settings.gradle index ea373ae1251..f5a3fcc6ce7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -41,7 +41,6 @@ include ":commons-proxy" include ":cassandra-partitioner" include ":qos-service-api" include ":qos-service-impl" -include ":qos-ete-tests" include ":timelock-agent" include ":timelock-impl" include ":timelock-server" From e6035d0668c4dd3fe0f1a91c7decf360484cdc53 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 17:56:57 +0000 Subject: [PATCH 38/67] fix test --- .../docker/conf/atlasdb-ete.qos.cassandra.yml | 2 +- .../palantir/atlasdb/ete/QosCassandraTestSuite.java | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml index de961db2036..0e1d14d0d40 100644 --- a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml +++ b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml @@ -30,5 +30,5 @@ atlasdbRuntime: qos: limits: readBytesPerSecond: 10000 - writeBytesPerSecond: 10000 + writeBytesPerSecond: 1000 diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 70d6b2790dd..ec84132436e 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -15,6 +15,7 @@ */ package com.palantir.atlasdb.ete; +import static org.assertj.core.api.Java6Assertions.assertThat; import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; import java.util.Collections; @@ -24,15 +25,17 @@ import org.junit.Test; import org.junit.rules.RuleChain; +import com.codahale.metrics.Meter; import com.google.common.collect.ImmutableList; import com.palantir.atlasdb.containers.CassandraEnvironment; import com.palantir.atlasdb.todo.ImmutableTodo; import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoResource; +import com.palantir.atlasdb.util.AtlasDbMetrics; public class QosCassandraTestSuite extends EteSetup { private static final List CLIENTS = ImmutableList.of("ete1"); - private static final Todo TODO = ImmutableTodo.of(String.join("", Collections.nCopies(10_000, "a"))); + private static final Todo TODO = ImmutableTodo.of(String.join("", Collections.nCopies(100_000, "a"))); @ClassRule public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( @@ -44,8 +47,11 @@ public class QosCassandraTestSuite extends EteSetup { @Test public void shouldFailIfWritingTooManyBytes() { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + todoClient.addTodo(TODO); assertThatThrownBy(() -> todoClient.addTodo(TODO)) - .isInstanceOf(RuntimeException.class) - .hasMessage("rate limited"); + .isInstanceOf(RuntimeException.class); + Meter meter = AtlasDbMetrics.getMetricRegistry().getMeters().get( + "com.palantir.atlasdb.qos.metrics.QosMetrics.numReadRequests"); + assertThat(meter.getCount()).isGreaterThan(200_000).isLessThan(201_000); } } From 887aa5b0dcbcee0fe5f2d4a8c1a34f2a93c3c5b8 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 17:57:30 +0000 Subject: [PATCH 39/67] rm file --- atlasdb-ete-tests/docker-compose.yml | 32 ---------------------------- 1 file changed, 32 deletions(-) delete mode 100644 atlasdb-ete-tests/docker-compose.yml diff --git a/atlasdb-ete-tests/docker-compose.yml b/atlasdb-ete-tests/docker-compose.yml deleted file mode 100644 index ceacb6f135a..00000000000 --- a/atlasdb-ete-tests/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2' - -services: - cassandra: - image: palantirtechnologies/docker-cassandra-atlasdb:$CASSANDRA_VERSION - ports: - - "9160" - - "9042" - - "7199" - environment: - - MAX_HEAP_SIZE=512m - - HEAP_NEWSIZE=64m - - LOCAL_JMX=no - - ete1: - build: . - command: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && dockerize -timeout 120s -wait tcp://cassandra:9160 && service/bin/init.sh console'] - ports: - - "3828" - depends_on: - - cassandra - environment: - - ME=ete1 - - ete-cli: - build: . - entrypoint: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && "$$@"', --] - command: exit - depends_on: - - ete1 - environment: - - ME=ete1 From 3a8f4675e8022054bfd5b7a072a5f7d74f337c20 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 18:12:06 +0000 Subject: [PATCH 40/67] Remove metrics --- .../java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index ec84132436e..5b95e449cd6 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -15,7 +15,6 @@ */ package com.palantir.atlasdb.ete; -import static org.assertj.core.api.Java6Assertions.assertThat; import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; import java.util.Collections; @@ -25,7 +24,6 @@ import org.junit.Test; import org.junit.rules.RuleChain; -import com.codahale.metrics.Meter; import com.google.common.collect.ImmutableList; import com.palantir.atlasdb.containers.CassandraEnvironment; import com.palantir.atlasdb.todo.ImmutableTodo; @@ -50,8 +48,5 @@ public void shouldFailIfWritingTooManyBytes() { todoClient.addTodo(TODO); assertThatThrownBy(() -> todoClient.addTodo(TODO)) .isInstanceOf(RuntimeException.class); - Meter meter = AtlasDbMetrics.getMetricRegistry().getMeters().get( - "com.palantir.atlasdb.qos.metrics.QosMetrics.numReadRequests"); - assertThat(meter.getCount()).isGreaterThan(200_000).isLessThan(201_000); } } From 08a07b2f503fce1fc9de205cd009be81c9aa2f7a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 20:36:26 +0000 Subject: [PATCH 41/67] Test shouldFailIfReadingTooManyBytes --- .../docker/conf/atlasdb-ete.qos.cassandra.yml | 2 +- .../com/palantir/atlasdb/todo/TodoClient.java | 2 +- .../atlasdb/ete/QosCassandraTestSuite.java | 33 ++++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml index 0e1d14d0d40..a880e325a28 100644 --- a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml +++ b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml @@ -29,6 +29,6 @@ atlasdb: atlasdbRuntime: qos: limits: - readBytesPerSecond: 10000 + readBytesPerSecond: 1000 writeBytesPerSecond: 1000 diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java index 34a41afbbfe..524a39193b5 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java @@ -54,7 +54,7 @@ public List getTodoList() { TodoSchema.todoTable(), RangeRequest.all()); ImmutableList.Builder> rowResults = ImmutableList.builder(); - rowResultBatchingVisitable.batchAccept(1000, items -> { + rowResultBatchingVisitable.batchAccept(10, items -> { rowResults.addAll(items); return true; }); diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 5b95e449cd6..9114fefd68f 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -19,21 +19,23 @@ import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import org.junit.After; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.RuleChain; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.containers.CassandraEnvironment; import com.palantir.atlasdb.todo.ImmutableTodo; import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoResource; -import com.palantir.atlasdb.util.AtlasDbMetrics; public class QosCassandraTestSuite extends EteSetup { private static final List CLIENTS = ImmutableList.of("ete1"); - private static final Todo TODO = ImmutableTodo.of(String.join("", Collections.nCopies(100_000, "a"))); @ClassRule public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( @@ -45,8 +47,29 @@ public class QosCassandraTestSuite extends EteSetup { @Test public void shouldFailIfWritingTooManyBytes() { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - todoClient.addTodo(TODO); - assertThatThrownBy(() -> todoClient.addTodo(TODO)) - .isInstanceOf(RuntimeException.class); + // Doesn't throw the first time as we estimate low values + todoClient.addTodo(getTodoOfSize(100_000)); + assertThatThrownBy(() -> todoClient.addTodo(getTodoOfSize(100_000))) + .isInstanceOf(RuntimeException.class) + .as("Cannot write 100_000 bytes the second time as write limit of 1000 was consumed and the burst isnt enough either."); + } + + @Test + public void shouldFailIfReadingTooManyBytes() throws InterruptedException { + TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + IntStream.range(0, 30).forEach(i -> todoClient.addTodo(getTodoOfSize(1_000))); + + assertThatThrownBy(todoClient::getTodoList) + .isInstanceOf(RuntimeException.class) + .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); + } + + @After + public void after() { + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); + } + + private Todo getTodoOfSize(int size) { + return ImmutableTodo.of(String.join("", Collections.nCopies(size, "a"))); } } From da40d2333a78ac0c7b779a85196db9a3093e8f0a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 20:39:29 +0000 Subject: [PATCH 42/67] canBeWritingLargeNumberOfBytesConcurrently --- .../atlasdb/ete/QosCassandraTestSuite.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 9114fefd68f..4ec35210430 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -15,18 +15,30 @@ */ package com.palantir.atlasdb.ete; +import static org.assertj.core.api.Java6Assertions.assertThat; import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; +import static org.junit.Assert.assertNull; import java.util.Collections; import java.util.List; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; +import org.assertj.core.util.Lists; import org.junit.After; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.RuleChain; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.containers.CassandraEnvironment; @@ -64,6 +76,44 @@ public void shouldFailIfReadingTooManyBytes() throws InterruptedException { .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); } + @Test + public void canBeWritingLargeNumberOfBytesConcurrently() throws InterruptedException { + TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + + CyclicBarrier barrier = new CyclicBarrier(100); + ForkJoinPool threadPool = new ForkJoinPool(100); + + List> futures = Lists.newArrayList(); + + IntStream.range(0, 100).parallel().forEach(i -> + futures.add(threadPool.submit(() -> + (Callable) () -> { + try { + barrier.await(); + todoClient.addTodo(getTodoOfSize(1_000)); + } catch (BrokenBarrierException | InterruptedException e) { + // Do nothing + } + return null; + }))); + + threadPool.shutdown(); + Preconditions.checkState(threadPool.awaitTermination(90, TimeUnit.SECONDS), + "Not all threads writing data finished in the expected time."); + + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException e) { + exceptionCounter.getAndIncrement(); + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isEqualTo(0); + } + @After public void after() { Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); From 27073f2d5a9c6549f08ef341f22ba6c156bad009 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 20:43:50 +0000 Subject: [PATCH 43/67] checkstyle --- .../com/palantir/atlasdb/ete/QosCassandraTestSuite.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 4ec35210430..5524a418827 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -15,9 +15,8 @@ */ package com.palantir.atlasdb.ete; -import static org.assertj.core.api.Java6Assertions.assertThat; -import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; -import static org.junit.Assert.assertNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Collections; import java.util.List; @@ -63,7 +62,8 @@ public void shouldFailIfWritingTooManyBytes() { todoClient.addTodo(getTodoOfSize(100_000)); assertThatThrownBy(() -> todoClient.addTodo(getTodoOfSize(100_000))) .isInstanceOf(RuntimeException.class) - .as("Cannot write 100_000 bytes the second time as write limit of 1000 was consumed and the burst isnt enough either."); + .as("Cannot write 100_000 bytes the second time as write limit of " + + "1000 was consumed and the burst isnt enough either."); } @Test From 05531a16062a269a4d7a768ce8097916736236f6 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 17 Nov 2017 20:54:46 +0000 Subject: [PATCH 44/67] cannotWriteLargeNumberOfBytesConcurrently --- .../atlasdb/ete/QosCassandraTestSuite.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java index 5524a418827..6bf8ae9f192 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java @@ -77,7 +77,8 @@ public void shouldFailIfReadingTooManyBytes() throws InterruptedException { } @Test - public void canBeWritingLargeNumberOfBytesConcurrently() throws InterruptedException { + public void canWriteLargeNumberOfBytesConcurrentlyIfAllRequestsComeAtTheExactSameTime() + throws InterruptedException { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); CyclicBarrier barrier = new CyclicBarrier(100); @@ -114,6 +115,34 @@ public void canBeWritingLargeNumberOfBytesConcurrently() throws InterruptedExcep assertThat(exceptionCounter.get()).isEqualTo(0); } + @Test + public void cannotWriteLargeNumberOfBytesConcurrently() throws InterruptedException { + TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + + ForkJoinPool threadPool = new ForkJoinPool(100); + + List> futures = Lists.newArrayList(); + + IntStream.range(0, 100).parallel() + .forEach(i -> futures.add(threadPool.submit(() -> todoClient.addTodo(getTodoOfSize(10_000))))); + + threadPool.shutdown(); + Preconditions.checkState(threadPool.awaitTermination(90, TimeUnit.SECONDS), + "Not all threads writing data finished in the expected time."); + + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException e) { + exceptionCounter.getAndIncrement(); + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isGreaterThan(90); + } + @After public void after() { Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); From e370121c5c16b7592d1ff8e87dd7d58b6ac64bbe Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Mon, 20 Nov 2017 18:52:29 +0000 Subject: [PATCH 45/67] fix tests --- .../docker/conf/atlasdb-ete.qos.cassandra.yml | 2 + .../ete/QosCassandraReadTestSuite.java | 70 +++++++++++++++++++ ...e.java => QosCassandraWriteTestSuite.java} | 67 ++++++++---------- 3 files changed, 102 insertions(+), 37 deletions(-) create mode 100644 atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java rename atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/{QosCassandraTestSuite.java => QosCassandraWriteTestSuite.java} (68%) diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml index a880e325a28..abc2a7d2517 100644 --- a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml +++ b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml @@ -27,6 +27,8 @@ atlasdb: autoRefreshNodes: false atlasdbRuntime: + sweep: + enabled: false qos: limits: readBytesPerSecond: 1000 diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java new file mode 100644 index 00000000000..92e1b5e4d01 --- /dev/null +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.palantir.atlasdb.ete; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; + +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Uninterruptibles; +import com.palantir.atlasdb.containers.CassandraEnvironment; +import com.palantir.atlasdb.todo.ImmutableTodo; +import com.palantir.atlasdb.todo.Todo; +import com.palantir.atlasdb.todo.TodoResource; + +public class QosCassandraReadTestSuite extends EteSetup { + private static final List CLIENTS = ImmutableList.of("ete1"); + private TodoResource todoClient; + + @ClassRule + public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( + QosCassandraReadTestSuite.class, + "docker-compose.qos.cassandra.yml", + CLIENTS, + CassandraEnvironment.get()); + + @Before + public void before() { + todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + IntStream.range(0, 30).forEach(i -> todoClient.addTodo(getTodoOfSize(1_000))); + } + + @Test + public void shouldFailIfReadingTooManyBytes() throws InterruptedException { + assertThatThrownBy(todoClient::getTodoList) + .isInstanceOf(RuntimeException.class) + .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); + } + + @After + public void after() { + Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); + } + + private Todo getTodoOfSize(int size) { + return ImmutableTodo.of(String.join("", Collections.nCopies(size, "a"))); + } +} diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java similarity index 68% rename from atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java rename to atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java index 6bf8ae9f192..fbac7812442 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java @@ -20,8 +20,6 @@ import java.util.Collections; import java.util.List; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.Callable; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; @@ -41,16 +39,17 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.containers.CassandraEnvironment; +import com.palantir.atlasdb.http.errors.AtlasDbRemoteException; import com.palantir.atlasdb.todo.ImmutableTodo; import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoResource; -public class QosCassandraTestSuite extends EteSetup { +public class QosCassandraWriteTestSuite extends EteSetup { private static final List CLIENTS = ImmutableList.of("ete1"); @ClassRule public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( - QosCassandraTestSuite.class, + QosCassandraWriteTestSuite.class, "docker-compose.qos.cassandra.yml", CLIENTS, CassandraEnvironment.get()); @@ -58,73 +57,65 @@ public class QosCassandraTestSuite extends EteSetup { @Test public void shouldFailIfWritingTooManyBytes() { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - // Doesn't throw the first time as we estimate low values - todoClient.addTodo(getTodoOfSize(100_000)); + + ensureOneWriteHasOccurred(todoClient); + assertThatThrownBy(() -> todoClient.addTodo(getTodoOfSize(100_000))) .isInstanceOf(RuntimeException.class) .as("Cannot write 100_000 bytes the second time as write limit of " + "1000 was consumed and the burst isnt enough either."); } - @Test - public void shouldFailIfReadingTooManyBytes() throws InterruptedException { - TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - IntStream.range(0, 30).forEach(i -> todoClient.addTodo(getTodoOfSize(1_000))); - - assertThatThrownBy(todoClient::getTodoList) - .isInstanceOf(RuntimeException.class) - .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); + private void ensureOneWriteHasOccurred(TodoResource todoClient) { + try { + todoClient.addTodo(getTodoOfSize(100_000)); + // okay as the first huge write is not rate limited. + } catch (Exception e) { + // okay as some other test might have written before + } } @Test - public void canWriteLargeNumberOfBytesConcurrentlyIfAllRequestsComeAtTheExactSameTime() + public void canNotWriteLargeNumberOfBytesConcurrentlyIfAllRequestsComeAtTheExactSameTime() throws InterruptedException { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); CyclicBarrier barrier = new CyclicBarrier(100); ForkJoinPool threadPool = new ForkJoinPool(100); - List> futures = Lists.newArrayList(); IntStream.range(0, 100).parallel().forEach(i -> - futures.add(threadPool.submit(() -> - (Callable) () -> { - try { - barrier.await(); - todoClient.addTodo(getTodoOfSize(1_000)); - } catch (BrokenBarrierException | InterruptedException e) { - // Do nothing - } + futures.add(threadPool.submit( + () -> { + barrier.await(); + todoClient.addTodo(getTodoOfSize(1_000)); return null; }))); - threadPool.shutdown(); - Preconditions.checkState(threadPool.awaitTermination(90, TimeUnit.SECONDS), - "Not all threads writing data finished in the expected time."); - - AtomicInteger exceptionCounter = new AtomicInteger(0); + AtomicInteger exceptionCounter = new AtomicInteger(90); futures.forEach(future -> { try { future.get(); } catch (ExecutionException e) { - exceptionCounter.getAndIncrement(); + if (e.getCause().getClass().equals(AtlasDbRemoteException.class)) { + exceptionCounter.getAndIncrement(); + } } catch (InterruptedException e) { throw Throwables.propagate(e); } }); - assertThat(exceptionCounter.get()).isEqualTo(0); + assertThat(exceptionCounter.get()).isGreaterThan(90); } @Test - public void cannotWriteLargeNumberOfBytesConcurrently() throws InterruptedException { + public void canNotWriteLargeNumberOfBytesConcurrently() throws InterruptedException { TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); ForkJoinPool threadPool = new ForkJoinPool(100); - List> futures = Lists.newArrayList(); IntStream.range(0, 100).parallel() - .forEach(i -> futures.add(threadPool.submit(() -> todoClient.addTodo(getTodoOfSize(10_000))))); + .forEach(i -> futures.add(threadPool.submit(() -> todoClient.addTodo(getTodoOfSize(1_000))))); threadPool.shutdown(); Preconditions.checkState(threadPool.awaitTermination(90, TimeUnit.SECONDS), @@ -135,17 +126,19 @@ public void cannotWriteLargeNumberOfBytesConcurrently() throws InterruptedExcept try { future.get(); } catch (ExecutionException e) { - exceptionCounter.getAndIncrement(); + if (e.getCause().getClass().equals(AtlasDbRemoteException.class)) { + exceptionCounter.getAndIncrement(); + } } catch (InterruptedException e) { throw Throwables.propagate(e); } }); - assertThat(exceptionCounter.get()).isGreaterThan(90); + assertThat(exceptionCounter.get()).isGreaterThan(0); } @After public void after() { - Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); + Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); } private Todo getTodoOfSize(int size) { From 8eef5d5f2e7a374c842ad781f5f173753d143fd4 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 14:26:22 +0000 Subject: [PATCH 46/67] create tm in test --- .../ete/QosCassandraReadTestSuite.java | 158 ++++++++++++++---- .../resources/cassandra-docker-compose.yml | 14 ++ 2 files changed, 143 insertions(+), 29 deletions(-) create mode 100644 atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 92e1b5e4d01..214ef110900 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -15,56 +15,156 @@ */ package com.palantir.atlasdb.ete; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThat; +import java.net.InetSocketAddress; import java.util.Collections; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.IntStream; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.stream.Collectors; -import org.junit.After; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.RuleChain; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Uninterruptibles; -import com.palantir.atlasdb.containers.CassandraEnvironment; +import com.google.common.collect.ImmutableMap; +import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; +import com.palantir.atlasdb.cassandra.ImmutableCassandraCredentialsConfig; +import com.palantir.atlasdb.cassandra.ImmutableCassandraKeyValueServiceConfig; +import com.palantir.atlasdb.config.AtlasDbConfig; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; +import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; +import com.palantir.atlasdb.config.ImmutableAtlasDbRuntimeConfig; +import com.palantir.atlasdb.config.ImmutableSweepConfig; +import com.palantir.atlasdb.factory.TransactionManagers; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.RangeRequest; +import com.palantir.atlasdb.keyvalue.api.RowResult; +import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; +import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; +import com.palantir.atlasdb.table.description.ValueType; import com.palantir.atlasdb.todo.ImmutableTodo; -import com.palantir.atlasdb.todo.Todo; -import com.palantir.atlasdb.todo.TodoResource; +import com.palantir.atlasdb.todo.TodoSchema; +import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; +import com.palantir.common.base.BatchingVisitable; +import com.palantir.docker.compose.DockerComposeRule; +import com.palantir.docker.compose.configuration.ShutdownStrategy; +import com.palantir.docker.compose.connection.Container; +import com.palantir.docker.compose.connection.DockerPort; +import com.palantir.docker.compose.logging.LogDirectory; public class QosCassandraReadTestSuite extends EteSetup { - private static final List CLIENTS = ImmutableList.of("ete1"); - private TodoResource todoClient; + private static final Random random = new Random(); + private static final int CASSANDRA_PORT_NUMBER = 9160; + private static SerializableTransactionManager serializableTransactionManager; @ClassRule - public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( - QosCassandraReadTestSuite.class, - "docker-compose.qos.cassandra.yml", - CLIENTS, - CassandraEnvironment.get()); + public static DockerComposeRule docker = DockerComposeRule.builder() + .file("src/test/resources/cassandra-docker-compose.yml") + .waitingForService("cassandra", Container::areAllPortsOpen) + .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) + .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) + .build(); @Before - public void before() { - todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - IntStream.range(0, 30).forEach(i -> todoClient.addTodo(getTodoOfSize(1_000))); + public void waitUntilTransactionManagersIsUp() { + + serializableTransactionManager = TransactionManagers.builder() + .config(getAtlasDbConfig()) + .runtimeConfigSupplier(() -> getAtlasDbRuntimeConfig()) + .schemas(ImmutableList.of(TodoSchema.getSchema())) + .userAgent("qos-test") + .buildSerializable(); + + Awaitility.await() + .atMost(Duration.ONE_MINUTE) + .pollInterval(Duration.ONE_SECOND) + .until(serializableTransactionManager::isInitialized); + + serializableTransactionManager.runTaskWithRetry((transaction) -> { + Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), + TodoSchema.todoTextColumn()); + Map write = ImmutableMap.of(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(100))); + transaction.put(TodoSchema.todoTable(), write); + return null; + }); } - @Test - public void shouldFailIfReadingTooManyBytes() throws InterruptedException { - assertThatThrownBy(todoClient::getTodoList) - .isInstanceOf(RuntimeException.class) - .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); + private AtlasDbConfig getAtlasDbConfig() { + DockerPort cassandraPort = docker.containers() + .container("cassandra") + .port(CASSANDRA_PORT_NUMBER); + + InetSocketAddress cassandraAddress = new InetSocketAddress(cassandraPort.getIp(), + cassandraPort.getExternalPort()); + + CassandraKeyValueServiceConfig kvsConfig = ImmutableCassandraKeyValueServiceConfig.builder() + .servers(ImmutableList.of(cassandraAddress)) + .credentials(ImmutableCassandraCredentialsConfig.builder() + .username("cassandra") + .password("cassandra") + .build()) + .ssl(false) + .replicationFactor(1) + .autoRefreshNodes(false) + .build(); + + return ImmutableAtlasDbConfig.builder() + .namespace("qosete") + .keyValueService(kvsConfig) + .initializeAsync(true) + .build(); } - @After - public void after() { - Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); + private Optional getAtlasDbRuntimeConfig() { + return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() + .sweep(ImmutableSweepConfig.builder().enabled(false).build()) + .qos(ImmutableQosClientConfig.builder() + .limits(ImmutableQosLimitsConfig.builder() + .readBytesPerSecond(10_000) + .writeBytesPerSecond(10_000) + .build()) + .build()) + .build()); + } + +// @Test +// public void shouldFailIfReadingTooManyBytes() throws InterruptedException { +// assertThatThrownBy(todoClient::getTodoList) +// .isInstanceOf(RuntimeException.class) +// .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); +// } + + @Test + public void shouldBeAbleToReadSmallAmounts() { + ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { + BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( + TodoSchema.todoTable(), RangeRequest.all()); + ImmutableList.Builder> rowResults = ImmutableList.builder(); + + rowResultBatchingVisitable.batchAccept(10, items -> { + rowResults.addAll(items); + return true; + }); + + return rowResults.build(); + }); + + List collect = results.stream() + .map(RowResult::getOnlyColumnValue) + .map(ValueType.STRING::convertToString) + .map(ImmutableTodo::of) + .collect(Collectors.toList()); + + assertThat(collect).hasSize(1); } - private Todo getTodoOfSize(int size) { - return ImmutableTodo.of(String.join("", Collections.nCopies(size, "a"))); + private static String getTodoOfSize(int size) { + return String.join("", Collections.nCopies(size, "a")); } } diff --git a/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml b/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml new file mode 100644 index 00000000000..60e6f7e9079 --- /dev/null +++ b/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' + +services: + cassandra: + image: palantirtechnologies/docker-cassandra-atlasdb:2.2.8 + ports: + - "9160" + - "9042" + - "7199" + environment: + - MAX_HEAP_SIZE=512m + - HEAP_NEWSIZE=64m + - LOCAL_JMX=no + - CASSANDRA_VERSION=2.2.8 \ No newline at end of file From 53f732e25465abb26bdf24e1387ce91d42183757 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 16:20:47 +0000 Subject: [PATCH 47/67] More read tests (after writing a lot of data at once) --- .../ete/QosCassandraReadTestSuite.java | 124 ++++++++++++------ 1 file changed, 83 insertions(+), 41 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 214ef110900..be9359d5783 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -24,10 +24,12 @@ import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.awaitility.Awaitility; import org.awaitility.Duration; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -70,13 +72,14 @@ public class QosCassandraReadTestSuite extends EteSetup { .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); + private static int readBytesPerSecond = 10_000; + private static int writeBytesPerSecond = 10_000; - @Before - public void waitUntilTransactionManagersIsUp() { - + @BeforeClass + public static void waitUntilTransactionManagersIsUp() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) - .runtimeConfigSupplier(() -> getAtlasDbRuntimeConfig()) + .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) .schemas(ImmutableList.of(TodoSchema.getSchema())) .userAgent("qos-test") .buildSerializable(); @@ -86,16 +89,86 @@ public void waitUntilTransactionManagersIsUp() { .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - serializableTransactionManager.runTaskWithRetry((transaction) -> { + IntStream.range(0, 100).forEach(i -> serializableTransactionManager + .runTaskWithRetry((transaction) -> { Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), TodoSchema.todoTextColumn()); - Map write = ImmutableMap.of(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(100))); + Map write = ImmutableMap.of(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(1_000))); transaction.put(TodoSchema.todoTable(), write); return null; + })); + } + + @Before + public void setup() { + // modifying the limits, forces creation of a new ratelimiter. + readBytesPerSecond = readBytesPerSecond + 10; + writeBytesPerSecond = writeBytesPerSecond + 10; + } + + + @Test + public void shouldBeAbleToReadSmallAmounts() { + ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { + BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( + TodoSchema.todoTable(), RangeRequest.all()); + ImmutableList.Builder> rowResults = ImmutableList.builder(); + + rowResultBatchingVisitable.batchAccept(10, items -> { + rowResults.addAll(items); + return false; + }); + + return rowResults.build(); + }); + + List collect = results.stream() + .map(RowResult::getOnlyColumnValue) + .map(ValueType.STRING::convertToString) + .map(ImmutableTodo::of) + .collect(Collectors.toList()); + + assertThat(collect).hasSize(10); + } + + @Test + public void shouldBeAbleToReadLargeAmountsTheFirstTime() { + assertThatAllTodosAreRead(); + } + + @Test + public void shouldNotBeAbleToReadLargeAmountsConsecutively() { + assertThatAllTodosAreRead(); + assertThatAllTodosAreRead(); + assertThatAllTodosAreRead(); + assertThatAllTodosAreRead(); + assertThatAllTodosAreRead(); + } + + private void assertThatAllTodosAreRead() { + ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { + BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( + TodoSchema.todoTable(), RangeRequest.all()); + ImmutableList.Builder> rowResults = ImmutableList.builder(); + + rowResultBatchingVisitable.batchAccept(10, items -> { + rowResults.addAll(items); + return true; + }); + + return rowResults.build(); }); + + List collect = results.stream() + .map(RowResult::getOnlyColumnValue) + .map(ValueType.STRING::convertToString) + .map(ImmutableTodo::of) + .collect(Collectors.toList()); + + assertThat(collect).hasSize(100); } - private AtlasDbConfig getAtlasDbConfig() { + public static AtlasDbConfig getAtlasDbConfig() { DockerPort cassandraPort = docker.containers() .container("cassandra") .port(CASSANDRA_PORT_NUMBER); @@ -121,49 +194,18 @@ private AtlasDbConfig getAtlasDbConfig() { .build(); } - private Optional getAtlasDbRuntimeConfig() { + public static Optional getAtlasDbRuntimeConfig() { return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() .sweep(ImmutableSweepConfig.builder().enabled(false).build()) .qos(ImmutableQosClientConfig.builder() .limits(ImmutableQosLimitsConfig.builder() - .readBytesPerSecond(10_000) - .writeBytesPerSecond(10_000) + .readBytesPerSecond(readBytesPerSecond) + .writeBytesPerSecond(writeBytesPerSecond) .build()) .build()) .build()); } -// @Test -// public void shouldFailIfReadingTooManyBytes() throws InterruptedException { -// assertThatThrownBy(todoClient::getTodoList) -// .isInstanceOf(RuntimeException.class) -// .as("Cant read 30_000 bytes in 10 batches i.e. 3000 bytes multiple times when limit is 1000."); -// } - - @Test - public void shouldBeAbleToReadSmallAmounts() { - ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { - BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( - TodoSchema.todoTable(), RangeRequest.all()); - ImmutableList.Builder> rowResults = ImmutableList.builder(); - - rowResultBatchingVisitable.batchAccept(10, items -> { - rowResults.addAll(items); - return true; - }); - - return rowResults.build(); - }); - - List collect = results.stream() - .map(RowResult::getOnlyColumnValue) - .map(ValueType.STRING::convertToString) - .map(ImmutableTodo::of) - .collect(Collectors.toList()); - - assertThat(collect).hasSize(1); - } - private static String getTodoOfSize(int size) { return String.join("", Collections.nCopies(size, "a")); } From e2642cf7975dd7547da60de368f2ce7bbd313d62 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 17:38:54 +0000 Subject: [PATCH 48/67] WIP --- .../ete/QosCassandraReadTestSuite.java | 109 +++++++++--------- .../atlasdb/qos/client/AtlasDbQosClient.java | 2 + 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index be9359d5783..472e70700a9 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.ete; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; import java.util.Collections; @@ -23,16 +24,18 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.awaitility.Awaitility; import org.awaitility.Duration; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; @@ -54,11 +57,13 @@ import com.palantir.atlasdb.todo.TodoSchema; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; import com.palantir.common.base.BatchingVisitable; +import com.palantir.common.exception.AtlasDbDependencyException; import com.palantir.docker.compose.DockerComposeRule; import com.palantir.docker.compose.configuration.ShutdownStrategy; import com.palantir.docker.compose.connection.Container; import com.palantir.docker.compose.connection.DockerPort; import com.palantir.docker.compose.logging.LogDirectory; +import com.palantir.remoting.api.config.service.HumanReadableDuration; public class QosCassandraReadTestSuite extends EteSetup { private static final Random random = new Random(); @@ -72,8 +77,20 @@ public class QosCassandraReadTestSuite extends EteSetup { .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); - private static int readBytesPerSecond = 10_000; - private static int writeBytesPerSecond = 10_000; + private static Supplier readBytesPerSecond = new Supplier() { + int initial = 10_000; + @Override + public Integer get() { + return initial + 1; + } + }; + private static Supplier writeBytesPerSecond = new Supplier() { + int initial = 10_000; + @Override + public Integer get() { + return initial + 1; + } + }; @BeforeClass public static void waitUntilTransactionManagersIsUp() { @@ -91,81 +108,65 @@ public static void waitUntilTransactionManagersIsUp() { IntStream.range(0, 100).forEach(i -> serializableTransactionManager .runTaskWithRetry((transaction) -> { - Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), - TodoSchema.todoTextColumn()); - Map write = ImmutableMap.of(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(1_000))); - transaction.put(TodoSchema.todoTable(), write); - return null; - })); + Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), + TodoSchema.todoTextColumn()); + Map write = ImmutableMap.of(thisCell, + ValueType.STRING.convertFromJava(getTodoOfSize(1_000))); + transaction.put(TodoSchema.todoTable(), write); + return null; + })); } - @Before - public void setup() { - // modifying the limits, forces creation of a new ratelimiter. - readBytesPerSecond = readBytesPerSecond + 10; - writeBytesPerSecond = writeBytesPerSecond + 10; - } - - @Test - public void shouldBeAbleToReadSmallAmounts() { - ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { - BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( - TodoSchema.todoTable(), RangeRequest.all()); - ImmutableList.Builder> rowResults = ImmutableList.builder(); - - rowResultBatchingVisitable.batchAccept(10, items -> { - rowResults.addAll(items); - return false; - }); - - return rowResults.build(); - }); - - List collect = results.stream() - .map(RowResult::getOnlyColumnValue) - .map(ValueType.STRING::convertToString) - .map(ImmutableTodo::of) - .collect(Collectors.toList()); - - assertThat(collect).hasSize(10); + public void shouldBeAbleToReadSmallAmountOfBytesThatDoesntExceedLimit() { + assertThat(readOneBatchOfSize(1)).hasSize(1); } @Test - public void shouldBeAbleToReadLargeAmountsTheFirstTime() { - assertThatAllTodosAreRead(); + public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyThatDoesntExceedLimit() { + assertThat(readOneBatchOfSize(1)).hasSize(1); } + @Test - public void shouldNotBeAbleToReadLargeAmountsConsecutively() { - assertThatAllTodosAreRead(); - assertThatAllTodosAreRead(); - assertThatAllTodosAreRead(); - assertThatAllTodosAreRead(); - assertThatAllTodosAreRead(); + public void shouldNotBeAbleToReadLargeAmountsTheFirstTime() { + assertThatThrownBy(() -> readOneBatchOfSize(100)) + .isInstanceOf(AtlasDbDependencyException.class) + .hasCause(new RuntimeException("rate limited")); } - private void assertThatAllTodosAreRead() { + private List readOneBatchOfSize(int batchSize) { ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( TodoSchema.todoTable(), RangeRequest.all()); ImmutableList.Builder> rowResults = ImmutableList.builder(); - rowResultBatchingVisitable.batchAccept(10, items -> { + rowResultBatchingVisitable.batchAccept(batchSize, items -> { rowResults.addAll(items); - return true; + return false; }); return rowResults.build(); }); - List collect = results.stream() + return results.stream() .map(RowResult::getOnlyColumnValue) .map(ValueType.STRING::convertToString) .map(ImmutableTodo::of) .collect(Collectors.toList()); + } + + @Test + public void shouldBeAbleToReadSmallAmountsConsecutively() { + Stopwatch stopwatch = Stopwatch.createStarted(); + readOneBatchOfSize(1); + long firstLargeReadLatency = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + stopwatch = Stopwatch.createStarted(); + readOneBatchOfSize(1); + long secondReadLatency = stopwatch.elapsed(TimeUnit.MILLISECONDS); - assertThat(collect).hasSize(100); + assertThat(secondReadLatency).isGreaterThan(firstLargeReadLatency); } public static AtlasDbConfig getAtlasDbConfig() { @@ -195,13 +196,15 @@ public static AtlasDbConfig getAtlasDbConfig() { } public static Optional getAtlasDbRuntimeConfig() { + return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() .sweep(ImmutableSweepConfig.builder().enabled(false).build()) .qos(ImmutableQosClientConfig.builder() .limits(ImmutableQosLimitsConfig.builder() - .readBytesPerSecond(readBytesPerSecond) - .writeBytesPerSecond(writeBytesPerSecond) + .readBytesPerSecond(readBytesPerSecond.get()) + .writeBytesPerSecond(writeBytesPerSecond.get()) .build()) + .maxBackoffSleepTime(HumanReadableDuration.seconds(10)) .build()) .build()); } diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index f1832ba2199..67b8ee2acaa 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -50,6 +50,7 @@ public static AtlasDbQosClient create(QosRateLimiters rateLimiters) { public T executeRead(ReadQuery query, QueryWeigher weigher) throws E { long estimatedNumBytes = weigher.estimate().numBytes(); rateLimiters.read().consumeWithBackoff(estimatedNumBytes); + System.out.println(""); // TODO(nziebart): decide what to do if we encounter a timeout exception long startTimeNanos = ticker.read(); @@ -60,6 +61,7 @@ public T executeRead(ReadQuery query, QueryWeighe metrics.recordRead(actualWeight); rateLimiters.read().recordAdjustment(actualWeight.numBytes() - estimatedNumBytes); + return result; } From d18c007e1932733b86b5479b9f6cd8efd5ce8df2 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 18:10:05 +0000 Subject: [PATCH 49/67] Tests that should pas --- .../ete/QosCassandraReadTestSuite.java | 67 ++++++++++++------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 472e70700a9..c0a63eaad4a 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -16,15 +16,20 @@ package com.palantir.atlasdb.ete; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -34,8 +39,8 @@ import org.junit.ClassRule; import org.junit.Test; -import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; @@ -52,12 +57,13 @@ import com.palantir.atlasdb.keyvalue.api.RowResult; import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; +import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; import com.palantir.atlasdb.table.description.ValueType; import com.palantir.atlasdb.todo.ImmutableTodo; +import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoSchema; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; import com.palantir.common.base.BatchingVisitable; -import com.palantir.common.exception.AtlasDbDependencyException; import com.palantir.docker.compose.DockerComposeRule; import com.palantir.docker.compose.configuration.ShutdownStrategy; import com.palantir.docker.compose.connection.Container; @@ -77,6 +83,8 @@ public class QosCassandraReadTestSuite extends EteSetup { .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); + + // Changing the read and write limits will force the rate limiter to be recreated for each test. private static Supplier readBytesPerSecond = new Supplier() { int initial = 10_000; @Override @@ -118,24 +126,48 @@ public static void waitUntilTransactionManagersIsUp() { } @Test - public void shouldBeAbleToReadSmallAmountOfBytesThatDoesntExceedLimit() { + public void shouldBeAbleToReadSmallAmountOfBytesIfDoesNotExceedLimit() { assertThat(readOneBatchOfSize(1)).hasSize(1); } @Test - public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyThatDoesntExceedLimit() { - assertThat(readOneBatchOfSize(1)).hasSize(1); + public void shouldBeAbleToReadSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { + IntStream.range(0, 50) + .forEach(i -> assertThat(readOneBatchOfSize(1)).hasSize(1)); } + @Test + public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyIfDoesNotExceedLimit() throws InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(50); + List>> futures = new ArrayList<>(); + + IntStream.range(0, 50) + .forEach(i -> futures.add(executorService.submit(() -> readOneBatchOfSize(1)))); + + executorService.shutdown(); + executorService.awaitTermination(10L, TimeUnit.SECONDS); + + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + assertThat(future.get()).hasSize(1); + } catch (ExecutionException e) { + if (e.getCause() instanceof RateLimitExceededException) { + exceptionCounter.getAndIncrement(); + } + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isEqualTo(0); + } @Test - public void shouldNotBeAbleToReadLargeAmountsTheFirstTime() { - assertThatThrownBy(() -> readOneBatchOfSize(100)) - .isInstanceOf(AtlasDbDependencyException.class) - .hasCause(new RuntimeException("rate limited")); + public void shouldBeAbleToReadLargeAmountsTheFirstTime() { + readOneBatchOfSize(100); } - private List readOneBatchOfSize(int batchSize) { + private List readOneBatchOfSize(int batchSize) { ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( TodoSchema.todoTable(), RangeRequest.all()); @@ -156,19 +188,6 @@ private List readOneBatchOfSize(int batchSize) { .collect(Collectors.toList()); } - @Test - public void shouldBeAbleToReadSmallAmountsConsecutively() { - Stopwatch stopwatch = Stopwatch.createStarted(); - readOneBatchOfSize(1); - long firstLargeReadLatency = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - stopwatch = Stopwatch.createStarted(); - readOneBatchOfSize(1); - long secondReadLatency = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - assertThat(secondReadLatency).isGreaterThan(firstLargeReadLatency); - } - public static AtlasDbConfig getAtlasDbConfig() { DockerPort cassandraPort = docker.containers() .container("cassandra") From fa0b7316bac5d9c65a6735709f21cb0455259722 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 18:23:04 +0000 Subject: [PATCH 50/67] Actually update the rate --- .../com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index c0a63eaad4a..8675d850101 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -89,14 +89,14 @@ public class QosCassandraReadTestSuite extends EteSetup { int initial = 10_000; @Override public Integer get() { - return initial + 1; + return initial++; } }; private static Supplier writeBytesPerSecond = new Supplier() { int initial = 10_000; @Override public Integer get() { - return initial + 1; + return initial++; } }; @@ -215,7 +215,6 @@ public static AtlasDbConfig getAtlasDbConfig() { } public static Optional getAtlasDbRuntimeConfig() { - return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() .sweep(ImmutableSweepConfig.builder().enabled(false).build()) .qos(ImmutableQosClientConfig.builder() From 6e4745ec020f080cf28a11aaf3c88458152448b5 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 21 Nov 2017 18:43:01 +0000 Subject: [PATCH 51/67] Add another test --- .../ete/QosCassandraReadTestSuite.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 8675d850101..021b3d56fdf 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -39,6 +39,7 @@ import org.junit.ClassRule; import org.junit.Test; +import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -86,14 +87,15 @@ public class QosCassandraReadTestSuite extends EteSetup { // Changing the read and write limits will force the rate limiter to be recreated for each test. private static Supplier readBytesPerSecond = new Supplier() { - int initial = 10_000; + int initial = 5_000; @Override public Integer get() { return initial++; } }; + private static Supplier writeBytesPerSecond = new Supplier() { - int initial = 10_000; + int initial = 5_000; @Override public Integer get() { return initial++; @@ -114,7 +116,7 @@ public static void waitUntilTransactionManagersIsUp() { .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - IntStream.range(0, 100).forEach(i -> serializableTransactionManager + IntStream.range(0, 200).forEach(i -> serializableTransactionManager .runTaskWithRetry((transaction) -> { Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), TodoSchema.todoTextColumn()); @@ -163,10 +165,23 @@ public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyIfDoesNotExceedLimit } @Test - public void shouldBeAbleToReadLargeAmountsTheFirstTime() { - readOneBatchOfSize(100); + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { + assertThat(readOneBatchOfSize(200)).hasSize(200); } + @Test + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTime() { + Stopwatch stopwatch = Stopwatch.createStarted(); + assertThat(readOneBatchOfSize(200)).hasSize(200); + long firstReadTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + stopwatch = Stopwatch.createStarted(); + assertThat(readOneBatchOfSize(200)).hasSize(200); + long secondReadTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + } + + private List readOneBatchOfSize(int batchSize) { ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( From a01eef796ca3298c2b906cac70ea7d002f522c36 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 15:54:40 +0000 Subject: [PATCH 52/67] More tests and address comments --- .../ete/QosCassandraReadTestSuite.java | 132 ++++++++++++------ 1 file changed, 86 insertions(+), 46 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 021b3d56fdf..4e98b09aaa7 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.ete; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -35,12 +36,15 @@ import org.awaitility.Awaitility; import org.awaitility.Duration; +import org.junit.After; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -64,6 +68,7 @@ import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoSchema; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; +import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.common.base.BatchingVisitable; import com.palantir.docker.compose.DockerComposeRule; import com.palantir.docker.compose.configuration.ShutdownStrategy; @@ -76,6 +81,9 @@ public class QosCassandraReadTestSuite extends EteSetup { private static final Random random = new Random(); private static final int CASSANDRA_PORT_NUMBER = 9160; private static SerializableTransactionManager serializableTransactionManager; + private static final int readBytesPerSecond = 10_000; + private static final int writeBytesPerSecond = 10_000; + private static final int ONE_TODO_SIZE_IN_BYTES = 1050; @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() @@ -85,23 +93,6 @@ public class QosCassandraReadTestSuite extends EteSetup { .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); - // Changing the read and write limits will force the rate limiter to be recreated for each test. - private static Supplier readBytesPerSecond = new Supplier() { - int initial = 5_000; - @Override - public Integer get() { - return initial++; - } - }; - - private static Supplier writeBytesPerSecond = new Supplier() { - int initial = 5_000; - @Override - public Integer get() { - return initial++; - } - }; - @BeforeClass public static void waitUntilTransactionManagersIsUp() { serializableTransactionManager = TransactionManagers.builder() @@ -125,6 +116,24 @@ public static void waitUntilTransactionManagersIsUp() { transaction.put(TodoSchema.todoTable(), write); return null; })); + serializableTransactionManager.close(); + } + + @Before + public void setup() { + serializableTransactionManager = TransactionManagers.builder() + .config(getAtlasDbConfig()) + .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) + .schemas(ImmutableList.of(TodoSchema.getSchema())) + .userAgent("qos-test") + .buildSerializable(); + + Awaitility.await() + .atMost(Duration.ONE_MINUTE) + .pollInterval(Duration.ONE_SECOND) + .until(serializableTransactionManager::isInitialized); + + AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); } @Test @@ -139,20 +148,63 @@ public void shouldBeAbleToReadSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { } @Test - public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyIfDoesNotExceedLimit() throws InterruptedException { - ExecutorService executorService = Executors.newFixedThreadPool(50); - List>> futures = new ArrayList<>(); + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { + assertThat(readOneBatchOfSize(12)).hasSize(12); + } - IntStream.range(0, 50) - .forEach(i -> futures.add(executorService.submit(() -> readOneBatchOfSize(1)))); + @Test + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { + assertThat(readOneBatchOfSize(12)).hasSize(12); + // The second read might actually be faster as the transaction/metadata + assertThat(readOneBatchOfSize(12)).hasSize(12); + } + @Test + public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepIsMoreThanConfiguredBackoffTime() { + Stopwatch started = Stopwatch.createStarted(); + assertThat(readOneBatchOfSize(12)).hasSize(12); + long firstReadTime = started.elapsed(TimeUnit.MILLISECONDS); + System.out.println(firstReadTime); + + //This one will throw because the read to the transaction table is rate limited. + assertThatThrownBy(() -> readOneBatchOfSize(200)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); + } + + + @Test + public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws InterruptedException { + int numThreads = 5; + int numReadsPerThread = 10; + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List>> futures = new ArrayList<>(numReadsPerThread); + + long start = System.nanoTime(); + IntStream.range(0, numReadsPerThread) + .forEach(i -> + futures.add(executorService.submit(() -> { + List results = new ArrayList<>(numReadsPerThread); + IntStream.range(0, numReadsPerThread) + .forEach(j -> results.addAll(readOneBatchOfSize(1))); + return results; + })) + ); executorService.shutdown(); - executorService.awaitTermination(10L, TimeUnit.SECONDS); + Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), + "Read tasks did not finish in 30s"); + long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + assertThatAllReadsWereSuccessful(futures, numReadsPerThread); + long actualReadRate = (numThreads * numReadsPerThread * ONE_TODO_SIZE_IN_BYTES) / timeTakenToReadInSeconds; + assertThat(actualReadRate).isLessThan(readBytesPerSecond + (readBytesPerSecond / 10 /* to allow burst time */)); + } + + private void assertThatAllReadsWereSuccessful(List>> futures, int numReadsPerThread) { AtomicInteger exceptionCounter = new AtomicInteger(0); futures.forEach(future -> { try { - assertThat(future.get()).hasSize(1); + assertThat(future.get()).hasSize(numReadsPerThread); } catch (ExecutionException e) { if (e.getCause() instanceof RateLimitExceededException) { exceptionCounter.getAndIncrement(); @@ -164,24 +216,6 @@ public void shouldBeAbleToReadSmallAmountOfBytesConcurrentlyIfDoesNotExceedLimit assertThat(exceptionCounter.get()).isEqualTo(0); } - @Test - public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { - assertThat(readOneBatchOfSize(200)).hasSize(200); - } - - @Test - public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTime() { - Stopwatch stopwatch = Stopwatch.createStarted(); - assertThat(readOneBatchOfSize(200)).hasSize(200); - long firstReadTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - stopwatch = Stopwatch.createStarted(); - assertThat(readOneBatchOfSize(200)).hasSize(200); - long secondReadTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - } - - private List readOneBatchOfSize(int batchSize) { ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( @@ -234,15 +268,21 @@ public static Optional getAtlasDbRuntimeConfig() { .sweep(ImmutableSweepConfig.builder().enabled(false).build()) .qos(ImmutableQosClientConfig.builder() .limits(ImmutableQosLimitsConfig.builder() - .readBytesPerSecond(readBytesPerSecond.get()) - .writeBytesPerSecond(writeBytesPerSecond.get()) + .readBytesPerSecond(readBytesPerSecond) + .writeBytesPerSecond(writeBytesPerSecond) .build()) - .maxBackoffSleepTime(HumanReadableDuration.seconds(10)) + .maxBackoffSleepTime(HumanReadableDuration.seconds(2)) .build()) .build()); } + @After + public void cleanup() { + serializableTransactionManager.close(); + } + private static String getTodoOfSize(int size) { + // Note that the size of the cell for 1000 length text is actually 11. return String.join("", Collections.nCopies(size, "a")); } } From 2aa69da2b541a9fb49be420f8429633ac01adf8a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:16:07 +0000 Subject: [PATCH 53/67] Dont extend etesetup --- .../cassandra/qos/QosCassandraClient.java | 17 +++++++++++++++-- .../atlasdb/ete/QosCassandraReadTestSuite.java | 4 ++-- .../atlasdb/qos/client/AtlasDbQosClient.java | 3 +++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java index 42f5d5d7137..7988acff8a1 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java @@ -67,8 +67,12 @@ public Cassandra.Client rawClient() { public Map> multiget_slice(String kvsMethodName, TableReference tableRef, List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { + System.out.println("multiget_slice from the table " + tableRef.getQualifiedName() + " with slice " + predicate.toString() + " and keys " + keys.size()); + return qosClient.executeRead( - () -> client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level), + () -> { + return client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level); + }, ThriftQueryWeighers.multigetSlice(keys)); } @@ -76,8 +80,11 @@ public Map> multiget_slice(String kvsMetho public List get_range_slices(String kvsMethodName, TableReference tableRef, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { + System.out.println("get_range_slices from the table " + tableRef.getQualifiedName() + " with slice " + predicate.toString() + " and keys " + range.toString()); return qosClient.executeRead( - () -> client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level), + () -> { + return client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); + }, ThriftQueryWeighers.getRangeSlices(range)); } @@ -85,6 +92,8 @@ public List get_range_slices(String kvsMethodName, TableReference tabl public void batch_mutate(String kvsMethodName, Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { + System.out.println("batch_mutate from the tables with mutation_map " + mutation_map.size()); + qosClient.executeWrite( () -> { client.batch_mutate(kvsMethodName, mutation_map, consistency_level); @@ -97,6 +106,8 @@ public void batch_mutate(String kvsMethodName, Map client.get(tableReference, key, column, consistency_level), ThriftQueryWeighers.GET); @@ -114,6 +125,8 @@ public CASResult cas(TableReference tableReference, ByteBuffer key, List public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { + System.out.println("execute_cql3_query from the query " + cqlQuery.getLazySafeLoggableObject()); + return qosClient.executeRead( () -> client.execute_cql3_query(cqlQuery, compression, consistency), ThriftQueryWeighers.EXECUTE_CQL3_QUERY); diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 4e98b09aaa7..f660aed158c 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -77,7 +77,7 @@ import com.palantir.docker.compose.logging.LogDirectory; import com.palantir.remoting.api.config.service.HumanReadableDuration; -public class QosCassandraReadTestSuite extends EteSetup { +public class QosCassandraReadTestSuite { private static final Random random = new Random(); private static final int CASSANDRA_PORT_NUMBER = 9160; private static SerializableTransactionManager serializableTransactionManager; @@ -94,7 +94,7 @@ public class QosCassandraReadTestSuite extends EteSetup { .build(); @BeforeClass - public static void waitUntilTransactionManagersIsUp() { + public static void createTransactionManagerAndWriteData() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index 455bfa57036..9bac6752156 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -73,6 +73,7 @@ private T execute( estimatedWeightMetric.ifPresent(metric -> metric.accept(estimatedWeight)); try { + System.out.println("Consuming estimated bytes of " + estimatedWeight.numBytes()); Duration waitTime = rateLimiter.consumeWithBackoff(estimatedWeight.numBytes()); metrics.recordBackoffMicros(TimeUnit.NANOSECONDS.toMicros(waitTime.toNanos())); } catch (RateLimitExceededException ex) { @@ -93,6 +94,8 @@ private T execute( } finally { weightMetric.accept(actualWeight); rateLimiter.recordAdjustment(actualWeight.numBytes() - estimatedWeight.numBytes()); + System.out.println("Recording adjusted bytes of " + (actualWeight.numBytes() - estimatedWeight.numBytes())); + } } From 55a82f20871ab3b61228bfd95c34a0cef6c1a99f Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:26:15 +0000 Subject: [PATCH 54/67] Make dumping data faster --- .../ete/QosCassandraReadTestSuite.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index f660aed158c..7a0a1698e20 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -21,6 +21,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,10 +45,8 @@ import com.codahale.metrics.MetricRegistry; import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; import com.palantir.atlasdb.cassandra.ImmutableCassandraCredentialsConfig; import com.palantir.atlasdb.cassandra.ImmutableCassandraKeyValueServiceConfig; @@ -67,6 +66,7 @@ import com.palantir.atlasdb.todo.ImmutableTodo; import com.palantir.atlasdb.todo.Todo; import com.palantir.atlasdb.todo.TodoSchema; +import com.palantir.atlasdb.transaction.api.Transaction; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.common.base.BatchingVisitable; @@ -107,18 +107,24 @@ public static void createTransactionManagerAndWriteData() { .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - IntStream.range(0, 200).forEach(i -> serializableTransactionManager + serializableTransactionManager .runTaskWithRetry((transaction) -> { - Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), - TodoSchema.todoTextColumn()); - Map write = ImmutableMap.of(thisCell, - ValueType.STRING.convertFromJava(getTodoOfSize(1_000))); - transaction.put(TodoSchema.todoTable(), write); + writeNTodosOfSize(transaction, 200, 1_000); return null; - })); + }); serializableTransactionManager.close(); } + private static void writeNTodosOfSize(Transaction transaction, int n, int size) { + Map write = new HashMap<>(); + for (int i = 0; i < n; i++) { + Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), + TodoSchema.todoTextColumn()); + write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); + } + transaction.put(TodoSchema.todoTable(), write); + } + @Before public void setup() { serializableTransactionManager = TransactionManagers.builder() @@ -160,19 +166,12 @@ public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTimeWithSoftLim } @Test - public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepIsMoreThanConfiguredBackoffTime() { - Stopwatch started = Stopwatch.createStarted(); - assertThat(readOneBatchOfSize(12)).hasSize(12); - long firstReadTime = started.elapsed(TimeUnit.MILLISECONDS); - System.out.println(firstReadTime); - - //This one will throw because the read to the transaction table is rate limited. + public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { assertThatThrownBy(() -> readOneBatchOfSize(200)) .isInstanceOf(RateLimitExceededException.class) .hasMessage("Rate limited. Available capacity has been exhausted."); } - @Test public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws InterruptedException { int numThreads = 5; From 2f49e0ec05ab30f244f32aabab188a33f6ff64b9 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:37:12 +0000 Subject: [PATCH 55/67] cleanup --- .../com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 7a0a1698e20..4e39cb356df 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -115,9 +115,9 @@ public static void createTransactionManagerAndWriteData() { serializableTransactionManager.close(); } - private static void writeNTodosOfSize(Transaction transaction, int n, int size) { + public static void writeNTodosOfSize(Transaction transaction, int numTodos, int size) { Map write = new HashMap<>(); - for (int i = 0; i < n; i++) { + for (int i = 0; i < numTodos; i++) { Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), TodoSchema.todoTextColumn()); write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); @@ -281,7 +281,7 @@ public void cleanup() { } private static String getTodoOfSize(int size) { - // Note that the size of the cell for 1000 length text is actually 11. + // Note that the size of the cell for 1000 length text is actually 1050. return String.join("", Collections.nCopies(size, "a")); } } From be06b19b3465e77fec2b86de2d02db592bef5552 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:41:47 +0000 Subject: [PATCH 56/67] wip --- .../ete/QosCassandraReadTestSuite.java | 7 +- .../ete/QosCassandraWriteTestSuite.java | 86 ++++++++++++++----- 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 4e39cb356df..f51d351a7a5 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -107,8 +107,7 @@ public static void createTransactionManagerAndWriteData() { .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - serializableTransactionManager - .runTaskWithRetry((transaction) -> { + serializableTransactionManager.runTaskWithRetry((transaction) -> { writeNTodosOfSize(transaction, 200, 1_000); return null; }); @@ -127,6 +126,8 @@ public static void writeNTodosOfSize(Transaction transaction, int numTodos, int @Before public void setup() { + AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); + serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) @@ -138,8 +139,6 @@ public void setup() { .atMost(Duration.ONE_MINUTE) .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - - AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); } @Test diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java index fbac7812442..09f40de5172 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java @@ -18,8 +18,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static com.palantir.atlasdb.ete.QosCassandraReadTestSuite.getAtlasDbConfig; + import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; @@ -29,41 +34,79 @@ import java.util.stream.IntStream; import org.assertj.core.util.Lists; +import org.awaitility.Awaitility; +import org.awaitility.Duration; import org.junit.After; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.RuleChain; +import com.codahale.metrics.MetricRegistry; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; -import com.palantir.atlasdb.containers.CassandraEnvironment; +import com.palantir.atlasdb.factory.TransactionManagers; import com.palantir.atlasdb.http.errors.AtlasDbRemoteException; -import com.palantir.atlasdb.todo.ImmutableTodo; -import com.palantir.atlasdb.todo.Todo; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.table.description.ValueType; import com.palantir.atlasdb.todo.TodoResource; +import com.palantir.atlasdb.todo.TodoSchema; +import com.palantir.atlasdb.transaction.api.Transaction; +import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; +import com.palantir.atlasdb.util.AtlasDbMetrics; +import com.palantir.docker.compose.DockerComposeRule; +import com.palantir.docker.compose.configuration.ShutdownStrategy; +import com.palantir.docker.compose.connection.Container; +import com.palantir.docker.compose.logging.LogDirectory; + +public class QosCassandraWriteTestSuite { -public class QosCassandraWriteTestSuite extends EteSetup { - private static final List CLIENTS = ImmutableList.of("ete1"); + private static final Random random = new Random(); + private static SerializableTransactionManager serializableTransactionManager; @ClassRule - public static final RuleChain COMPOSITION_SETUP = EteSetup.setupComposition( - QosCassandraWriteTestSuite.class, - "docker-compose.qos.cassandra.yml", - CLIENTS, - CassandraEnvironment.get()); + public static DockerComposeRule docker = DockerComposeRule.builder() + .file("src/test/resources/cassandra-docker-compose.yml") + .waitingForService("cassandra", Container::areAllPortsOpen) + .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) + .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) + .build(); + + @Before + public void setup() { + AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); + + serializableTransactionManager = TransactionManagers.builder() + .config(getAtlasDbConfig()) + .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) + .schemas(ImmutableList.of(TodoSchema.getSchema())) + .userAgent("qos-test") + .buildSerializable(); + + Awaitility.await() + .atMost(Duration.ONE_MINUTE) + .pollInterval(Duration.ONE_SECOND) + .until(serializableTransactionManager::isInitialized); - @Test - public void shouldFailIfWritingTooManyBytes() { - TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); + } - ensureOneWriteHasOccurred(todoClient); + @Test + public void shouldBeAbleToWriteBytesExcee() { + serializableTransactionManager.runTaskWithRetry((transaction) -> { + writeNTodosOfSize(transaction, 200, 1_000); + return null; + }); + } - assertThatThrownBy(() -> todoClient.addTodo(getTodoOfSize(100_000))) - .isInstanceOf(RuntimeException.class) - .as("Cannot write 100_000 bytes the second time as write limit of " - + "1000 was consumed and the burst isnt enough either."); + public static void writeNTodosOfSize(Transaction transaction, int numTodos, int size) { + Map write = new HashMap<>(); + for (int i = 0; i < numTodos; i++) { + Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), + TodoSchema.todoTextColumn()); + write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); + } + transaction.put(TodoSchema.todoTable(), write); } private void ensureOneWriteHasOccurred(TodoResource todoClient) { @@ -141,7 +184,8 @@ public void after() { Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); } - private Todo getTodoOfSize(int size) { - return ImmutableTodo.of(String.join("", Collections.nCopies(size, "a"))); + private static String getTodoOfSize(int size) { + // Note that the size of the cell for 1000 length text is actually 1050. + return String.join("", Collections.nCopies(size, "a")); } } From d5b5a8a049ace72069075e59ffe0bb45596c754a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:45:46 +0000 Subject: [PATCH 57/67] Add back lost file --- .../RateLimitExceededExceptionMapperTest.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 atlasdb-config/src/test/java/com/palantir/atlasdb/http/RateLimitExceededExceptionMapperTest.java diff --git a/atlasdb-config/src/test/java/com/palantir/atlasdb/http/RateLimitExceededExceptionMapperTest.java b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/RateLimitExceededExceptionMapperTest.java new file mode 100644 index 00000000000..d4e144e212d --- /dev/null +++ b/atlasdb-config/src/test/java/com/palantir/atlasdb/http/RateLimitExceededExceptionMapperTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.http; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Test; + +import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; + +public class RateLimitExceededExceptionMapperTest { + @Test + public void responsesHaveErrorCode429() { + RateLimitExceededException rateLimitExceededException = new RateLimitExceededException("Stop!"); + assertThat(new RateLimitExceededExceptionMapper().toResponse(rateLimitExceededException).getStatus()) + .isEqualTo(429); + } +} From 3f080db8967eae96485f1ec4b9565b9549416959 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 16:50:37 +0000 Subject: [PATCH 58/67] Cleanup --- .../cassandra/qos/QosCassandraClient.java | 17 ++--------------- .../com/palantir/atlasdb/todo/TodoClient.java | 2 +- .../atlasdb/qos/client/AtlasDbQosClient.java | 3 --- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java index 7988acff8a1..42f5d5d7137 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/qos/QosCassandraClient.java @@ -67,12 +67,8 @@ public Cassandra.Client rawClient() { public Map> multiget_slice(String kvsMethodName, TableReference tableRef, List keys, SlicePredicate predicate, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - System.out.println("multiget_slice from the table " + tableRef.getQualifiedName() + " with slice " + predicate.toString() + " and keys " + keys.size()); - return qosClient.executeRead( - () -> { - return client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level); - }, + () -> client.multiget_slice(kvsMethodName, tableRef, keys, predicate, consistency_level), ThriftQueryWeighers.multigetSlice(keys)); } @@ -80,11 +76,8 @@ public Map> multiget_slice(String kvsMetho public List get_range_slices(String kvsMethodName, TableReference tableRef, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - System.out.println("get_range_slices from the table " + tableRef.getQualifiedName() + " with slice " + predicate.toString() + " and keys " + range.toString()); return qosClient.executeRead( - () -> { - return client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level); - }, + () -> client.get_range_slices(kvsMethodName, tableRef, predicate, range, consistency_level), ThriftQueryWeighers.getRangeSlices(range)); } @@ -92,8 +85,6 @@ public List get_range_slices(String kvsMethodName, TableReference tabl public void batch_mutate(String kvsMethodName, Map>> mutation_map, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { - System.out.println("batch_mutate from the tables with mutation_map " + mutation_map.size()); - qosClient.executeWrite( () -> { client.batch_mutate(kvsMethodName, mutation_map, consistency_level); @@ -106,8 +97,6 @@ public void batch_mutate(String kvsMethodName, Map client.get(tableReference, key, column, consistency_level), ThriftQueryWeighers.GET); @@ -125,8 +114,6 @@ public CASResult cas(TableReference tableReference, ByteBuffer key, List public CqlResult execute_cql3_query(CqlQuery cqlQuery, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { - System.out.println("execute_cql3_query from the query " + cqlQuery.getLazySafeLoggableObject()); - return qosClient.executeRead( () -> client.execute_cql3_query(cqlQuery, compression, consistency), ThriftQueryWeighers.EXECUTE_CQL3_QUERY); diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java index 524a39193b5..34a41afbbfe 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/todo/TodoClient.java @@ -54,7 +54,7 @@ public List getTodoList() { TodoSchema.todoTable(), RangeRequest.all()); ImmutableList.Builder> rowResults = ImmutableList.builder(); - rowResultBatchingVisitable.batchAccept(10, items -> { + rowResultBatchingVisitable.batchAccept(1000, items -> { rowResults.addAll(items); return true; }); diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java index 9bac6752156..455bfa57036 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/client/AtlasDbQosClient.java @@ -73,7 +73,6 @@ private T execute( estimatedWeightMetric.ifPresent(metric -> metric.accept(estimatedWeight)); try { - System.out.println("Consuming estimated bytes of " + estimatedWeight.numBytes()); Duration waitTime = rateLimiter.consumeWithBackoff(estimatedWeight.numBytes()); metrics.recordBackoffMicros(TimeUnit.NANOSECONDS.toMicros(waitTime.toNanos())); } catch (RateLimitExceededException ex) { @@ -94,8 +93,6 @@ private T execute( } finally { weightMetric.accept(actualWeight); rateLimiter.recordAdjustment(actualWeight.numBytes() - estimatedWeight.numBytes()); - System.out.println("Recording adjusted bytes of " + (actualWeight.numBytes() - estimatedWeight.numBytes())); - } } From 504f345c5eff3a21b554d25cc30a643d0cfd64ae Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 17:02:02 +0000 Subject: [PATCH 59/67] Write tests --- .../ete/QosCassandraReadTestSuite.java | 4 +- .../ete/QosCassandraWriteTestSuite.java | 183 +++++++++--------- 2 files changed, 94 insertions(+), 93 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index f51d351a7a5..42e466eac0a 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -235,7 +235,7 @@ private List readOneBatchOfSize(int batchSize) { .collect(Collectors.toList()); } - public static AtlasDbConfig getAtlasDbConfig() { + private static AtlasDbConfig getAtlasDbConfig() { DockerPort cassandraPort = docker.containers() .container("cassandra") .port(CASSANDRA_PORT_NUMBER); @@ -261,7 +261,7 @@ public static AtlasDbConfig getAtlasDbConfig() { .build(); } - public static Optional getAtlasDbRuntimeConfig() { + private static Optional getAtlasDbRuntimeConfig() { return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() .sweep(ImmutableSweepConfig.builder().enabled(false).build()) .qos(ImmutableQosClientConfig.builder() diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java index 09f40de5172..140581bfd7d 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java @@ -15,25 +15,17 @@ */ package com.palantir.atlasdb.ete; -import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static com.palantir.atlasdb.ete.QosCassandraReadTestSuite.getAtlasDbConfig; - +import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; -import org.assertj.core.util.Lists; import org.awaitility.Awaitility; import org.awaitility.Duration; import org.junit.After; @@ -42,28 +34,38 @@ import org.junit.Test; import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; +import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; +import com.palantir.atlasdb.cassandra.ImmutableCassandraCredentialsConfig; +import com.palantir.atlasdb.cassandra.ImmutableCassandraKeyValueServiceConfig; +import com.palantir.atlasdb.config.AtlasDbConfig; +import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; +import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; +import com.palantir.atlasdb.config.ImmutableAtlasDbRuntimeConfig; +import com.palantir.atlasdb.config.ImmutableSweepConfig; import com.palantir.atlasdb.factory.TransactionManagers; -import com.palantir.atlasdb.http.errors.AtlasDbRemoteException; import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; +import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; +import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; import com.palantir.atlasdb.table.description.ValueType; -import com.palantir.atlasdb.todo.TodoResource; import com.palantir.atlasdb.todo.TodoSchema; -import com.palantir.atlasdb.transaction.api.Transaction; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.docker.compose.DockerComposeRule; import com.palantir.docker.compose.configuration.ShutdownStrategy; import com.palantir.docker.compose.connection.Container; +import com.palantir.docker.compose.connection.DockerPort; import com.palantir.docker.compose.logging.LogDirectory; +import com.palantir.remoting.api.config.service.HumanReadableDuration; public class QosCassandraWriteTestSuite { - private static final Random random = new Random(); private static SerializableTransactionManager serializableTransactionManager; + private static final int readBytesPerSecond = 10_000; + private static final int writeBytesPerSecond = 10_000; + private static final int CASSANDRA_PORT_NUMBER = 9160; @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() @@ -79,7 +81,7 @@ public void setup() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) + .runtimeConfigSupplier(QosCassandraWriteTestSuite::getAtlasDbRuntimeConfig) .schemas(ImmutableList.of(TodoSchema.getSchema())) .userAgent("qos-test") .buildSerializable(); @@ -88,95 +90,94 @@ public void setup() { .atMost(Duration.ONE_MINUTE) .pollInterval(Duration.ONE_SECOND) .until(serializableTransactionManager::isInitialized); - } @Test - public void shouldBeAbleToWriteBytesExcee() { - serializableTransactionManager.runTaskWithRetry((transaction) -> { - writeNTodosOfSize(transaction, 200, 1_000); - return null; - }); + public void shouldBeAbleToWriteSmallAmountOfBytesIfDoesNotExceedLimit() { + writeNTodosOfSize(1, 100); } - public static void writeNTodosOfSize(Transaction transaction, int numTodos, int size) { - Map write = new HashMap<>(); - for (int i = 0; i < numTodos; i++) { - Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), - TodoSchema.todoTextColumn()); - write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); - } - transaction.put(TodoSchema.todoTable(), write); + @Test + public void shouldBeAbleToWriteSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { + IntStream.range(0, 50) + .forEach(i -> writeNTodosOfSize(1, 100)); } - private void ensureOneWriteHasOccurred(TodoResource todoClient) { - try { - todoClient.addTodo(getTodoOfSize(100_000)); - // okay as the first huge write is not rate limited. - } catch (Exception e) { - // okay as some other test might have written before - } + @Test + public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitFirstTime() { + writeNTodosOfSize(12, 1_000); } @Test - public void canNotWriteLargeNumberOfBytesConcurrentlyIfAllRequestsComeAtTheExactSameTime() - throws InterruptedException { - TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - - CyclicBarrier barrier = new CyclicBarrier(100); - ForkJoinPool threadPool = new ForkJoinPool(100); - List> futures = Lists.newArrayList(); - - IntStream.range(0, 100).parallel().forEach(i -> - futures.add(threadPool.submit( - () -> { - barrier.await(); - todoClient.addTodo(getTodoOfSize(1_000)); - return null; - }))); - - AtomicInteger exceptionCounter = new AtomicInteger(90); - futures.forEach(future -> { - try { - future.get(); - } catch (ExecutionException e) { - if (e.getCause().getClass().equals(AtlasDbRemoteException.class)) { - exceptionCounter.getAndIncrement(); - } - } catch (InterruptedException e) { - throw Throwables.propagate(e); - } - }); - assertThat(exceptionCounter.get()).isGreaterThan(90); + public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { + writeNTodosOfSize(12, 1_000); + + writeNTodosOfSize(12, 1_000); } @Test - public void canNotWriteLargeNumberOfBytesConcurrently() throws InterruptedException { - TodoResource todoClient = EteSetup.createClientToSingleNode(TodoResource.class); - - ForkJoinPool threadPool = new ForkJoinPool(100); - List> futures = Lists.newArrayList(); - - IntStream.range(0, 100).parallel() - .forEach(i -> futures.add(threadPool.submit(() -> todoClient.addTodo(getTodoOfSize(1_000))))); - - threadPool.shutdown(); - Preconditions.checkState(threadPool.awaitTermination(90, TimeUnit.SECONDS), - "Not all threads writing data finished in the expected time."); - - AtomicInteger exceptionCounter = new AtomicInteger(0); - futures.forEach(future -> { - try { - future.get(); - } catch (ExecutionException e) { - if (e.getCause().getClass().equals(AtlasDbRemoteException.class)) { - exceptionCounter.getAndIncrement(); - } - } catch (InterruptedException e) { - throw Throwables.propagate(e); + public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { + // Have one limit-exceeding write + // as the rate-limiter will let anything pass through until the limit is exceeded. + writeNTodosOfSize(12, 1_000); + + assertThatThrownBy(() -> writeNTodosOfSize(200, 1_000)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); + } + + + public static void writeNTodosOfSize(int numTodos, int size) { + serializableTransactionManager.runTaskWithRetry((transaction) -> { + Map write = new HashMap<>(); + for (int i = 0; i < numTodos; i++) { + Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), + TodoSchema.todoTextColumn()); + write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); } + transaction.put(TodoSchema.todoTable(), write); + return null; }); - assertThat(exceptionCounter.get()).isGreaterThan(0); + + } + + private static AtlasDbConfig getAtlasDbConfig() { + DockerPort cassandraPort = docker.containers() + .container("cassandra") + .port(CASSANDRA_PORT_NUMBER); + + InetSocketAddress cassandraAddress = new InetSocketAddress(cassandraPort.getIp(), + cassandraPort.getExternalPort()); + + CassandraKeyValueServiceConfig kvsConfig = ImmutableCassandraKeyValueServiceConfig.builder() + .servers(ImmutableList.of(cassandraAddress)) + .credentials(ImmutableCassandraCredentialsConfig.builder() + .username("cassandra") + .password("cassandra") + .build()) + .ssl(false) + .replicationFactor(1) + .autoRefreshNodes(false) + .build(); + + return ImmutableAtlasDbConfig.builder() + .namespace("qosete") + .keyValueService(kvsConfig) + .initializeAsync(true) + .build(); + } + + private static Optional getAtlasDbRuntimeConfig() { + return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() + .sweep(ImmutableSweepConfig.builder().enabled(false).build()) + .qos(ImmutableQosClientConfig.builder() + .limits(ImmutableQosLimitsConfig.builder() + .readBytesPerSecond(readBytesPerSecond) + .writeBytesPerSecond(writeBytesPerSecond) + .build()) + .maxBackoffSleepTime(HumanReadableDuration.seconds(2)) + .build()) + .build()); } @After From 59e2b6b01e2cb5dafa772d934943ee6ba16504f7 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 17:09:52 +0000 Subject: [PATCH 60/67] numReadsPerThread -> numThreads --- .../com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index 42e466eac0a..c5fad07741f 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -176,10 +176,10 @@ public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws In int numThreads = 5; int numReadsPerThread = 10; ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - List>> futures = new ArrayList<>(numReadsPerThread); + List>> futures = new ArrayList<>(numThreads); long start = System.nanoTime(); - IntStream.range(0, numReadsPerThread) + IntStream.range(0, numThreads) .forEach(i -> futures.add(executorService.submit(() -> { List results = new ArrayList<>(numReadsPerThread); From 35327a91d6a48bc1a53ab4c73bab74e00ccca52d Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 17:55:11 +0000 Subject: [PATCH 61/67] More write tests, cleanup, check style fixes --- .../ete/QosCassandraReadTestSuite.java | 22 +++-- .../ete/QosCassandraWriteTestSuite.java | 83 ++++++++++++++++--- 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java index c5fad07741f..292439ca0ba 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java @@ -108,9 +108,9 @@ public static void createTransactionManagerAndWriteData() { .until(serializableTransactionManager::isInitialized); serializableTransactionManager.runTaskWithRetry((transaction) -> { - writeNTodosOfSize(transaction, 200, 1_000); - return null; - }); + writeNTodosOfSize(transaction, 200, 1_000); + return null; + }); serializableTransactionManager.close(); } @@ -179,15 +179,13 @@ public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws In List>> futures = new ArrayList<>(numThreads); long start = System.nanoTime(); - IntStream.range(0, numThreads) - .forEach(i -> - futures.add(executorService.submit(() -> { - List results = new ArrayList<>(numReadsPerThread); - IntStream.range(0, numReadsPerThread) - .forEach(j -> results.addAll(readOneBatchOfSize(1))); - return results; - })) - ); + IntStream.range(0, numThreads).forEach(i -> + futures.add(executorService.submit(() -> { + List results = new ArrayList<>(numReadsPerThread); + IntStream.range(0, numReadsPerThread) + .forEach(j -> results.addAll(readOneBatchOfSize(1))); + return results; + }))); executorService.shutdown(); Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), "Read tasks did not finish in 30s"); diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java index 140581bfd7d..4e9bf36b7a0 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java @@ -15,15 +15,24 @@ */ package com.palantir.atlasdb.ete; + +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; import org.awaitility.Awaitility; @@ -34,6 +43,9 @@ import org.junit.Test; import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; @@ -66,6 +78,8 @@ public class QosCassandraWriteTestSuite { private static final int readBytesPerSecond = 10_000; private static final int writeBytesPerSecond = 10_000; private static final int CASSANDRA_PORT_NUMBER = 9160; + private static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; + @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() @@ -99,8 +113,7 @@ public void shouldBeAbleToWriteSmallAmountOfBytesIfDoesNotExceedLimit() { @Test public void shouldBeAbleToWriteSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { - IntStream.range(0, 50) - .forEach(i -> writeNTodosOfSize(1, 100)); + IntStream.range(0, 50).forEach(i -> writeNTodosOfSize(1, 100)); } @Test @@ -110,24 +123,72 @@ public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitFirstTime() { @Test public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { - writeNTodosOfSize(12, 1_000); - - writeNTodosOfSize(12, 1_000); + // Have one quick limit-exceeding write, as the rate-limiter + // will let anything pass through until the limit is exceeded. + Stopwatch stopwatch = Stopwatch.createStarted(); + writeNTodosOfSize(1, 20_000); + long firstWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + stopwatch = Stopwatch.createStarted(); + writeNTodosOfSize(200, 1_000); + long secondWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + assertThat(secondWriteTime).isGreaterThan(firstWriteTime); + assertThat(secondWriteTime - firstWriteTime).isLessThan(MAX_SOFT_LIMITING_SLEEP_MILLIS); } @Test public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { - // Have one limit-exceeding write - // as the rate-limiter will let anything pass through until the limit is exceeded. - writeNTodosOfSize(12, 1_000); + // Have one quick limit-exceeding write, as the rate-limiter + // will let anything pass through until the limit is exceeded. + writeNTodosOfSize(1, 100_000); - assertThatThrownBy(() -> writeNTodosOfSize(200, 1_000)) + assertThatThrownBy(() -> writeNTodosOfSize(200, 10_000)) .isInstanceOf(RateLimitExceededException.class) .hasMessage("Rate limited. Available capacity has been exhausted."); } + @Test + public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws InterruptedException { + int numThreads = 5; + int numWritesPerThread = 10; + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List futures = new ArrayList<>(numThreads); + + long start = System.nanoTime(); + IntStream.range(0, numThreads).forEach(i -> + futures.add(executorService.submit(() -> { + IntStream.range(0, numWritesPerThread).forEach(j -> writeNTodosOfSize(1, 100)); + return null; + }))); + executorService.shutdown(); + Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), + "Read tasks did not finish in 30s"); + long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + + assertThatAllWritesWereSuccessful(futures); + long actualWriteRate = (numThreads * numWritesPerThread * 167) / timeTakenToReadInSeconds; + assertThat(actualWriteRate).isLessThan( + writeBytesPerSecond + (writeBytesPerSecond / 10 /* to allow burst time */)); + } + + private void assertThatAllWritesWereSuccessful(List futures) { + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof RateLimitExceededException) { + exceptionCounter.getAndIncrement(); + } + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isEqualTo(0); + } - public static void writeNTodosOfSize(int numTodos, int size) { + private static void writeNTodosOfSize(int numTodos, int size) { serializableTransactionManager.runTaskWithRetry((transaction) -> { Map write = new HashMap<>(); for (int i = 0; i < numTodos; i++) { @@ -175,7 +236,7 @@ private static Optional getAtlasDbRuntimeConfig() { .readBytesPerSecond(readBytesPerSecond) .writeBytesPerSecond(writeBytesPerSecond) .build()) - .maxBackoffSleepTime(HumanReadableDuration.seconds(2)) + .maxBackoffSleepTime(HumanReadableDuration.milliseconds(MAX_SOFT_LIMITING_SLEEP_MILLIS)) .build()) .build()); } From b04b4fd99ed52dec815dc4bcd6192671e7a2986a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 18:17:05 +0000 Subject: [PATCH 62/67] Refactor to avoid code duplication --- .../atlasdb/ete/QosCassandraReadEteTest.java | 155 ++++++++++ .../ete/QosCassandraReadTestSuite.java | 284 ------------------ ...tSuite.java => QosCassandraTestSetup.java} | 131 ++------ .../atlasdb/ete/QosCassandraWriteEteTest.java | 121 ++++++++ 4 files changed, 300 insertions(+), 391 deletions(-) create mode 100644 atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java delete mode 100644 atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java rename atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/{QosCassandraWriteTestSuite.java => QosCassandraTestSetup.java} (57%) create mode 100644 atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java new file mode 100644 index 00000000000..13e8cca09c1 --- /dev/null +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.palantir.atlasdb.ete; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.palantir.atlasdb.keyvalue.api.RangeRequest; +import com.palantir.atlasdb.keyvalue.api.RowResult; +import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; +import com.palantir.atlasdb.table.description.ValueType; +import com.palantir.atlasdb.todo.ImmutableTodo; +import com.palantir.atlasdb.todo.Todo; +import com.palantir.atlasdb.todo.TodoSchema; +import com.palantir.common.base.BatchingVisitable; + +public class QosCassandraReadEteTest extends QosCassandraTestSetup { + + private static final int ONE_TODO_SIZE_IN_BYTES = 1050; + + @BeforeClass + public static void createTransactionManagerAndWriteData() { + ensureTransactionManagerIsCreated(); + writeNTodosOfSize(200, 1_000); + serializableTransactionManager.close(); + } + + @Test + public void shouldBeAbleToReadSmallAmountOfBytesIfDoesNotExceedLimit() { + assertThat(readOneBatchOfSize(1)).hasSize(1); + } + + @Test + public void shouldBeAbleToReadSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { + IntStream.range(0, 50).forEach(i -> assertThat(readOneBatchOfSize(1)).hasSize(1)); + } + + @Test + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { + assertThat(readOneBatchOfSize(12)).hasSize(12); + } + + @Test + public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { + assertThat(readOneBatchOfSize(12)).hasSize(12); + // The second read might actually be faster as the transaction/metadata + assertThat(readOneBatchOfSize(12)).hasSize(12); + } + + @Test + public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { + assertThatThrownBy(() -> readOneBatchOfSize(200)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); + } + + @Test + public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws InterruptedException { + int numThreads = 5; + int numReadsPerThread = 10; + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List>> futures = new ArrayList<>(numThreads); + + long start = System.nanoTime(); + IntStream.range(0, numThreads).forEach(i -> + futures.add(executorService.submit(() -> { + List results = new ArrayList<>(numReadsPerThread); + IntStream.range(0, numReadsPerThread) + .forEach(j -> results.addAll(readOneBatchOfSize(1))); + return results; + }))); + executorService.shutdown(); + Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), + "Read tasks did not finish in 30s"); + long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + + assertThatAllReadsWereSuccessful(futures, numReadsPerThread); + long actualReadRate = (numThreads * numReadsPerThread * ONE_TODO_SIZE_IN_BYTES) / timeTakenToReadInSeconds; + assertThat(actualReadRate).isLessThan(readBytesPerSecond + (readBytesPerSecond / 10 /* to allow burst time */)); + } + + private void assertThatAllReadsWereSuccessful(List>> futures, int numReadsPerThread) { + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + assertThat(future.get()).hasSize(numReadsPerThread); + } catch (ExecutionException e) { + if (e.getCause() instanceof RateLimitExceededException) { + exceptionCounter.getAndIncrement(); + } + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isEqualTo(0); + } + + private List readOneBatchOfSize(int batchSize) { + ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { + BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( + TodoSchema.todoTable(), RangeRequest.all()); + ImmutableList.Builder> rowResults = ImmutableList.builder(); + + rowResultBatchingVisitable.batchAccept(batchSize, items -> { + rowResults.addAll(items); + return false; + }); + + return rowResults.build(); + }); + + return results.stream() + .map(RowResult::getOnlyColumnValue) + .map(ValueType.STRING::convertToString) + .map(ImmutableTodo::of) + .collect(Collectors.toList()); + } + + @After + public void cleanup() { + serializableTransactionManager.close(); + } + +} diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java deleted file mode 100644 index 292439ca0ba..00000000000 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadTestSuite.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2017 Palantir Technologies, Inc. All rights reserved. - * - * Licensed under the BSD-3 License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.palantir.atlasdb.ete; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Random; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import org.awaitility.Awaitility; -import org.awaitility.Duration; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; -import com.palantir.atlasdb.cassandra.ImmutableCassandraCredentialsConfig; -import com.palantir.atlasdb.cassandra.ImmutableCassandraKeyValueServiceConfig; -import com.palantir.atlasdb.config.AtlasDbConfig; -import com.palantir.atlasdb.config.AtlasDbRuntimeConfig; -import com.palantir.atlasdb.config.ImmutableAtlasDbConfig; -import com.palantir.atlasdb.config.ImmutableAtlasDbRuntimeConfig; -import com.palantir.atlasdb.config.ImmutableSweepConfig; -import com.palantir.atlasdb.factory.TransactionManagers; -import com.palantir.atlasdb.keyvalue.api.Cell; -import com.palantir.atlasdb.keyvalue.api.RangeRequest; -import com.palantir.atlasdb.keyvalue.api.RowResult; -import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; -import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; -import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; -import com.palantir.atlasdb.table.description.ValueType; -import com.palantir.atlasdb.todo.ImmutableTodo; -import com.palantir.atlasdb.todo.Todo; -import com.palantir.atlasdb.todo.TodoSchema; -import com.palantir.atlasdb.transaction.api.Transaction; -import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; -import com.palantir.atlasdb.util.AtlasDbMetrics; -import com.palantir.common.base.BatchingVisitable; -import com.palantir.docker.compose.DockerComposeRule; -import com.palantir.docker.compose.configuration.ShutdownStrategy; -import com.palantir.docker.compose.connection.Container; -import com.palantir.docker.compose.connection.DockerPort; -import com.palantir.docker.compose.logging.LogDirectory; -import com.palantir.remoting.api.config.service.HumanReadableDuration; - -public class QosCassandraReadTestSuite { - private static final Random random = new Random(); - private static final int CASSANDRA_PORT_NUMBER = 9160; - private static SerializableTransactionManager serializableTransactionManager; - private static final int readBytesPerSecond = 10_000; - private static final int writeBytesPerSecond = 10_000; - private static final int ONE_TODO_SIZE_IN_BYTES = 1050; - - @ClassRule - public static DockerComposeRule docker = DockerComposeRule.builder() - .file("src/test/resources/cassandra-docker-compose.yml") - .waitingForService("cassandra", Container::areAllPortsOpen) - .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) - .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) - .build(); - - @BeforeClass - public static void createTransactionManagerAndWriteData() { - serializableTransactionManager = TransactionManagers.builder() - .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) - .schemas(ImmutableList.of(TodoSchema.getSchema())) - .userAgent("qos-test") - .buildSerializable(); - - Awaitility.await() - .atMost(Duration.ONE_MINUTE) - .pollInterval(Duration.ONE_SECOND) - .until(serializableTransactionManager::isInitialized); - - serializableTransactionManager.runTaskWithRetry((transaction) -> { - writeNTodosOfSize(transaction, 200, 1_000); - return null; - }); - serializableTransactionManager.close(); - } - - public static void writeNTodosOfSize(Transaction transaction, int numTodos, int size) { - Map write = new HashMap<>(); - for (int i = 0; i < numTodos; i++) { - Cell thisCell = Cell.create(ValueType.FIXED_LONG.convertFromJava(random.nextLong()), - TodoSchema.todoTextColumn()); - write.put(thisCell, ValueType.STRING.convertFromJava(getTodoOfSize(size))); - } - transaction.put(TodoSchema.todoTable(), write); - } - - @Before - public void setup() { - AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); - - serializableTransactionManager = TransactionManagers.builder() - .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraReadTestSuite::getAtlasDbRuntimeConfig) - .schemas(ImmutableList.of(TodoSchema.getSchema())) - .userAgent("qos-test") - .buildSerializable(); - - Awaitility.await() - .atMost(Duration.ONE_MINUTE) - .pollInterval(Duration.ONE_SECOND) - .until(serializableTransactionManager::isInitialized); - } - - @Test - public void shouldBeAbleToReadSmallAmountOfBytesIfDoesNotExceedLimit() { - assertThat(readOneBatchOfSize(1)).hasSize(1); - } - - @Test - public void shouldBeAbleToReadSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { - IntStream.range(0, 50) - .forEach(i -> assertThat(readOneBatchOfSize(1)).hasSize(1)); - } - - @Test - public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { - assertThat(readOneBatchOfSize(12)).hasSize(12); - } - - @Test - public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { - assertThat(readOneBatchOfSize(12)).hasSize(12); - // The second read might actually be faster as the transaction/metadata - assertThat(readOneBatchOfSize(12)).hasSize(12); - } - - @Test - public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { - assertThatThrownBy(() -> readOneBatchOfSize(200)) - .isInstanceOf(RateLimitExceededException.class) - .hasMessage("Rate limited. Available capacity has been exhausted."); - } - - @Test - public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws InterruptedException { - int numThreads = 5; - int numReadsPerThread = 10; - ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - List>> futures = new ArrayList<>(numThreads); - - long start = System.nanoTime(); - IntStream.range(0, numThreads).forEach(i -> - futures.add(executorService.submit(() -> { - List results = new ArrayList<>(numReadsPerThread); - IntStream.range(0, numReadsPerThread) - .forEach(j -> results.addAll(readOneBatchOfSize(1))); - return results; - }))); - executorService.shutdown(); - Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), - "Read tasks did not finish in 30s"); - long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); - - assertThatAllReadsWereSuccessful(futures, numReadsPerThread); - long actualReadRate = (numThreads * numReadsPerThread * ONE_TODO_SIZE_IN_BYTES) / timeTakenToReadInSeconds; - assertThat(actualReadRate).isLessThan(readBytesPerSecond + (readBytesPerSecond / 10 /* to allow burst time */)); - } - - private void assertThatAllReadsWereSuccessful(List>> futures, int numReadsPerThread) { - AtomicInteger exceptionCounter = new AtomicInteger(0); - futures.forEach(future -> { - try { - assertThat(future.get()).hasSize(numReadsPerThread); - } catch (ExecutionException e) { - if (e.getCause() instanceof RateLimitExceededException) { - exceptionCounter.getAndIncrement(); - } - } catch (InterruptedException e) { - throw Throwables.propagate(e); - } - }); - assertThat(exceptionCounter.get()).isEqualTo(0); - } - - private List readOneBatchOfSize(int batchSize) { - ImmutableList> results = serializableTransactionManager.runTaskWithRetry((transaction) -> { - BatchingVisitable> rowResultBatchingVisitable = transaction.getRange( - TodoSchema.todoTable(), RangeRequest.all()); - ImmutableList.Builder> rowResults = ImmutableList.builder(); - - rowResultBatchingVisitable.batchAccept(batchSize, items -> { - rowResults.addAll(items); - return false; - }); - - return rowResults.build(); - }); - - return results.stream() - .map(RowResult::getOnlyColumnValue) - .map(ValueType.STRING::convertToString) - .map(ImmutableTodo::of) - .collect(Collectors.toList()); - } - - private static AtlasDbConfig getAtlasDbConfig() { - DockerPort cassandraPort = docker.containers() - .container("cassandra") - .port(CASSANDRA_PORT_NUMBER); - - InetSocketAddress cassandraAddress = new InetSocketAddress(cassandraPort.getIp(), - cassandraPort.getExternalPort()); - - CassandraKeyValueServiceConfig kvsConfig = ImmutableCassandraKeyValueServiceConfig.builder() - .servers(ImmutableList.of(cassandraAddress)) - .credentials(ImmutableCassandraCredentialsConfig.builder() - .username("cassandra") - .password("cassandra") - .build()) - .ssl(false) - .replicationFactor(1) - .autoRefreshNodes(false) - .build(); - - return ImmutableAtlasDbConfig.builder() - .namespace("qosete") - .keyValueService(kvsConfig) - .initializeAsync(true) - .build(); - } - - private static Optional getAtlasDbRuntimeConfig() { - return Optional.of(ImmutableAtlasDbRuntimeConfig.builder() - .sweep(ImmutableSweepConfig.builder().enabled(false).build()) - .qos(ImmutableQosClientConfig.builder() - .limits(ImmutableQosLimitsConfig.builder() - .readBytesPerSecond(readBytesPerSecond) - .writeBytesPerSecond(writeBytesPerSecond) - .build()) - .maxBackoffSleepTime(HumanReadableDuration.seconds(2)) - .build()) - .build()); - } - - @After - public void cleanup() { - serializableTransactionManager.close(); - } - - private static String getTodoOfSize(int size) { - // Note that the size of the cell for 1000 length text is actually 1050. - return String.join("", Collections.nCopies(size, "a")); - } -} diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java similarity index 57% rename from atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java rename to atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java index 4e9bf36b7a0..47b0bbfd126 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteTestSuite.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java @@ -16,36 +16,22 @@ package com.palantir.atlasdb.ete; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; import org.awaitility.Awaitility; import org.awaitility.Duration; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Test; import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; @@ -60,7 +46,6 @@ import com.palantir.atlasdb.keyvalue.api.Cell; import com.palantir.atlasdb.qos.config.ImmutableQosClientConfig; import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig; -import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; import com.palantir.atlasdb.table.description.ValueType; import com.palantir.atlasdb.todo.TodoSchema; import com.palantir.atlasdb.transaction.impl.SerializableTransactionManager; @@ -72,30 +57,27 @@ import com.palantir.docker.compose.logging.LogDirectory; import com.palantir.remoting.api.config.service.HumanReadableDuration; -public class QosCassandraWriteTestSuite { +public class QosCassandraTestSetup { private static final Random random = new Random(); - private static SerializableTransactionManager serializableTransactionManager; - private static final int readBytesPerSecond = 10_000; - private static final int writeBytesPerSecond = 10_000; + protected static SerializableTransactionManager serializableTransactionManager; + protected static final int readBytesPerSecond = 10_000; + protected static final int writeBytesPerSecond = 10_000; private static final int CASSANDRA_PORT_NUMBER = 9160; - private static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; - + protected static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() .file("src/test/resources/cassandra-docker-compose.yml") .waitingForService("cassandra", Container::areAllPortsOpen) - .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadTestSuite.class)) + .saveLogsTo(LogDirectory.circleAwareLogDirectory(QosCassandraReadEteTest.class)) .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); - @Before - public void setup() { - AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); - + @BeforeClass + public static void ensureTransactionManagerIsCreated() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraWriteTestSuite::getAtlasDbRuntimeConfig) + .runtimeConfigSupplier(QosCassandraTestSetup::getAtlasDbRuntimeConfig) .schemas(ImmutableList.of(TodoSchema.getSchema())) .userAgent("qos-test") .buildSerializable(); @@ -106,89 +88,24 @@ public void setup() { .until(serializableTransactionManager::isInitialized); } - @Test - public void shouldBeAbleToWriteSmallAmountOfBytesIfDoesNotExceedLimit() { - writeNTodosOfSize(1, 100); - } - - @Test - public void shouldBeAbleToWriteSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { - IntStream.range(0, 50).forEach(i -> writeNTodosOfSize(1, 100)); - } - - @Test - public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitFirstTime() { - writeNTodosOfSize(12, 1_000); - } - - @Test - public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { - // Have one quick limit-exceeding write, as the rate-limiter - // will let anything pass through until the limit is exceeded. - Stopwatch stopwatch = Stopwatch.createStarted(); - writeNTodosOfSize(1, 20_000); - long firstWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - stopwatch = Stopwatch.createStarted(); - writeNTodosOfSize(200, 1_000); - long secondWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - assertThat(secondWriteTime).isGreaterThan(firstWriteTime); - assertThat(secondWriteTime - firstWriteTime).isLessThan(MAX_SOFT_LIMITING_SLEEP_MILLIS); - } - - @Test - public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { - // Have one quick limit-exceeding write, as the rate-limiter - // will let anything pass through until the limit is exceeded. - writeNTodosOfSize(1, 100_000); - - assertThatThrownBy(() -> writeNTodosOfSize(200, 10_000)) - .isInstanceOf(RateLimitExceededException.class) - .hasMessage("Rate limited. Available capacity has been exhausted."); - } + @Before + public void setup() { + AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); - @Test - public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws InterruptedException { - int numThreads = 5; - int numWritesPerThread = 10; - ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - List futures = new ArrayList<>(numThreads); - - long start = System.nanoTime(); - IntStream.range(0, numThreads).forEach(i -> - futures.add(executorService.submit(() -> { - IntStream.range(0, numWritesPerThread).forEach(j -> writeNTodosOfSize(1, 100)); - return null; - }))); - executorService.shutdown(); - Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), - "Read tasks did not finish in 30s"); - long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); - - assertThatAllWritesWereSuccessful(futures); - long actualWriteRate = (numThreads * numWritesPerThread * 167) / timeTakenToReadInSeconds; - assertThat(actualWriteRate).isLessThan( - writeBytesPerSecond + (writeBytesPerSecond / 10 /* to allow burst time */)); - } + serializableTransactionManager = TransactionManagers.builder() + .config(getAtlasDbConfig()) + .runtimeConfigSupplier(QosCassandraTestSetup::getAtlasDbRuntimeConfig) + .schemas(ImmutableList.of(TodoSchema.getSchema())) + .userAgent("qos-test") + .buildSerializable(); - private void assertThatAllWritesWereSuccessful(List futures) { - AtomicInteger exceptionCounter = new AtomicInteger(0); - futures.forEach(future -> { - try { - future.get(); - } catch (ExecutionException e) { - if (e.getCause() instanceof RateLimitExceededException) { - exceptionCounter.getAndIncrement(); - } - } catch (InterruptedException e) { - throw Throwables.propagate(e); - } - }); - assertThat(exceptionCounter.get()).isEqualTo(0); + Awaitility.await() + .atMost(Duration.ONE_MINUTE) + .pollInterval(Duration.ONE_SECOND) + .until(serializableTransactionManager::isInitialized); } - private static void writeNTodosOfSize(int numTodos, int size) { + protected static void writeNTodosOfSize(int numTodos, int size) { serializableTransactionManager.runTaskWithRetry((transaction) -> { Map write = new HashMap<>(); for (int i = 0; i < numTodos; i++) { diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java new file mode 100644 index 00000000000..4246e873811 --- /dev/null +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java @@ -0,0 +1,121 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.ete; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; + +import org.junit.Test; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; + +public class QosCassandraWriteEteTest extends QosCassandraTestSetup { + @Test + public void shouldBeAbleToWriteSmallAmountOfBytesIfDoesNotExceedLimit() { + writeNTodosOfSize(1, 100); + } + + @Test + public void shouldBeAbleToWriteSmallAmountOfBytesSeriallyIfDoesNotExceedLimit() { + IntStream.range(0, 50).forEach(i -> writeNTodosOfSize(1, 100)); + } + + @Test + public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitFirstTime() { + writeNTodosOfSize(12, 1_000); + } + + @Test + public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { + // Have one quick limit-exceeding write, as the rate-limiter + // will let anything pass through until the limit is exceeded. + Stopwatch stopwatch = Stopwatch.createStarted(); + writeNTodosOfSize(1, 20_000); + long firstWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + stopwatch = Stopwatch.createStarted(); + writeNTodosOfSize(200, 1_000); + long secondWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + assertThat(secondWriteTime).isGreaterThan(firstWriteTime); + assertThat(secondWriteTime - firstWriteTime).isLessThan(MAX_SOFT_LIMITING_SLEEP_MILLIS); + } + + @Test + public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConfiguredBackoffTime() { + // Have one quick limit-exceeding write, as the rate-limiter + // will let anything pass through until the limit is exceeded. + writeNTodosOfSize(1, 100_000); + + assertThatThrownBy(() -> writeNTodosOfSize(200, 10_000)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); + } + + @Test + public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws InterruptedException { + int numThreads = 5; + int numWritesPerThread = 10; + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List futures = new ArrayList<>(numThreads); + + long start = System.nanoTime(); + IntStream.range(0, numThreads).forEach(i -> + futures.add(executorService.submit(() -> { + IntStream.range(0, numWritesPerThread).forEach(j -> writeNTodosOfSize(1, 100)); + return null; + }))); + executorService.shutdown(); + Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), + "Read tasks did not finish in 30s"); + long timeTakenToWriteInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + + assertThatAllWritesWereSuccessful(futures); + long actualWriteRate = (numThreads * numWritesPerThread * 167) / timeTakenToWriteInSeconds; + assertThat(actualWriteRate).isLessThan( + writeBytesPerSecond + (writeBytesPerSecond / 10 /* to allow burst time */)); + } + + private void assertThatAllWritesWereSuccessful(List futures) { + AtomicInteger exceptionCounter = new AtomicInteger(0); + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof RateLimitExceededException) { + exceptionCounter.getAndIncrement(); + } + } catch (InterruptedException e) { + throw Throwables.propagate(e); + } + }); + assertThat(exceptionCounter.get()).isEqualTo(0); + } +} From c9ebe6e3fc8ba7f7d6648dcc4afff68153d061cb Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 22 Nov 2017 19:16:46 +0000 Subject: [PATCH 63/67] Cleanup --- .../docker-compose.qos.cassandra.yml | 32 ------------- .../docker/conf/atlasdb-ete.qos.cassandra.yml | 36 --------------- .../palantir/atlasdb/AtlasDbEteServer.java | 10 ++-- ...tup.java => QosCassandraEteTestSetup.java} | 46 ++++++------------- .../atlasdb/ete/QosCassandraReadEteTest.java | 12 +---- .../atlasdb/ete/QosCassandraWriteEteTest.java | 10 ++-- 6 files changed, 29 insertions(+), 117 deletions(-) delete mode 100644 atlasdb-ete-tests/docker-compose.qos.cassandra.yml delete mode 100644 atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml rename atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/{QosCassandraTestSetup.java => QosCassandraEteTestSetup.java} (82%) diff --git a/atlasdb-ete-tests/docker-compose.qos.cassandra.yml b/atlasdb-ete-tests/docker-compose.qos.cassandra.yml deleted file mode 100644 index ceacb6f135a..00000000000 --- a/atlasdb-ete-tests/docker-compose.qos.cassandra.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2' - -services: - cassandra: - image: palantirtechnologies/docker-cassandra-atlasdb:$CASSANDRA_VERSION - ports: - - "9160" - - "9042" - - "7199" - environment: - - MAX_HEAP_SIZE=512m - - HEAP_NEWSIZE=64m - - LOCAL_JMX=no - - ete1: - build: . - command: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && dockerize -timeout 120s -wait tcp://cassandra:9160 && service/bin/init.sh console'] - ports: - - "3828" - depends_on: - - cassandra - environment: - - ME=ete1 - - ete-cli: - build: . - entrypoint: [bash, -c, 'cp var/conf/atlasdb-ete.qos.cassandra.yml var/conf/atlasdb-ete.yml && "$$@"', --] - command: exit - depends_on: - - ete1 - environment: - - ME=ete1 diff --git a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml b/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml deleted file mode 100644 index abc2a7d2517..00000000000 --- a/atlasdb-ete-tests/docker/conf/atlasdb-ete.qos.cassandra.yml +++ /dev/null @@ -1,36 +0,0 @@ -server: - #rootPath: /api/* - #applicationContextPath: / - #adminContextPath: / - applicationConnectors: - - type: http - port: 3828 - adminConnectors: - - type: http - port: 3829 - -atlasdb: - keyValueService: - type: cassandra - servers: - - cassandra:9160 - poolSize: 20 - keyspace: atlasete - credentials: - username: cassandra - password: cassandra - ssl: false - replicationFactor: 1 - mutationBatchCount: 10000 - mutationBatchSizeBytes: 10000000 - fetchBatchCount: 1000 - autoRefreshNodes: false - -atlasdbRuntime: - sweep: - enabled: false - qos: - limits: - readBytesPerSecond: 1000 - writeBytesPerSecond: 1000 - diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java index 1298a7a8407..553cdbe1a9e 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java @@ -81,13 +81,15 @@ private TransactionManager tryToCreateTransactionManager(AtlasDbEteConfiguration if (config.getAtlasDbConfig().initializeAsync()) { return createTransactionManager(config.getAtlasDbConfig(), environment, config.getAtlasDbRuntimeConfig()); } else { - return createTransactionManagerWithRetry(config.getAtlasDbConfig(), environment, - config.getAtlasDbRuntimeConfig()); + return createTransactionManagerWithRetry(config.getAtlasDbConfig(), config.getAtlasDbRuntimeConfig(), + environment + ); } } - private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig config, Environment environment, - Optional atlasDbRuntimeConfig) + private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig config, + Optional atlasDbRuntimeConfig, + Environment environment) throws InterruptedException { Stopwatch sw = Stopwatch.createStarted(); while (sw.elapsed(TimeUnit.SECONDS) < CREATE_TRANSACTION_MANAGER_MAX_WAIT_TIME_SECS) { diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java similarity index 82% rename from atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java rename to atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java index 47b0bbfd126..650235b045f 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraTestSetup.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java @@ -22,18 +22,15 @@ import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; import org.awaitility.Duration; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.ClassRule; import com.codahale.metrics.MetricRegistry; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Uninterruptibles; import com.palantir.atlasdb.cassandra.CassandraKeyValueServiceConfig; import com.palantir.atlasdb.cassandra.ImmutableCassandraCredentialsConfig; import com.palantir.atlasdb.cassandra.ImmutableCassandraKeyValueServiceConfig; @@ -57,13 +54,13 @@ import com.palantir.docker.compose.logging.LogDirectory; import com.palantir.remoting.api.config.service.HumanReadableDuration; -public class QosCassandraTestSetup { +public class QosCassandraEteTestSetup { private static final Random random = new Random(); - protected static SerializableTransactionManager serializableTransactionManager; - protected static final int readBytesPerSecond = 10_000; - protected static final int writeBytesPerSecond = 10_000; + static SerializableTransactionManager serializableTransactionManager; + static final int readBytesPerSecond = 10_000; + static final int writeBytesPerSecond = 10_000; private static final int CASSANDRA_PORT_NUMBER = 9160; - protected static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; + static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() @@ -73,28 +70,16 @@ public class QosCassandraTestSetup { .shutdownStrategy(ShutdownStrategy.AGGRESSIVE_WITH_NETWORK_CLEANUP) .build(); - @BeforeClass - public static void ensureTransactionManagerIsCreated() { - serializableTransactionManager = TransactionManagers.builder() - .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraTestSetup::getAtlasDbRuntimeConfig) - .schemas(ImmutableList.of(TodoSchema.getSchema())) - .userAgent("qos-test") - .buildSerializable(); - - Awaitility.await() - .atMost(Duration.ONE_MINUTE) - .pollInterval(Duration.ONE_SECOND) - .until(serializableTransactionManager::isInitialized); - } - @Before public void setup() { AtlasDbMetrics.setMetricRegistry(new MetricRegistry()); + ensureTransactionManagerIsCreated(); + } + static void ensureTransactionManagerIsCreated() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) - .runtimeConfigSupplier(QosCassandraTestSetup::getAtlasDbRuntimeConfig) + .runtimeConfigSupplier(QosCassandraEteTestSetup::getAtlasDbRuntimeConfig) .schemas(ImmutableList.of(TodoSchema.getSchema())) .userAgent("qos-test") .buildSerializable(); @@ -105,7 +90,7 @@ public void setup() { .until(serializableTransactionManager::isInitialized); } - protected static void writeNTodosOfSize(int numTodos, int size) { + static void writeNTodosOfSize(int numTodos, int size) { serializableTransactionManager.runTaskWithRetry((transaction) -> { Map write = new HashMap<>(); for (int i = 0; i < numTodos; i++) { @@ -116,7 +101,6 @@ protected static void writeNTodosOfSize(int numTodos, int size) { transaction.put(TodoSchema.todoTable(), write); return null; }); - } private static AtlasDbConfig getAtlasDbConfig() { @@ -158,13 +142,13 @@ private static Optional getAtlasDbRuntimeConfig() { .build()); } - @After - public void after() { - Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); - } - private static String getTodoOfSize(int size) { // Note that the size of the cell for 1000 length text is actually 1050. return String.join("", Collections.nCopies(size, "a")); } + + @After + public void after() { + serializableTransactionManager.close(); + } } diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java index 13e8cca09c1..925f180749d 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java @@ -29,7 +29,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; @@ -45,8 +44,7 @@ import com.palantir.atlasdb.todo.TodoSchema; import com.palantir.common.base.BatchingVisitable; -public class QosCassandraReadEteTest extends QosCassandraTestSetup { - +public class QosCassandraReadEteTest extends QosCassandraEteTestSetup { private static final int ONE_TODO_SIZE_IN_BYTES = 1050; @BeforeClass @@ -74,7 +72,7 @@ public void shouldBeAbleToReadLargeAmountsExceedingTheLimitFirstTime() { @Test public void shouldBeAbleToReadLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { assertThat(readOneBatchOfSize(12)).hasSize(12); - // The second read might actually be faster as the transaction/metadata + // The second read might actually be faster as the transaction/metadata is cached assertThat(readOneBatchOfSize(12)).hasSize(12); } @@ -146,10 +144,4 @@ private List readOneBatchOfSize(int batchSize) { .map(ImmutableTodo::of) .collect(Collectors.toList()); } - - @After - public void cleanup() { - serializableTransactionManager.close(); - } - } diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java index 4246e873811..06b3c9516ec 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java @@ -36,7 +36,8 @@ import com.google.common.base.Throwables; import com.palantir.atlasdb.qos.ratelimit.RateLimitExceededException; -public class QosCassandraWriteEteTest extends QosCassandraTestSetup { +public class QosCassandraWriteEteTest extends QosCassandraEteTestSetup { + @Test public void shouldBeAbleToWriteSmallAmountOfBytesIfDoesNotExceedLimit() { writeNTodosOfSize(1, 100); @@ -54,8 +55,6 @@ public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitFirstTime() { @Test public void shouldBeAbleToWriteLargeAmountsExceedingTheLimitSecondTimeWithSoftLimiting() { - // Have one quick limit-exceeding write, as the rate-limiter - // will let anything pass through until the limit is exceeded. Stopwatch stopwatch = Stopwatch.createStarted(); writeNTodosOfSize(1, 20_000); long firstWriteTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); @@ -81,8 +80,11 @@ public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConf @Test public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws InterruptedException { + int oneTodoSizeInBytes = 167; + int numThreads = 5; int numWritesPerThread = 10; + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); List futures = new ArrayList<>(numThreads); @@ -98,7 +100,7 @@ public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws I long timeTakenToWriteInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); assertThatAllWritesWereSuccessful(futures); - long actualWriteRate = (numThreads * numWritesPerThread * 167) / timeTakenToWriteInSeconds; + long actualWriteRate = (numThreads * numWritesPerThread * oneTodoSizeInBytes) / timeTakenToWriteInSeconds; assertThat(actualWriteRate).isLessThan( writeBytesPerSecond + (writeBytesPerSecond / 10 /* to allow burst time */)); } From af9d67ca1e20906b6aafeeef12975f6cada0cf15 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 23 Nov 2017 14:05:00 +0000 Subject: [PATCH 64/67] cr comments --- .../palantir/atlasdb/AtlasDbEteConfiguration.java | 2 +- .../com/palantir/atlasdb/AtlasDbEteServer.java | 14 +++++++------- .../atlasdb/ete/QosCassandraEteTestSetup.java | 12 ++++++------ .../atlasdb/ete/QosCassandraReadEteTest.java | 9 ++++++--- .../atlasdb/ete/QosCassandraWriteEteTest.java | 9 +++++---- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java index 2729b38d1e2..a3a137580a7 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteConfiguration.java @@ -29,7 +29,7 @@ public class AtlasDbEteConfiguration extends Configuration implements AtlasDbCon private final Optional atlasdbRuntime; public AtlasDbEteConfiguration(@JsonProperty("atlasdb") AtlasDbConfig atlasdb, - @JsonProperty("atlasdbRuntime") Optional atlasDbRuntimeConfig) { + @JsonProperty("atlasDbRuntime") Optional atlasDbRuntimeConfig) { this.atlasdb = atlasdb; this.atlasdbRuntime = atlasDbRuntimeConfig; } diff --git a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java index 553cdbe1a9e..4917373614c 100644 --- a/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java +++ b/atlasdb-ete-tests/src/main/java/com/palantir/atlasdb/AtlasDbEteServer.java @@ -79,11 +79,11 @@ public void run(AtlasDbEteConfiguration config, final Environment environment) t private TransactionManager tryToCreateTransactionManager(AtlasDbEteConfiguration config, Environment environment) throws InterruptedException { if (config.getAtlasDbConfig().initializeAsync()) { - return createTransactionManager(config.getAtlasDbConfig(), environment, config.getAtlasDbRuntimeConfig()); + return createTransactionManager(config.getAtlasDbConfig(), config.getAtlasDbRuntimeConfig(), environment); } else { - return createTransactionManagerWithRetry(config.getAtlasDbConfig(), config.getAtlasDbRuntimeConfig(), - environment - ); + return createTransactionManagerWithRetry(config.getAtlasDbConfig(), + config.getAtlasDbRuntimeConfig(), + environment); } } @@ -94,7 +94,7 @@ private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig confi Stopwatch sw = Stopwatch.createStarted(); while (sw.elapsed(TimeUnit.SECONDS) < CREATE_TRANSACTION_MANAGER_MAX_WAIT_TIME_SECS) { try { - return createTransactionManager(config, environment, atlasDbRuntimeConfig); + return createTransactionManager(config, atlasDbRuntimeConfig, environment); } catch (RuntimeException e) { log.warn("An error occurred while trying to create transaction manager. Retrying...", e); Thread.sleep(CREATE_TRANSACTION_MANAGER_POLL_INTERVAL_SECS); @@ -103,8 +103,8 @@ private TransactionManager createTransactionManagerWithRetry(AtlasDbConfig confi throw new IllegalStateException("Timed-out because we were unable to create transaction manager"); } - private TransactionManager createTransactionManager(AtlasDbConfig config, Environment environment, - Optional atlasDbRuntimeConfigOptional) { + private TransactionManager createTransactionManager(AtlasDbConfig config, + Optional atlasDbRuntimeConfigOptional, Environment environment) { return TransactionManagers.builder() .config(config) .schemas(ETE_SCHEMAS) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java index 650235b045f..676f6599548 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraEteTestSetup.java @@ -56,11 +56,11 @@ public class QosCassandraEteTestSetup { private static final Random random = new Random(); - static SerializableTransactionManager serializableTransactionManager; - static final int readBytesPerSecond = 10_000; - static final int writeBytesPerSecond = 10_000; + protected static SerializableTransactionManager serializableTransactionManager; + protected static final int readBytesPerSecond = 10_000; + protected static final int writeBytesPerSecond = 10_000; private static final int CASSANDRA_PORT_NUMBER = 9160; - static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; + protected static final int MAX_SOFT_LIMITING_SLEEP_MILLIS = 2000; @ClassRule public static DockerComposeRule docker = DockerComposeRule.builder() @@ -76,7 +76,7 @@ public void setup() { ensureTransactionManagerIsCreated(); } - static void ensureTransactionManagerIsCreated() { + protected static void ensureTransactionManagerIsCreated() { serializableTransactionManager = TransactionManagers.builder() .config(getAtlasDbConfig()) .runtimeConfigSupplier(QosCassandraEteTestSetup::getAtlasDbRuntimeConfig) @@ -90,7 +90,7 @@ static void ensureTransactionManagerIsCreated() { .until(serializableTransactionManager::isInitialized); } - static void writeNTodosOfSize(int numTodos, int size) { + protected static void writeNTodosOfSize(int numTodos, int size) { serializableTransactionManager.runTaskWithRetry((transaction) -> { Map write = new HashMap<>(); for (int i = 0; i < numTodos; i++) { diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java index 925f180749d..df6b0bba0b0 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java @@ -101,11 +101,14 @@ public void readRateLimitShouldBeRespectedByConcurrentReadingThreads() throws In executorService.shutdown(); Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), "Read tasks did not finish in 30s"); - long timeTakenToReadInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + long readTime = System.nanoTime() - start; assertThatAllReadsWereSuccessful(futures, numReadsPerThread); - long actualReadRate = (numThreads * numReadsPerThread * ONE_TODO_SIZE_IN_BYTES) / timeTakenToReadInSeconds; - assertThat(actualReadRate).isLessThan(readBytesPerSecond + (readBytesPerSecond / 10 /* to allow burst time */)); + double actualBytesRead = numThreads * numReadsPerThread * ONE_TODO_SIZE_IN_BYTES; + double maxReadBytesLimit = readBytesPerSecond * ((double) readTime / TimeUnit.SECONDS.toNanos(1) + + 5 /* to allow for rate-limiter burst */); + + assertThat(actualBytesRead).isLessThan(maxReadBytesLimit); } private void assertThatAllReadsWereSuccessful(List>> futures, int numReadsPerThread) { diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java index 06b3c9516ec..4c4fafaffdc 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java @@ -97,12 +97,13 @@ public void writeRateLimitShouldBeRespectedByConcurrentWritingThreads() throws I executorService.shutdown(); Preconditions.checkState(executorService.awaitTermination(30L, TimeUnit.SECONDS), "Read tasks did not finish in 30s"); - long timeTakenToWriteInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + long writeTime = System.nanoTime() - start; assertThatAllWritesWereSuccessful(futures); - long actualWriteRate = (numThreads * numWritesPerThread * oneTodoSizeInBytes) / timeTakenToWriteInSeconds; - assertThat(actualWriteRate).isLessThan( - writeBytesPerSecond + (writeBytesPerSecond / 10 /* to allow burst time */)); + double actualBytesWritten = numThreads * numWritesPerThread * oneTodoSizeInBytes; + double maxReadBytesLimit = readBytesPerSecond * ((double) writeTime / TimeUnit.SECONDS.toNanos(1) + + 5 /* to allow for rate-limiter burst */); + assertThat(actualBytesWritten).isLessThan(maxReadBytesLimit); } private void assertThatAllWritesWereSuccessful(List futures) { From 6f483d9dde02a5c2ea3d3ebd34fb1cbe0ac42670 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 23 Nov 2017 14:49:57 +0000 Subject: [PATCH 65/67] Small read/write after a rate-limited read/write --- .../com/palantir/atlasdb/ete/QosCassandraReadEteTest.java | 5 +++++ .../com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java index df6b0bba0b0..b49402e19ca 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraReadEteTest.java @@ -81,6 +81,11 @@ public void shouldNotBeAbleToReadLargeAmountsIfSoftLimitSleepWillBeMoreThanConfi assertThatThrownBy(() -> readOneBatchOfSize(200)) .isInstanceOf(RateLimitExceededException.class) .hasMessage("Rate limited. Available capacity has been exhausted."); + + // TODO(hsaraogi): This should not happen. + assertThatThrownBy(() -> readOneBatchOfSize(1)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); } @Test diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java index 4c4fafaffdc..c2ee8906413 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java @@ -76,6 +76,11 @@ public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConf assertThatThrownBy(() -> writeNTodosOfSize(200, 10_000)) .isInstanceOf(RateLimitExceededException.class) .hasMessage("Rate limited. Available capacity has been exhausted."); + + // One write smaller than the rate limit should also be rate limited. + assertThatThrownBy(() -> writeNTodosOfSize(5, 10)) + .isInstanceOf(RateLimitExceededException.class) + .hasMessage("Rate limited. Available capacity has been exhausted."); } @Test From dd18730a578bb4ec99c802a6b0649d391bad2f6f Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 23 Nov 2017 14:51:10 +0000 Subject: [PATCH 66/67] annoying no new linw at eof --- .../src/test/resources/cassandra-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml b/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml index 60e6f7e9079..be21170b800 100644 --- a/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml +++ b/atlasdb-ete-tests/src/test/resources/cassandra-docker-compose.yml @@ -11,4 +11,4 @@ services: - MAX_HEAP_SIZE=512m - HEAP_NEWSIZE=64m - LOCAL_JMX=no - - CASSANDRA_VERSION=2.2.8 \ No newline at end of file + - CASSANDRA_VERSION=2.2.8 From be0759c879e00171cc38a301d592b366011c7d69 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 23 Nov 2017 19:24:23 +0000 Subject: [PATCH 67/67] Uniform parameters for hard limiting --- .../java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java index c2ee8906413..cf965c249a1 100644 --- a/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java +++ b/atlasdb-ete-tests/src/test/java/com/palantir/atlasdb/ete/QosCassandraWriteEteTest.java @@ -73,7 +73,7 @@ public void shouldNotBeAbleToWriteLargeAmountsIfSoftLimitSleepWillBeMoreThanConf // will let anything pass through until the limit is exceeded. writeNTodosOfSize(1, 100_000); - assertThatThrownBy(() -> writeNTodosOfSize(200, 10_000)) + assertThatThrownBy(() -> writeNTodosOfSize(1, 100_000)) .isInstanceOf(RateLimitExceededException.class) .hasMessage("Rate limited. Available capacity has been exhausted.");