From 60e359308d8faf8dfcb97517c50865cdca1e4023 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Mon, 6 Nov 2017 17:30:09 +0000 Subject: [PATCH 01/15] Metrics for bytes and counts in each read/write --- .../keyvalue/cassandra/CassandraClient.java | 42 +++++++++++++- .../keyvalue/cassandra/QosMetrics.java | 57 +++++++++++++++++++ .../palantir/atlasdb/util/MetricsManager.java | 15 +++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index ba3b05045fa..7d05e6fc44b 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -35,6 +35,7 @@ import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.TimedOutException; import org.apache.cassandra.thrift.UnavailableException; +import org.apache.commons.lang.SerializationUtils; import org.apache.thrift.TException; import com.palantir.atlasdb.qos.AtlasDbQosClient; @@ -48,11 +49,13 @@ public class CassandraClient extends AutoDelegate_Client { private final Cassandra.Client delegate; private final AtlasDbQosClient qosClient; + private final QosMetrics qosMetrics; public CassandraClient(Cassandra.Client delegate, AtlasDbQosClient qosClient) { super(delegate.getInputProtocol()); this.delegate = delegate; this.qosClient = qosClient; + this.qosMetrics = new QosMetrics(); } @Override @@ -65,7 +68,19 @@ public Map> multiget_slice(List> result = delegate.multiget_slice(keys, column_parent, + predicate, consistency_level); + qosMetrics.updateReadCount(); + qosMetrics.updateBytesRead(getApproximateReadByteCount(result)); + return result; + } + + private int getApproximateReadByteCount(Map> result) { + return result.entrySet().stream() + .mapToInt((entry) -> entry.getKey().array().length + entry.getValue().stream() + .mapToInt(columnOrSuperColumn -> SerializationUtils.serialize(columnOrSuperColumn).length) + .sum()) + .sum(); } @Override @@ -74,6 +89,24 @@ public void batch_mutate(Map>> mutation_m throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); delegate.batch_mutate(mutation_map, consistency_level); + qosMetrics.updateWriteCount(); + qosMetrics.updateBytesWritten(getApproximateWriteByteCount(mutation_map)); + } + + private int getApproximateWriteByteCount(Map>> mutation_map) { + int approxBytesForKeys = mutation_map.keySet().stream().mapToInt(e -> e.array().length).sum(); + int approxBytesForValues = mutation_map.values().stream() + .mapToInt(singleMap -> { + int approximateBytesInStrings = singleMap.keySet().stream().mapToInt(String::length).sum(); + int approximateBytesInMutations = singleMap.values().stream() + .mapToInt(listOfMutations -> listOfMutations.stream() + .mapToInt(mutation -> SerializationUtils.serialize(mutation).length) + .sum()) + .sum(); + return approximateBytesInStrings + approximateBytesInMutations; + }) + .sum(); + return approxBytesForKeys + approxBytesForValues; } @Override @@ -89,6 +122,11 @@ public List get_range_slices(ColumnParent column_parent, SlicePredicat ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); - return delegate.get_range_slices(column_parent, predicate, range, consistency_level); + List result = super.get_range_slices(column_parent, predicate, range, consistency_level); + int approximateBytesRead = result.stream() + .mapToInt(keySlice -> SerializationUtils.serialize(keySlice).length) + .sum(); + qosMetrics.updateBytesRead(approximateBytesRead); + return result; } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java new file mode 100644 index 00000000000..397b05fdbb2 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.palantir.atlasdb.util.MetricsManager; + +public class QosMetrics { + + private final MetricsManager metricsManager = new MetricsManager(); + + private final Meter readRequestCount; + private final Meter writeRequestCount; + private final Histogram bytesRead; + private final Histogram bytesWritten; + + public QosMetrics() { + readRequestCount = metricsManager.registerMeter(QosMetrics.class, "numReadRequests"); + writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); + + //TODO: The histogram should have a sliding window reservoir. + bytesRead = metricsManager.registerHistogram(QosMetrics.class, "bytesRead"); + bytesWritten = metricsManager.registerHistogram(QosMetrics.class, "bytesWritten"); + } + + + public void updateReadCount() { + readRequestCount.mark(); + } + + public void updateWriteCount() { + writeRequestCount.mark(); + } + + public void updateBytesRead(int numBytes) { + bytesRead.update(numBytes); + } + + public void updateBytesWritten(int numBytes) { + bytesWritten.update(numBytes); + } +} diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java index ae884890c2d..c75ed948476 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; @@ -65,6 +66,16 @@ public void registerMetric(Class clazz, String metricName, Metric metric) { registerMetricWithFqn(MetricRegistry.name(clazz, metricName), metric); } + public Histogram registerHistogram(Class clazz, String metricName) { + return registerHistogram(MetricRegistry.name(clazz, metricName)); + } + + private Histogram registerHistogram(String fullyQualifiedHistogramName) { + Histogram histogram = metricRegistry.histogram(fullyQualifiedHistogramName); + registeredMetrics.add(fullyQualifiedHistogramName); + return histogram; + } + private synchronized void registerMetricWithFqn(String fullyQualifiedMetricName, Metric metric) { try { metricRegistry.register(fullyQualifiedMetricName, metric); @@ -75,6 +86,10 @@ private synchronized void registerMetricWithFqn(String fullyQualifiedMetricName, } } + public Meter registerMeter(Class clazz, String meterName) { + return registerMeter(MetricRegistry.name(clazz, "", meterName)); + } + public Meter registerMeter(Class clazz, String metricPrefix, String meterName) { return registerMeter(MetricRegistry.name(clazz, metricPrefix, meterName)); } From 9756bccc573e79523b0ba6dabb760b39a685787a Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Tue, 7 Nov 2017 11:31:17 +0000 Subject: [PATCH 02/15] Refactors, dont throw if recordMetrics throws --- .../keyvalue/cassandra/CassandraClient.java | 56 ++++++++++++++++--- .../keyvalue/cassandra/QosMetrics.java | 2 - 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 7d05e6fc44b..ef5c8a860db 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -37,6 +37,8 @@ import org.apache.cassandra.thrift.UnavailableException; import org.apache.commons.lang.SerializationUtils; import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.palantir.atlasdb.qos.AtlasDbQosClient; import com.palantir.processors.AutoDelegate; @@ -47,6 +49,8 @@ @AutoDelegate(typeToExtend = Cassandra.Client.class) @SuppressWarnings({"checkstyle:all", "DuplicateThrows"}) // :'( public class CassandraClient extends AutoDelegate_Client { + private final Logger log = LoggerFactory.getLogger(CassandraClient.class); + private final Cassandra.Client delegate; private final AtlasDbQosClient qosClient; private final QosMetrics qosMetrics; @@ -70,16 +74,27 @@ public Map> multiget_slice(List> result = delegate.multiget_slice(keys, column_parent, predicate, consistency_level); - qosMetrics.updateReadCount(); - qosMetrics.updateBytesRead(getApproximateReadByteCount(result)); + try { + recordBytesRead(getApproximateReadByteCount(result)); + } catch (Exception e) { + log.warn("Encountered an exception when recording write metrics for multiget_slice.", e); + } return result; } private int getApproximateReadByteCount(Map> result) { return result.entrySet().stream() - .mapToInt((entry) -> entry.getKey().array().length + entry.getValue().stream() - .mapToInt(columnOrSuperColumn -> SerializationUtils.serialize(columnOrSuperColumn).length) - .sum()) + .mapToInt((entry) -> getRowKeySize(entry) + totalColumnOrSuperColumnSize(entry)) + .sum(); + } + + private int getRowKeySize(Map.Entry> entry) { + return entry.getKey().array().length; + } + + private int totalColumnOrSuperColumnSize(Map.Entry> entry) { + return entry.getValue().stream() + .mapToInt(columnOrSuperColumn -> SerializationUtils.serialize(columnOrSuperColumn).length) .sum(); } @@ -89,8 +104,11 @@ public void batch_mutate(Map>> mutation_m throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); delegate.batch_mutate(mutation_map, consistency_level); - qosMetrics.updateWriteCount(); - qosMetrics.updateBytesWritten(getApproximateWriteByteCount(mutation_map)); + try { + recordBytesWritten(getApproximateWriteByteCount(mutation_map)); + } catch (Exception e) { + log.warn("Encountered an exception when recording write metrics for batch_mutate.", e); + } } private int getApproximateWriteByteCount(Map>> mutation_map) { @@ -114,7 +132,13 @@ public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, C throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, TException { qosClient.checkLimit(); - return delegate.execute_cql3_query(query, compression, consistency); + CqlResult cqlResult = delegate.execute_cql3_query(query, compression, consistency); + try { + recordBytesRead(SerializationUtils.serialize(cqlResult).length); + } catch (Exception e) { + log.warn("Encountered an exception when recording read metrics for execute_cql3_query.", e); + } + return cqlResult; } @Override @@ -126,7 +150,21 @@ public List get_range_slices(ColumnParent column_parent, SlicePredicat int approximateBytesRead = result.stream() .mapToInt(keySlice -> SerializationUtils.serialize(keySlice).length) .sum(); - qosMetrics.updateBytesRead(approximateBytesRead); + try { + recordBytesRead(approximateBytesRead); + } catch (Exception e) { + log.warn("Encountered an exception when recording read metrics for get_range_slices.", e); + } return result; } + + private void recordBytesRead(int numBytesRead) { + qosMetrics.updateReadCount(); + qosMetrics.updateBytesRead(numBytesRead); + } + + private void recordBytesWritten(int numBytesWitten) { + qosMetrics.updateWriteCount(); + qosMetrics.updateBytesWritten(numBytesWitten); + } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java index 397b05fdbb2..36020587066 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java @@ -21,7 +21,6 @@ import com.palantir.atlasdb.util.MetricsManager; public class QosMetrics { - private final MetricsManager metricsManager = new MetricsManager(); private final Meter readRequestCount; @@ -38,7 +37,6 @@ public QosMetrics() { bytesWritten = metricsManager.registerHistogram(QosMetrics.class, "bytesWritten"); } - public void updateReadCount() { readRequestCount.mark(); } From b54863396a6e6d829e0edb8cb5742528bb28f0a6 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 8 Nov 2017 18:16:06 +0000 Subject: [PATCH 03/15] Use meters instead of histograms --- .../atlasdb/keyvalue/cassandra/QosMetrics.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java index 36020587066..dc1c95ec478 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java @@ -16,7 +16,6 @@ package com.palantir.atlasdb.keyvalue.cassandra; -import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.palantir.atlasdb.util.MetricsManager; @@ -25,16 +24,15 @@ public class QosMetrics { private final Meter readRequestCount; private final Meter writeRequestCount; - private final Histogram bytesRead; - private final Histogram bytesWritten; + private final Meter bytesRead; + private final Meter bytesWritten; public QosMetrics() { readRequestCount = metricsManager.registerMeter(QosMetrics.class, "numReadRequests"); writeRequestCount = metricsManager.registerMeter(QosMetrics.class, "numWriteRequests"); - //TODO: The histogram should have a sliding window reservoir. - bytesRead = metricsManager.registerHistogram(QosMetrics.class, "bytesRead"); - bytesWritten = metricsManager.registerHistogram(QosMetrics.class, "bytesWritten"); + bytesRead = metricsManager.registerMeter(QosMetrics.class, "bytesRead"); + bytesWritten = metricsManager.registerMeter(QosMetrics.class, "bytesWritten"); } public void updateReadCount() { @@ -46,10 +44,10 @@ public void updateWriteCount() { } public void updateBytesRead(int numBytes) { - bytesRead.update(numBytes); + bytesRead.mark(numBytes); } public void updateBytesWritten(int numBytes) { - bytesWritten.update(numBytes); + bytesWritten.mark(numBytes); } } From 89c920a5e49a0d31828823390f13946e26252fd1 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 8 Nov 2017 18:58:07 +0000 Subject: [PATCH 04/15] Multiget bytes --- .../keyvalue/cassandra/CassandraClient.java | 16 ++-- .../keyvalue/cassandra/QosMetrics.java | 4 +- .../cassandra/ThriftObjectSizeUtils.java | 81 +++++++++++++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index ef5c8a860db..290d36e5c8e 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -82,19 +82,19 @@ public Map> multiget_slice(List> result) { + private long getApproximateReadByteCount(Map> result) { return result.entrySet().stream() - .mapToInt((entry) -> getRowKeySize(entry) + totalColumnOrSuperColumnSize(entry)) + .mapToLong((entry) -> getRowKeySize(entry) + totalSizeOfValues(entry)) .sum(); } - private int getRowKeySize(Map.Entry> entry) { - return entry.getKey().array().length; + private long getRowKeySize(Map.Entry> entry) { + return ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()); } - private int totalColumnOrSuperColumnSize(Map.Entry> entry) { + private long totalSizeOfValues(Map.Entry> entry) { return entry.getValue().stream() - .mapToInt(columnOrSuperColumn -> SerializationUtils.serialize(columnOrSuperColumn).length) + .mapToLong(ThriftObjectSizeUtils::getSizeOfColumnOrSuperColumn) .sum(); } @@ -158,12 +158,12 @@ public List get_range_slices(ColumnParent column_parent, SlicePredicat return result; } - private void recordBytesRead(int numBytesRead) { + private void recordBytesRead(long numBytesRead) { qosMetrics.updateReadCount(); qosMetrics.updateBytesRead(numBytesRead); } - private void recordBytesWritten(int numBytesWitten) { + private void recordBytesWritten(long numBytesWitten) { qosMetrics.updateWriteCount(); qosMetrics.updateBytesWritten(numBytesWitten); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java index dc1c95ec478..23b3d9e58a8 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/QosMetrics.java @@ -43,11 +43,11 @@ public void updateWriteCount() { writeRequestCount.mark(); } - public void updateBytesRead(int numBytes) { + public void updateBytesRead(long numBytes) { bytesRead.mark(numBytes); } - public void updateBytesWritten(int numBytes) { + public void updateBytesWritten(long numBytes) { bytesWritten.mark(numBytes); } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java new file mode 100644 index 00000000000..2e337a44a26 --- /dev/null +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -0,0 +1,81 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.cassandra; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CounterColumn; +import org.apache.cassandra.thrift.CounterSuperColumn; +import org.apache.cassandra.thrift.SuperColumn; + +public class ThriftObjectSizeUtils { + + private ThriftObjectSizeUtils() { + // utility class + } + + public static long getSizeOfColumnOrSuperColumn(ColumnOrSuperColumn columnOrSuperColumn) { + return getColumnSize(columnOrSuperColumn.getColumn()) + + getSuperColumnSize(columnOrSuperColumn.getSuper_column()) + + getCounterColumnSize(columnOrSuperColumn.getCounter_column()) + + getCounterSuperColumnSize(columnOrSuperColumn.getCounter_super_column()); + } + + private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { + return getByteArraySize(counterSuperColumn.getName()) + + counterSuperColumn.getColumns().stream() + .mapToLong(ThriftObjectSizeUtils::getCounterColumnSize) + .sum(); + } + + public static long getCounterColumnSize(CounterColumn counterColumn) { + return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); + } + + public static long getSuperColumnSize(SuperColumn superColumn) { + return getByteBufferSize(superColumn.name) + + superColumn.getColumns().stream() + .mapToLong(ThriftObjectSizeUtils::getColumnSize) + .sum(); + } + + public static long getByteBufferSize(ByteBuffer byteBuffer) { + return getByteArraySize(byteBuffer.array()); + } + + public static long getByteArraySize(byte[] byteArray) { + return byteArray.length; + } + + public static long getColumnSize(Column column) { + return column.getValue().length + column.getName().length + getTtlSize() + getTimestampSize(); + } + + public static long getTimestampSize() { + return Long.BYTES; + } + + public static long getTtlSize() { + return Integer.BYTES; + } + + public static long getCounterValueSize() { + return Long.SIZE; + } +} From 06613a2d650896e568426219d5ea2be38d8cc42b Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 8 Nov 2017 19:41:41 +0000 Subject: [PATCH 05/15] Batch mutate exact size --- .../keyvalue/cassandra/CassandraClient.java | 55 +++++++++---------- .../cassandra/ThriftObjectSizeUtils.java | 49 ++++++++++++++++- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 290d36e5c8e..54fafd6c3c2 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -17,8 +17,10 @@ package com.palantir.atlasdb.keyvalue.cassandra; import java.nio.ByteBuffer; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Function; import org.apache.cassandra.thrift.AutoDelegate_Client; import org.apache.cassandra.thrift.Cassandra; @@ -83,50 +85,45 @@ public Map> multiget_slice(List> result) { - return result.entrySet().stream() - .mapToLong((entry) -> getRowKeySize(entry) + totalSizeOfValues(entry)) - .sum(); - } - - private long getRowKeySize(Map.Entry> entry) { - return ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()); - } - - private long totalSizeOfValues(Map.Entry> entry) { - return entry.getValue().stream() - .mapToLong(ThriftObjectSizeUtils::getSizeOfColumnOrSuperColumn) - .sum(); + return getCollectionSize(result.entrySet(), + rowResult -> + ThriftObjectSizeUtils.getByteBufferSize(rowResult.getKey()) + + getCollectionSize(rowResult.getValue(), ThriftObjectSizeUtils::getColumnOrSuperColumnSize)); } @Override - public void batch_mutate(Map>> mutation_map, + public void batch_mutate(Map>> mutationMap, ConsistencyLevel consistency_level) throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); - delegate.batch_mutate(mutation_map, consistency_level); + delegate.batch_mutate(mutationMap, consistency_level); try { - recordBytesWritten(getApproximateWriteByteCount(mutation_map)); + recordBytesWritten(getApproximateWriteByteCount(mutationMap)); } catch (Exception e) { log.warn("Encountered an exception when recording write metrics for batch_mutate.", e); } } - private int getApproximateWriteByteCount(Map>> mutation_map) { - int approxBytesForKeys = mutation_map.keySet().stream().mapToInt(e -> e.array().length).sum(); - int approxBytesForValues = mutation_map.values().stream() - .mapToInt(singleMap -> { - int approximateBytesInStrings = singleMap.keySet().stream().mapToInt(String::length).sum(); - int approximateBytesInMutations = singleMap.values().stream() - .mapToInt(listOfMutations -> listOfMutations.stream() - .mapToInt(mutation -> SerializationUtils.serialize(mutation).length) - .sum()) - .sum(); - return approximateBytesInStrings + approximateBytesInMutations; - }) - .sum(); + private long getApproximateWriteByteCount(Map>> mutationResultMap) { + long approxBytesForKeys = getCollectionSize(mutationResultMap.keySet(), + ThriftObjectSizeUtils::getByteBufferSize); + long approxBytesForValues = getCollectionSize(mutationResultMap.values(), currentMap -> + getCollectionSize(currentMap.keySet(), + this::getStringSize) + + getCollectionSize(currentMap.values(), + mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); return approxBytesForKeys + approxBytesForValues; } + + private long getCollectionSize(Collection collection, Function sizeFunction) { + return collection.stream().mapToLong(sizeFunction::apply).sum(); + } + + private long getStringSize(String string) { + return string.length() * Character.SIZE; + } + @Override public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 2e337a44a26..e631efad676 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -17,20 +17,27 @@ package com.palantir.atlasdb.keyvalue.cassandra; import java.nio.ByteBuffer; +import java.util.Collection; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; import org.apache.cassandra.thrift.CounterColumn; import org.apache.cassandra.thrift.CounterSuperColumn; +import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SliceRange; import org.apache.cassandra.thrift.SuperColumn; public class ThriftObjectSizeUtils { + private static final long ONE_BYTE = 1L; + private ThriftObjectSizeUtils() { // utility class } - public static long getSizeOfColumnOrSuperColumn(ColumnOrSuperColumn columnOrSuperColumn) { + public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { return getColumnSize(columnOrSuperColumn.getColumn()) + getSuperColumnSize(columnOrSuperColumn.getSuper_column()) + getCounterColumnSize(columnOrSuperColumn.getCounter_column()) @@ -56,7 +63,8 @@ public static long getSuperColumnSize(SuperColumn superColumn) { } public static long getByteBufferSize(ByteBuffer byteBuffer) { - return getByteArraySize(byteBuffer.array()); + // Position is the size unless something has been read from the ByteBuffer + return byteBuffer.position(); } public static long getByteArraySize(byte[] byteArray) { @@ -76,6 +84,41 @@ public static long getTtlSize() { } public static long getCounterValueSize() { - return Long.SIZE; + return Long.BYTES; + } + + public static long getMutationSize(Mutation mutation) { + return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize(mutation.getDeletion()); + } + + public static long getDeletionSize(Deletion deletion) { + return getTimestampSize() + + getByteArraySize(deletion.getSuper_column()) + + getSlicePredicateSize(deletion.getPredicate()); + } + + private static long getSlicePredicateSize(SlicePredicate predicate) { + return getByteBufferCollectionSize(predicate.getColumn_names()) + getSliceRangeSize(predicate.getSlice_range()); + } + + private static long getSliceRangeSize(SliceRange sliceRange) { + return getByteArraySize(sliceRange.getStart()) + + getByteArraySize(sliceRange.getFinish()) + + getReversedBooleanSize() + + getSliceRangeCountSize(); + } + + private static long getReversedBooleanSize() { + return ONE_BYTE; + } + + private static int getSliceRangeCountSize() { + return Integer.BYTES; + } + + public static long getByteBufferCollectionSize(Collection byteBufferList) { + return byteBufferList.stream() + .mapToLong(ThriftObjectSizeUtils::getByteBufferSize) + .sum(); } } From 2f593826d6305e863627abd645a5dbd1c967929d Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 8 Nov 2017 20:09:35 +0000 Subject: [PATCH 06/15] Cqlresult size --- .../keyvalue/cassandra/CassandraClient.java | 16 ++--- .../cassandra/ThriftObjectSizeUtils.java | 64 +++++++++++++++---- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 54fafd6c3c2..7f94a666027 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -104,12 +104,10 @@ public void batch_mutate(Map>> mutationMa } } - private long getApproximateWriteByteCount(Map>> mutationResultMap) { - long approxBytesForKeys = getCollectionSize(mutationResultMap.keySet(), - ThriftObjectSizeUtils::getByteBufferSize); - long approxBytesForValues = getCollectionSize(mutationResultMap.values(), currentMap -> - getCollectionSize(currentMap.keySet(), - this::getStringSize) + private long getApproximateWriteByteCount(Map>> batchMutateMap) { + long approxBytesForKeys = getCollectionSize(batchMutateMap.keySet(), ThriftObjectSizeUtils::getByteBufferSize); + long approxBytesForValues = getCollectionSize(batchMutateMap.values(), currentMap -> + getCollectionSize(currentMap.keySet(), ThriftObjectSizeUtils::getStringSize) + getCollectionSize(currentMap.values(), mutations -> getCollectionSize(mutations, ThriftObjectSizeUtils::getMutationSize))); return approxBytesForKeys + approxBytesForValues; @@ -120,10 +118,6 @@ private long getCollectionSize(Collection collection, Function s return collection.stream().mapToLong(sizeFunction::apply).sum(); } - private long getStringSize(String string) { - return string.length() * Character.SIZE; - } - @Override public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, @@ -131,7 +125,7 @@ public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, C qosClient.checkLimit(); CqlResult cqlResult = delegate.execute_cql3_query(query, compression, consistency); try { - recordBytesRead(SerializationUtils.serialize(cqlResult).length); + recordBytesRead(ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); } catch (Exception e) { log.warn("Encountered an exception when recording read metrics for execute_cql3_query.", e); } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index e631efad676..5cf9ececbfb 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -18,18 +18,23 @@ import java.nio.ByteBuffer; import java.util.Collection; +import java.util.Map; +import java.util.function.Function; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; import org.apache.cassandra.thrift.CounterColumn; import org.apache.cassandra.thrift.CounterSuperColumn; +import org.apache.cassandra.thrift.CqlMetadata; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlRow; import org.apache.cassandra.thrift.Deletion; import org.apache.cassandra.thrift.Mutation; import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.SliceRange; import org.apache.cassandra.thrift.SuperColumn; -public class ThriftObjectSizeUtils { +public final class ThriftObjectSizeUtils { private static final long ONE_BYTE = 1L; @@ -45,10 +50,8 @@ public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperC } private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { - return getByteArraySize(counterSuperColumn.getName()) + - counterSuperColumn.getColumns().stream() - .mapToLong(ThriftObjectSizeUtils::getCounterColumnSize) - .sum(); + return getByteArraySize(counterSuperColumn.getName()) + + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); } public static long getCounterColumnSize(CounterColumn counterColumn) { @@ -56,10 +59,8 @@ public static long getCounterColumnSize(CounterColumn counterColumn) { } public static long getSuperColumnSize(SuperColumn superColumn) { - return getByteBufferSize(superColumn.name) + - superColumn.getColumns().stream() - .mapToLong(ThriftObjectSizeUtils::getColumnSize) - .sum(); + return getByteBufferSize(superColumn.name) + + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); } public static long getByteBufferSize(ByteBuffer byteBuffer) { @@ -98,7 +99,7 @@ public static long getDeletionSize(Deletion deletion) { } private static long getSlicePredicateSize(SlicePredicate predicate) { - return getByteBufferCollectionSize(predicate.getColumn_names()) + getSliceRangeSize(predicate.getSlice_range()); + return getCollectionSize(predicate.getColumn_names(), ThriftObjectSizeUtils::getByteBufferSize) + getSliceRangeSize(predicate.getSlice_range()); } private static long getSliceRangeSize(SliceRange sliceRange) { @@ -116,9 +117,44 @@ private static int getSliceRangeCountSize() { return Integer.BYTES; } - public static long getByteBufferCollectionSize(Collection byteBufferList) { - return byteBufferList.stream() - .mapToLong(ThriftObjectSizeUtils::getByteBufferSize) - .sum(); + + public static long getCqlResultSize(CqlResult cqlResult) { + return getThriftEnumSize() + + getCollectionSize(cqlResult.getRows(), ThriftObjectSizeUtils::getCqlRowSize) + + Integer.BYTES + + getCqlMetadataSize(cqlResult.getSchema()); + } + + private static long getCqlMetadataSize(CqlMetadata schema) { + if (schema == null) { + return Integer.BYTES; + } + return getByteBufferStringMapSize(schema.getName_types()) + + getByteBufferStringMapSize(schema.getValue_types()) + + getStringSize(schema.getDefault_name_type()) + + getStringSize(schema.getDefault_value_type()); + } + + private static long getByteBufferStringMapSize(Map nameTypes) { + return getCollectionSize(nameTypes.entrySet(), + entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + + ThriftObjectSizeUtils.getStringSize(entry.getValue())); + } + + private static Long getCqlRowSize(CqlRow cqlRow) { + return getByteArraySize(cqlRow.getKey()) + + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); + } + + public static long getThriftEnumSize() { + return Integer.BYTES; + } + + private static long getCollectionSize(Collection collection, Function sizeFunction) { + return collection.stream().mapToLong(sizeFunction::apply).sum(); + } + + public static long getStringSize(String string) { + return string.length() * Character.SIZE; } } From a94c331796f22dd21d5b2487a9c7e498b6f06eb9 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Wed, 8 Nov 2017 20:25:58 +0000 Subject: [PATCH 07/15] Calculate exact byte sizes for all thrift objects --- .../keyvalue/cassandra/CassandraClient.java | 15 +- .../cassandra/ThriftObjectSizeUtils.java | 154 ++++++++++++------ 2 files changed, 112 insertions(+), 57 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 7f94a666027..2574dec2b6e 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -37,7 +37,6 @@ import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.TimedOutException; import org.apache.cassandra.thrift.UnavailableException; -import org.apache.commons.lang.SerializationUtils; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,11 +112,6 @@ private long getApproximateWriteByteCount(Map long getCollectionSize(Collection collection, Function sizeFunction) { - return collection.stream().mapToLong(sizeFunction::apply).sum(); - } - @Override public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, ConsistencyLevel consistency) throws InvalidRequestException, UnavailableException, TimedOutException, SchemaDisagreementException, @@ -138,11 +132,8 @@ public List get_range_slices(ColumnParent column_parent, SlicePredicat throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); List result = super.get_range_slices(column_parent, predicate, range, consistency_level); - int approximateBytesRead = result.stream() - .mapToInt(keySlice -> SerializationUtils.serialize(keySlice).length) - .sum(); try { - recordBytesRead(approximateBytesRead); + recordBytesRead(getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); } catch (Exception e) { log.warn("Encountered an exception when recording read metrics for get_range_slices.", e); } @@ -158,4 +149,8 @@ private void recordBytesWritten(long numBytesWitten) { qosMetrics.updateWriteCount(); qosMetrics.updateBytesWritten(numBytesWitten); } + + private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { + return collection.stream().mapToLong(singleObjectSizeFunction::apply).sum(); + } } diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 5cf9ececbfb..59b58b0350c 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -29,6 +29,7 @@ import org.apache.cassandra.thrift.CqlResult; import org.apache.cassandra.thrift.CqlRow; import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.KeySlice; import org.apache.cassandra.thrift.Mutation; import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.SliceRange; @@ -49,86 +50,117 @@ public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperC + getCounterSuperColumnSize(columnOrSuperColumn.getCounter_super_column()); } - private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { - return getByteArraySize(counterSuperColumn.getName()) - + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); - } + public static long getByteBufferSize(ByteBuffer byteBuffer) { + if (byteBuffer == null) { + getNullSize(); + } - public static long getCounterColumnSize(CounterColumn counterColumn) { - return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); + return byteBuffer.position(); } - public static long getSuperColumnSize(SuperColumn superColumn) { - return getByteBufferSize(superColumn.name) - + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); + public static long getMutationSize(Mutation mutation) { + if (mutation == null) { + return getNullSize(); + } + return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize(mutation.getDeletion()); } - public static long getByteBufferSize(ByteBuffer byteBuffer) { - // Position is the size unless something has been read from the ByteBuffer - return byteBuffer.position(); + public static long getCqlResultSize(CqlResult cqlResult) { + if (cqlResult == null) { + return getNullSize(); + } + return getThriftEnumSize() + + getCollectionSize(cqlResult.getRows(), ThriftObjectSizeUtils::getCqlRowSize) + + Integer.BYTES + + getCqlMetadataSize(cqlResult.getSchema()); } - public static long getByteArraySize(byte[] byteArray) { - return byteArray.length; + public static long getKeySliceSize(KeySlice keySlice) { + if (keySlice == null) { + return getNullSize(); + } + + return getByteArraySize(keySlice.getKey()) + + getCollectionSize(keySlice.getColumns(), ThriftObjectSizeUtils::getColumnOrSuperColumnSize); } - public static long getColumnSize(Column column) { - return column.getValue().length + column.getName().length + getTtlSize() + getTimestampSize(); + public static long getStringSize(String string) { + if (string == null) { + return getNullSize(); + } + + return string.length() * Character.SIZE; } - public static long getTimestampSize() { - return Long.BYTES; + private static long getCounterSuperColumnSize(CounterSuperColumn counterSuperColumn) { + if (counterSuperColumn == null) { + return getNullSize(); + } + + return getByteArraySize(counterSuperColumn.getName()) + + getCollectionSize(counterSuperColumn.getColumns(), ThriftObjectSizeUtils::getCounterColumnSize); } - public static long getTtlSize() { - return Integer.BYTES; + private static long getCounterColumnSize(CounterColumn counterColumn) { + if (counterColumn == null) { + return getNullSize(); + } + + return getByteArraySize(counterColumn.getName()) + getCounterValueSize(); } - public static long getCounterValueSize() { - return Long.BYTES; + private static long getSuperColumnSize(SuperColumn superColumn) { + if (superColumn == null) { + return getNullSize(); + } + + return getByteBufferSize(superColumn.name) + + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - public static long getMutationSize(Mutation mutation) { - return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize(mutation.getDeletion()); + + private static long getColumnSize(Column column) { + if (column == null) { + return getNullSize(); + } + + return column.getValue().length + column.getName().length + getTtlSize() + getTimestampSize(); } - public static long getDeletionSize(Deletion deletion) { + private static long getDeletionSize(Deletion deletion) { + if (deletion == null) { + return getNullSize(); + } + return getTimestampSize() + getByteArraySize(deletion.getSuper_column()) + getSlicePredicateSize(deletion.getPredicate()); } private static long getSlicePredicateSize(SlicePredicate predicate) { + if (predicate == null) { + return getNullSize(); + } + return getCollectionSize(predicate.getColumn_names(), ThriftObjectSizeUtils::getByteBufferSize) + getSliceRangeSize(predicate.getSlice_range()); } private static long getSliceRangeSize(SliceRange sliceRange) { + if (sliceRange == null) { + return getNullSize(); + } + return getByteArraySize(sliceRange.getStart()) + getByteArraySize(sliceRange.getFinish()) + getReversedBooleanSize() + getSliceRangeCountSize(); } - private static long getReversedBooleanSize() { - return ONE_BYTE; - } - - private static int getSliceRangeCountSize() { - return Integer.BYTES; - } - - - public static long getCqlResultSize(CqlResult cqlResult) { - return getThriftEnumSize() - + getCollectionSize(cqlResult.getRows(), ThriftObjectSizeUtils::getCqlRowSize) - + Integer.BYTES - + getCqlMetadataSize(cqlResult.getSchema()); - } - private static long getCqlMetadataSize(CqlMetadata schema) { if (schema == null) { - return Integer.BYTES; + return getNullSize(); } + return getByteBufferStringMapSize(schema.getName_types()) + getByteBufferStringMapSize(schema.getValue_types()) + getStringSize(schema.getDefault_name_type()) @@ -142,19 +174,47 @@ private static long getByteBufferStringMapSize(Map nameTypes } private static Long getCqlRowSize(CqlRow cqlRow) { + if (cqlRow == null) { + return getNullSize(); + } + return getByteArraySize(cqlRow.getKey()) + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); } - public static long getThriftEnumSize() { + private static long getThriftEnumSize() { return Integer.BYTES; } - private static long getCollectionSize(Collection collection, Function sizeFunction) { - return collection.stream().mapToLong(sizeFunction::apply).sum(); + private static long getByteArraySize(byte[] byteArray) { + return byteArray.length; } - public static long getStringSize(String string) { - return string.length() * Character.SIZE; + private static long getTimestampSize() { + return Long.BYTES; + } + + private static long getTtlSize() { + return Integer.BYTES; + } + + private static long getCounterValueSize() { + return Long.BYTES; + } + + private static long getReversedBooleanSize() { + return ONE_BYTE; + } + + private static long getSliceRangeCountSize() { + return Integer.BYTES; + } + + private static long getNullSize() { + return Integer.BYTES; + } + + private static long getCollectionSize(Collection collection, Function sizeFunction) { + return collection.stream().mapToLong(sizeFunction::apply).sum(); } } From 9cfbb967d1faf3b089cc0bd9207fa09fd20b0923 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 9 Nov 2017 12:04:12 +0000 Subject: [PATCH 08/15] tests and bugfixes - partial --- .../keyvalue/cassandra/CassandraClient.java | 4 +- .../cassandra/ThriftObjectSizeUtils.java | 15 +++- .../atlasdb/ThriftObjectSizeUtilsTest.java | 75 +++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 2574dec2b6e..65c3f3b1e18 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -145,9 +145,9 @@ private void recordBytesRead(long numBytesRead) { qosMetrics.updateBytesRead(numBytesRead); } - private void recordBytesWritten(long numBytesWitten) { + private void recordBytesWritten(long numBytesWritten) { qosMetrics.updateWriteCount(); - qosMetrics.updateBytesWritten(numBytesWitten); + qosMetrics.updateBytesWritten(numBytesWritten); } private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 59b58b0350c..234efbb13d9 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -44,6 +44,9 @@ private ThriftObjectSizeUtils() { } public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperColumn) { + if (columnOrSuperColumn == null) { + return getNullSize(); + } return getColumnSize(columnOrSuperColumn.getColumn()) + getSuperColumnSize(columnOrSuperColumn.getSuper_column()) + getCounterColumnSize(columnOrSuperColumn.getCounter_column()) @@ -62,6 +65,7 @@ public static long getMutationSize(Mutation mutation) { if (mutation == null) { return getNullSize(); } + return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize(mutation.getDeletion()); } @@ -114,7 +118,7 @@ private static long getSuperColumnSize(SuperColumn superColumn) { return getNullSize(); } - return getByteBufferSize(superColumn.name) + return getByteArraySize(superColumn.getName()) + getCollectionSize(superColumn.getColumns(), ThriftObjectSizeUtils::getColumnSize); } @@ -124,7 +128,10 @@ private static long getColumnSize(Column column) { return getNullSize(); } - return column.getValue().length + column.getName().length + getTtlSize() + getTimestampSize(); + return getByteArraySize(column.getValue()) + + getByteArraySize(column.getName()) + + getTtlSize() + + getTimestampSize(); } private static long getDeletionSize(Deletion deletion) { @@ -177,7 +184,6 @@ private static Long getCqlRowSize(CqlRow cqlRow) { if (cqlRow == null) { return getNullSize(); } - return getByteArraySize(cqlRow.getKey()) + getCollectionSize(cqlRow.getColumns(), ThriftObjectSizeUtils::getColumnSize); } @@ -187,6 +193,9 @@ private static long getThriftEnumSize() { } private static long getByteArraySize(byte[] byteArray) { + if (byteArray == null) { + return getNullSize(); + } return byteArray.length; } diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java new file mode 100644 index 00000000000..bf3d5cb6ca3 --- /dev/null +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017 Palantir Technologies, Inc. All rights reserved. + * + * Licensed under the BSD-3 License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb; + +import static org.assertj.core.api.Java6Assertions.assertThat; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.SuperColumn; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.palantir.atlasdb.keyvalue.cassandra.ThriftObjectSizeUtils; + +public class ThriftObjectSizeUtilsTest { + + private static final String TEST_MAME = "test"; + private static final Column TEST_COLUMN = new Column(ByteBuffer.wrap(TEST_MAME.getBytes())); + + + private static final long TEST_COLUMN_SIZE = 4L + TEST_MAME.getBytes().length + 4L + 8L; + + + @Test + public void returnEightForNullColumnOrSuperColumn() throws Exception { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyColumnOrSuperColumn() throws Exception { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn())).isEqualTo( + Integer.BYTES * 4); + } + + @Test + public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() throws Exception { + ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); + columnOrSuperColumn.setColumn(new Column()); + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(columnOrSuperColumn)).isEqualTo( + Integer.BYTES * 8); + } + + @Test + public void getSizeForColumnOrSuperColumnWithANonEmptyColumn() throws Exception { + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn().setColumn(TEST_COLUMN))) + .isEqualTo(Integer.BYTES * 3 + TEST_COLUMN_SIZE); + } + + @Test + public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() throws Exception { + ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); + columnOrSuperColumn.setColumn(TEST_COLUMN); + columnOrSuperColumn.setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), ImmutableList.of(TEST_COLUMN))); + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(columnOrSuperColumn)).isEqualTo( + Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); + } + + +} From dd73344534b8a10f3f45076ea36240e4498e3c84 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Thu, 9 Nov 2017 15:04:20 +0000 Subject: [PATCH 09/15] More tests and bugs fixed --- .../cassandra/ThriftObjectSizeUtils.java | 3 ++ .../atlasdb/ThriftObjectSizeUtilsTest.java | 29 +++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 234efbb13d9..8dda73e1bd1 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -224,6 +224,9 @@ private static long getNullSize() { } private static long getCollectionSize(Collection collection, Function sizeFunction) { + if (collection == null) { + return getNullSize(); + } return collection.stream().mapToLong(sizeFunction::apply).sum(); } } diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index bf3d5cb6ca3..d094d34b8c4 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -22,6 +22,9 @@ import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.CqlResultType; +import org.apache.cassandra.thrift.CqlRow; import org.apache.cassandra.thrift.SuperColumn; import org.junit.Test; @@ -38,18 +41,18 @@ public class ThriftObjectSizeUtilsTest { @Test - public void returnEightForNullColumnOrSuperColumn() throws Exception { + public void returnEightForNullColumnOrSuperColumn() { assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(null)).isEqualTo(Integer.BYTES); } @Test - public void getSizeForEmptyColumnOrSuperColumn() throws Exception { + public void getSizeForEmptyColumnOrSuperColumn() { assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn())).isEqualTo( Integer.BYTES * 4); } @Test - public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() throws Exception { + public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() { ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); columnOrSuperColumn.setColumn(new Column()); assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(columnOrSuperColumn)).isEqualTo( @@ -57,13 +60,13 @@ public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() throws Exception { } @Test - public void getSizeForColumnOrSuperColumnWithANonEmptyColumn() throws Exception { + public void getSizeForColumnOrSuperColumnWithANonEmptyColumn() { assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn().setColumn(TEST_COLUMN))) .isEqualTo(Integer.BYTES * 3 + TEST_COLUMN_SIZE); } @Test - public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() throws Exception { + public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() { ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); columnOrSuperColumn.setColumn(TEST_COLUMN); columnOrSuperColumn.setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), ImmutableList.of(TEST_COLUMN))); @@ -71,5 +74,21 @@ public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() thr Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); } + @Test + public void getSizeForNullCqlResult() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize(null)).isEqualTo(Integer.BYTES); + } + @Test + public void getSizeForVoidCqlResult() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize(new CqlResult(CqlResultType.VOID))) + .isEqualTo(Integer.BYTES * 4); + } + + @Test + public void getSizeForCqlResultWithRows() { + assertThat(ThriftObjectSizeUtils.getCqlResultSize( + new CqlResult(CqlResultType.ROWS).setRows(ImmutableList.of(new CqlRow())))) + .isEqualTo(Integer.BYTES * 5); + } } From de9dc41361dd114568c1ffdba54bb3290dda2753 Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 10 Nov 2017 18:24:10 +0000 Subject: [PATCH 10/15] More tests and cr comments --- .../keyvalue/cassandra/CassandraClient.java | 40 +++--- .../cassandra/ThriftObjectSizeUtils.java | 2 +- .../atlasdb/ThriftObjectSizeUtilsTest.java | 116 ++++++++++++++++-- 3 files changed, 120 insertions(+), 38 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java index 65c3f3b1e18..a83a1739751 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClient.java @@ -75,11 +75,7 @@ public Map> multiget_slice(List> result = delegate.multiget_slice(keys, column_parent, predicate, consistency_level); - try { - recordBytesRead(getApproximateReadByteCount(result)); - } catch (Exception e) { - log.warn("Encountered an exception when recording write metrics for multiget_slice.", e); - } + recordBytesRead(getApproximateReadByteCount(result)); return result; } @@ -96,11 +92,7 @@ public void batch_mutate(Map>> mutationMa throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); delegate.batch_mutate(mutationMap, consistency_level); - try { - recordBytesWritten(getApproximateWriteByteCount(mutationMap)); - } catch (Exception e) { - log.warn("Encountered an exception when recording write metrics for batch_mutate.", e); - } + recordBytesWritten(getApproximateWriteByteCount(mutationMap)); } private long getApproximateWriteByteCount(Map>> batchMutateMap) { @@ -118,11 +110,7 @@ public CqlResult execute_cql3_query(ByteBuffer query, Compression compression, C TException { qosClient.checkLimit(); CqlResult cqlResult = delegate.execute_cql3_query(query, compression, consistency); - try { - recordBytesRead(ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); - } catch (Exception e) { - log.warn("Encountered an exception when recording read metrics for execute_cql3_query.", e); - } + recordBytesRead(ThriftObjectSizeUtils.getCqlResultSize(cqlResult)); return cqlResult; } @@ -132,22 +120,26 @@ public List get_range_slices(ColumnParent column_parent, SlicePredicat throws InvalidRequestException, UnavailableException, TimedOutException, TException { qosClient.checkLimit(); List result = super.get_range_slices(column_parent, predicate, range, consistency_level); - try { - recordBytesRead(getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); - } catch (Exception e) { - log.warn("Encountered an exception when recording read metrics for get_range_slices.", e); - } + recordBytesRead(getCollectionSize(result, ThriftObjectSizeUtils::getKeySliceSize)); return result; } private void recordBytesRead(long numBytesRead) { - qosMetrics.updateReadCount(); - qosMetrics.updateBytesRead(numBytesRead); + try { + qosMetrics.updateReadCount(); + qosMetrics.updateBytesRead(numBytesRead); + } catch (Exception e) { + log.warn("Encountered an exception when recording read metrics.", e); + } } private void recordBytesWritten(long numBytesWritten) { - qosMetrics.updateWriteCount(); - qosMetrics.updateBytesWritten(numBytesWritten); + try { + qosMetrics.updateWriteCount(); + qosMetrics.updateBytesWritten(numBytesWritten); + } catch (Exception e) { + log.warn("Encountered an exception when recording write metrics.", e); + } } private long getCollectionSize(Collection collection, Function singleObjectSizeFunction) { diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 8dda73e1bd1..6ed3c2df1ec 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -58,7 +58,7 @@ public static long getByteBufferSize(ByteBuffer byteBuffer) { getNullSize(); } - return byteBuffer.position(); + return byteBuffer.remaining(); } public static long getMutationSize(Mutation mutation) { diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index d094d34b8c4..90670d70554 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -16,7 +16,7 @@ package com.palantir.atlasdb; -import static org.assertj.core.api.Java6Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThat; import java.nio.ByteBuffer; @@ -25,6 +25,10 @@ import org.apache.cassandra.thrift.CqlResult; import org.apache.cassandra.thrift.CqlResultType; import org.apache.cassandra.thrift.CqlRow; +import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.KeySlice; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.SuperColumn; import org.junit.Test; @@ -38,7 +42,8 @@ public class ThriftObjectSizeUtilsTest { private static final long TEST_COLUMN_SIZE = 4L + TEST_MAME.getBytes().length + 4L + 8L; - + private static final ColumnOrSuperColumn EMPTY_COLUMN_OR_SUPERCOLUMN = new ColumnOrSuperColumn(); + private static final long EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE = Integer.BYTES * 4; @Test public void returnEightForNullColumnOrSuperColumn() { @@ -47,16 +52,14 @@ public void returnEightForNullColumnOrSuperColumn() { @Test public void getSizeForEmptyColumnOrSuperColumn() { - assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn())).isEqualTo( - Integer.BYTES * 4); + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(EMPTY_COLUMN_OR_SUPERCOLUMN)) + .isEqualTo(EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); } @Test public void getSizeForColumnOrSuperColumnWithAnEmptyColumn() { - ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); - columnOrSuperColumn.setColumn(new Column()); - assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(columnOrSuperColumn)).isEqualTo( - Integer.BYTES * 8); + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn().setColumn(new Column()))) + .isEqualTo(Integer.BYTES * 8); } @Test @@ -67,11 +70,11 @@ public void getSizeForColumnOrSuperColumnWithANonEmptyColumn() { @Test public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() { - ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn(); - columnOrSuperColumn.setColumn(TEST_COLUMN); - columnOrSuperColumn.setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), ImmutableList.of(TEST_COLUMN))); - assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(columnOrSuperColumn)).isEqualTo( - Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); + assertThat(ThriftObjectSizeUtils.getColumnOrSuperColumnSize(new ColumnOrSuperColumn() + .setColumn(TEST_COLUMN) + .setSuper_column(new SuperColumn(ByteBuffer.wrap(TEST_MAME.getBytes()), + ImmutableList.of(TEST_COLUMN))))) + .isEqualTo(Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); } @Test @@ -91,4 +94,91 @@ public void getSizeForCqlResultWithRows() { new CqlResult(CqlResultType.ROWS).setRows(ImmutableList.of(new CqlRow())))) .isEqualTo(Integer.BYTES * 5); } + + @Test + public void getSizeForNullMutation() { + assertThat(ThriftObjectSizeUtils.getMutationSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyMutation() { + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation())).isEqualTo(Integer.BYTES * 2); + } + + @Test + public void getSizeForMutationWithColumnOrSuperColumn() { + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setColumn_or_supercolumn(EMPTY_COLUMN_OR_SUPERCOLUMN))) + .isEqualTo(Integer.BYTES + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } + + @Test + public void getSizeForMutationWithEmptyDeletion() { + long emptyDeletionSize = Long.BYTES + 2 * Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion()))) + .isEqualTo(Integer.BYTES + emptyDeletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingSuperColumn() { + long nonEmptyDeletionSize = Long.BYTES + TEST_MAME.getBytes().length + Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion().setSuper_column(TEST_MAME.getBytes())))) + .isEqualTo(Integer.BYTES + nonEmptyDeletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingEmptySlicePredicate() { + long deletionSize = Long.BYTES + Integer.BYTES + Integer.BYTES * 2; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setDeletion(new Deletion().setPredicate(new SlicePredicate())))) + .isEqualTo(Integer.BYTES + deletionSize); + } + + @Test + public void getSizeForMutationWithDeletionContainingNonEmptySlicePredicate() { + long deletionSize = (Long.BYTES) + (Integer.BYTES) + (TEST_MAME.getBytes().length + Integer.BYTES); + assertThat(ThriftObjectSizeUtils. + getMutationSize(new Mutation() + .setDeletion(new Deletion() + .setPredicate(new SlicePredicate() + .setColumn_names(ImmutableList.of(ByteBuffer.wrap(TEST_MAME.getBytes()))))))) + .isEqualTo(Integer.BYTES + deletionSize); + } + + @Test + public void getSizeForMutationWithColumnOrSuperColumnAndDeletion() { + long emptyDeletionSize = Long.BYTES + 2 * Integer.BYTES; + assertThat(ThriftObjectSizeUtils.getMutationSize(new Mutation() + .setColumn_or_supercolumn(EMPTY_COLUMN_OR_SUPERCOLUMN) + .setDeletion(new Deletion()))) + .isEqualTo(EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE + emptyDeletionSize); + } + + @Test + public void getSizeForNullKeySlice() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForKeySliceWithKeyNotSetButColumnsSet() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice() + .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)))) + .isEqualTo(Integer.BYTES + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } + + @Test + public void getSizeForKeySliceWithKeySetSetButColumnsNotSet() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice().setKey(TEST_MAME.getBytes()))) + .isEqualTo(Integer.BYTES + TEST_MAME.getBytes().length); + } + + @Test + public void getSizeForKeySliceWithKeyAndColumns() { + assertThat(ThriftObjectSizeUtils.getKeySliceSize(new KeySlice() + .setKey(TEST_MAME.getBytes()) + .setColumns(ImmutableList.of(EMPTY_COLUMN_OR_SUPERCOLUMN)))) + .isEqualTo(TEST_MAME.getBytes().length + EMPTY_COLUMN_OR_SUPERCOLUMN_SIZE); + } } From 5414df96b945937f3d92a3299cbe1c76ad6c904b Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 10 Nov 2017 18:50:35 +0000 Subject: [PATCH 11/15] byte buffer size --- .../cassandra/ThriftObjectSizeUtils.java | 3 +-- .../atlasdb/ThriftObjectSizeUtilsTest.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 6ed3c2df1ec..8133b5541e3 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -55,9 +55,8 @@ public static long getColumnOrSuperColumnSize(ColumnOrSuperColumn columnOrSuperC public static long getByteBufferSize(ByteBuffer byteBuffer) { if (byteBuffer == null) { - getNullSize(); + return getNullSize(); } - return byteBuffer.remaining(); } diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index 90670d70554..bbfe4f69c3c 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -77,6 +77,23 @@ public void getSizeForColumnOrSuperColumnWithANonEmptyColumnAndSuperColumn() { .isEqualTo(Integer.BYTES * 2 + TEST_COLUMN_SIZE + TEST_MAME.getBytes().length + TEST_COLUMN_SIZE); } + @Test + public void getSizeForNullByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(null)).isEqualTo(Integer.BYTES); + } + + @Test + public void getSizeForEmptyByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(new byte[]{}))) + .isEqualTo(0L); + } + + @Test + public void getSizeForNonEmptyByteBuffer() { + assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(TEST_MAME.getBytes()))) + .isEqualTo(TEST_MAME.getBytes().length); + } + @Test public void getSizeForNullCqlResult() { assertThat(ThriftObjectSizeUtils.getCqlResultSize(null)).isEqualTo(Integer.BYTES); From 2e262fe02915394c3a3d4a20b795ead8073654bd Mon Sep 17 00:00:00 2001 From: Himangi Saraogi Date: Fri, 10 Nov 2017 18:54:04 +0000 Subject: [PATCH 12/15] Remove register histogram --- .../com/palantir/atlasdb/util/MetricsManager.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java index c75ed948476..c60f147d47f 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/util/MetricsManager.java @@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; @@ -66,16 +65,6 @@ public void registerMetric(Class clazz, String metricName, Metric metric) { registerMetricWithFqn(MetricRegistry.name(clazz, metricName), metric); } - public Histogram registerHistogram(Class clazz, String metricName) { - return registerHistogram(MetricRegistry.name(clazz, metricName)); - } - - private Histogram registerHistogram(String fullyQualifiedHistogramName) { - Histogram histogram = metricRegistry.histogram(fullyQualifiedHistogramName); - registeredMetrics.add(fullyQualifiedHistogramName); - return histogram; - } - private synchronized void registerMetricWithFqn(String fullyQualifiedMetricName, Metric metric) { try { metricRegistry.register(fullyQualifiedMetricName, metric); From d642c079a64bf757e0ece55d21b2cbe1f6fb34d4 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Fri, 10 Nov 2017 22:20:04 +0000 Subject: [PATCH 13/15] checkstyle --- .../cassandra/ThriftObjectSizeUtils.java | 17 ++++++++++++----- .../atlasdb/qos/ratelimit/QosRateLimiter.java | 2 +- .../atlasdb/qos/ratelimit/RateLimiter.java | 2 +- .../qos/ratelimit/SmoothRateLimiter.java | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java index 8133b5541e3..322249248bf 100644 --- a/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java +++ b/atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/ThriftObjectSizeUtils.java @@ -65,7 +65,8 @@ public static long getMutationSize(Mutation mutation) { return getNullSize(); } - return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize(mutation.getDeletion()); + return getColumnOrSuperColumnSize(mutation.getColumn_or_supercolumn()) + getDeletionSize( + mutation.getDeletion()); } public static long getCqlResultSize(CqlResult cqlResult) { @@ -148,7 +149,8 @@ private static long getSlicePredicateSize(SlicePredicate predicate) { return getNullSize(); } - return getCollectionSize(predicate.getColumn_names(), ThriftObjectSizeUtils::getByteBufferSize) + getSliceRangeSize(predicate.getSlice_range()); + return getCollectionSize(predicate.getColumn_names(), ThriftObjectSizeUtils::getByteBufferSize) + + getSliceRangeSize(predicate.getSlice_range()); } private static long getSliceRangeSize(SliceRange sliceRange) { @@ -175,8 +177,8 @@ private static long getCqlMetadataSize(CqlMetadata schema) { private static long getByteBufferStringMapSize(Map nameTypes) { return getCollectionSize(nameTypes.entrySet(), - entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + - ThriftObjectSizeUtils.getStringSize(entry.getValue())); + entry -> ThriftObjectSizeUtils.getByteBufferSize(entry.getKey()) + + ThriftObjectSizeUtils.getStringSize(entry.getValue())); } private static Long getCqlRowSize(CqlRow cqlRow) { @@ -226,6 +228,11 @@ private static long getCollectionSize(Collection collection, Function waitTime = rateLimiter.tryAcquire( diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/RateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/RateLimiter.java index d11f885ba18..b53460dd180 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/RateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/RateLimiter.java @@ -1,3 +1,4 @@ +//CHECKSTYLE:OFF /* * Copyright (C) 2012 The Guava Authors * @@ -15,7 +16,6 @@ */ package com.palantir.atlasdb.qos.ratelimit; -// CHECKSTYLE:OFF import static java.lang.Math.max; import static java.util.concurrent.TimeUnit.MICROSECONDS; diff --git a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java index d42e402c4c8..465b0141152 100644 --- a/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java +++ b/qos-service-impl/src/main/java/com/palantir/atlasdb/qos/ratelimit/SmoothRateLimiter.java @@ -1,3 +1,4 @@ +// CHECKSTYLE:OFF /* * Copyright (C) 2012 The Guava Authors * @@ -15,7 +16,6 @@ */ package com.palantir.atlasdb.qos.ratelimit; -// CHECKSTYLE:OFF import static java.lang.Math.min; import static java.util.concurrent.TimeUnit.SECONDS; From 71d668d2a3714ea4b81ac52936822b0ed1845010 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Fri, 10 Nov 2017 22:46:22 +0000 Subject: [PATCH 14/15] checkstyle --- .../com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java index bbfe4f69c3c..a9f623361e0 100644 --- a/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java +++ b/atlasdb-cassandra/src/test/java/com/palantir/atlasdb/ThriftObjectSizeUtilsTest.java @@ -84,7 +84,7 @@ public void getSizeForNullByteBuffer() { @Test public void getSizeForEmptyByteBuffer() { - assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(new byte[]{}))) + assertThat(ThriftObjectSizeUtils.getByteBufferSize(ByteBuffer.wrap(new byte[] {}))) .isEqualTo(0L); } @@ -156,8 +156,8 @@ public void getSizeForMutationWithDeletionContainingEmptySlicePredicate() { @Test public void getSizeForMutationWithDeletionContainingNonEmptySlicePredicate() { long deletionSize = (Long.BYTES) + (Integer.BYTES) + (TEST_MAME.getBytes().length + Integer.BYTES); - assertThat(ThriftObjectSizeUtils. - getMutationSize(new Mutation() + assertThat(ThriftObjectSizeUtils + .getMutationSize(new Mutation() .setDeletion(new Deletion() .setPredicate(new SlicePredicate() .setColumn_names(ImmutableList.of(ByteBuffer.wrap(TEST_MAME.getBytes()))))))) From 21125a6ae88aad7e475a701f0a570d5643aed07c Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Fri, 10 Nov 2017 23:20:46 +0000 Subject: [PATCH 15/15] locks and license --- gradle/shared.gradle | 2 + qos-service-impl/versions.lock | 510 ++------------------------------- 2 files changed, 20 insertions(+), 492 deletions(-) diff --git a/gradle/shared.gradle b/gradle/shared.gradle index cf5280e5680..5297fae5791 100644 --- a/gradle/shared.gradle +++ b/gradle/shared.gradle @@ -60,6 +60,8 @@ license { include '**/*.java' exclude '**/generated/**/*.java' + exclude '**/RateLimiter.java' + exclude '**/SmoothRateLimiter.java' } ext { diff --git a/qos-service-impl/versions.lock b/qos-service-impl/versions.lock index 6a64bcca0d2..664dc4edbce 100644 --- a/qos-service-impl/versions.lock +++ b/qos-service-impl/versions.lock @@ -14,10 +14,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", - "com.fasterxml.jackson.module:jackson-module-afterburner", - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations" + "com.fasterxml.jackson.module:jackson-module-afterburner" ] }, "com.fasterxml.jackson.core:jackson-databind": { @@ -26,10 +23,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", "com.fasterxml.jackson.module:jackson-module-afterburner", - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", "com.netflix.feign:feign-jackson", "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", @@ -44,7 +38,6 @@ "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { "locked": "2.6.7", "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", "com.palantir.remoting3:jackson-support" ] }, @@ -69,49 +62,17 @@ "com.palantir.remoting3:jackson-support" ] }, - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base": { - "locked": "2.6.7", - "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider" - ] - }, - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider": { - "locked": "2.6.7", - "transitive": [ - "com.palantir.remoting3:jersey-servers" - ] - }, "com.fasterxml.jackson.module:jackson-module-afterburner": { "locked": "2.6.7", "transitive": [ "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:jersey-servers", "com.palantir.remoting3:tracing" ] }, - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations": { - "locked": "2.6.7", - "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider" - ] - }, "com.google.code.findbugs:annotations": { "locked": "2.0.3", "transitive": [ - "com.palantir.atlasdb:qos-service-api", - "com.palantir.tritium:tritium-api", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.1", - "transitive": [ - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:qos-service-api" ] }, "com.google.guava:guava": { @@ -123,16 +84,7 @@ "com.palantir.remoting3:keystores", "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:refresh-utils", - "com.palantir.remoting3:tracing", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics" - ] - }, - "com.jcraft:jzlib": { - "locked": "1.1.3", - "transitive": [ - "com.palantir.remoting3:jersey-servers" + "com.palantir.remoting3:tracing" ] }, "com.netflix.feign:feign-core": { @@ -200,7 +152,6 @@ "locked": "3.5.1", "transitive": [ "com.palantir.remoting3:jaxrs-clients", - "com.palantir.remoting3:jersey-servers", "com.palantir.remoting3:okhttp-clients" ] }, @@ -225,9 +176,6 @@ "com.palantir.atlasdb:qos-service-api" ] }, - "com.palantir.remoting3:jersey-servers": { - "locked": "3.5.1" - }, "com.palantir.remoting3:keystores": { "locked": "3.5.1", "transitive": [ @@ -250,9 +198,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.remoting3:jersey-servers", - "com.palantir.remoting3:tracing-okhttp3", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:tracing-okhttp3" ] }, "com.palantir.remoting3:tracing-okhttp3": { @@ -267,13 +213,7 @@ "transitive": [ "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:jersey-servers", - "com.palantir.remoting3:tracing", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:tracing" ] }, "com.palantir.tokens:auth-tokens": { @@ -282,52 +222,6 @@ "com.palantir.remoting-api:service-config" ] }, - "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3" - }, - "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, "com.squareup.okhttp3:logging-interceptor": { "locked": "3.8.1", "transitive": [ @@ -335,31 +229,9 @@ ] }, "io.dropwizard.metrics:metrics-core": { - "locked": "3.2.5", - "transitive": [ - "com.palantir.remoting3:okhttp-clients", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" - ] - }, - "javax.annotation:javax.annotation-api": { - "locked": "1.2", + "locked": "3.2.3", "transitive": [ - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server" - ] - }, - "javax.inject:javax.inject": { - "locked": "1", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-utils" - ] - }, - "javax.validation:validation-api": { - "locked": "1.1.0.Final", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" + "com.palantir.remoting3:okhttp-clients" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -368,105 +240,7 @@ "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", "com.palantir.remoting3:error-handling", - "com.palantir.remoting3:jaxrs-clients", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.glassfish.hk2.external:aopalliance-repackaged": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-locator" - ] - }, - "org.glassfish.hk2.external:javax.inject": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-locator", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-api": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-locator", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-locator": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-utils": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-locator" - ] - }, - "org.glassfish.hk2:osgi-resource-locator": { - "locked": "1.0.1", - "transitive": [ - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.jersey.bundles.repackaged:jersey-guava": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-common" - ] - }, - "org.glassfish.jersey.core:jersey-client": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.glassfish.jersey.core:jersey-common": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.jersey.core:jersey-server": { - "locked": "2.23.2", - "transitive": [ - "com.palantir.remoting3:jersey-servers" - ] - }, - "org.glassfish.jersey.media:jersey-media-jaxb": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.hdrhistogram:HdrHistogram": { - "locked": "2.1.10", - "transitive": [ - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" - ] - }, - "org.javassist:javassist": { - "locked": "3.20.0-GA", - "transitive": [ - "org.glassfish.hk2:hk2-locator" + "com.palantir.remoting3:jaxrs-clients" ] }, "org.jvnet:animal-sniffer-annotation": { @@ -475,14 +249,8 @@ "com.netflix.feign:feign-core" ] }, - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir": { - "locked": "1.1.2", - "transitive": [ - "com.palantir.tritium:tritium-metrics" - ] - }, "org.slf4j:slf4j-api": { - "locked": "1.7.21", + "locked": "1.7.25", "transitive": [ "com.netflix.feign:feign-slf4j", "com.palantir.remoting3:error-handling", @@ -490,11 +258,6 @@ "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:tracing", "com.palantir.tokens:auth-tokens", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing", "io.dropwizard.metrics:metrics-core" ] } @@ -514,10 +277,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", - "com.fasterxml.jackson.module:jackson-module-afterburner", - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations" + "com.fasterxml.jackson.module:jackson-module-afterburner" ] }, "com.fasterxml.jackson.core:jackson-databind": { @@ -526,10 +286,7 @@ "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", "com.fasterxml.jackson.module:jackson-module-afterburner", - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", "com.netflix.feign:feign-jackson", "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", @@ -544,7 +301,6 @@ "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { "locked": "2.6.7", "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider", "com.palantir.remoting3:jackson-support" ] }, @@ -569,49 +325,17 @@ "com.palantir.remoting3:jackson-support" ] }, - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base": { - "locked": "2.6.7", - "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider" - ] - }, - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider": { - "locked": "2.6.7", - "transitive": [ - "com.palantir.remoting3:jersey-servers" - ] - }, "com.fasterxml.jackson.module:jackson-module-afterburner": { "locked": "2.6.7", "transitive": [ "com.palantir.remoting3:jackson-support", - "com.palantir.remoting3:jersey-servers", "com.palantir.remoting3:tracing" ] }, - "com.fasterxml.jackson.module:jackson-module-jaxb-annotations": { - "locked": "2.6.7", - "transitive": [ - "com.fasterxml.jackson.jaxrs:jackson-jaxrs-cbor-provider" - ] - }, "com.google.code.findbugs:annotations": { "locked": "2.0.3", "transitive": [ - "com.palantir.atlasdb:qos-service-api", - "com.palantir.tritium:tritium-api", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.1", - "transitive": [ - "com.palantir.remoting-api:errors", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" + "com.palantir.atlasdb:qos-service-api" ] }, "com.google.guava:guava": { @@ -623,16 +347,7 @@ "com.palantir.remoting3:keystores", "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:refresh-utils", - "com.palantir.remoting3:tracing", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics" - ] - }, - "com.jcraft:jzlib": { - "locked": "1.1.3", - "transitive": [ - "com.palantir.remoting3:jersey-servers" + "com.palantir.remoting3:tracing" ] }, "com.netflix.feign:feign-core": { @@ -700,7 +415,6 @@ "locked": "3.5.1", "transitive": [ "com.palantir.remoting3:jaxrs-clients", - "com.palantir.remoting3:jersey-servers", "com.palantir.remoting3:okhttp-clients" ] }, @@ -725,9 +439,6 @@ "com.palantir.atlasdb:qos-service-api" ] }, - "com.palantir.remoting3:jersey-servers": { - "locked": "3.5.1" - }, "com.palantir.remoting3:keystores": { "locked": "3.5.1", "transitive": [ @@ -750,9 +461,7 @@ "com.palantir.remoting3:tracing": { "locked": "3.5.1", "transitive": [ - "com.palantir.remoting3:jersey-servers", - "com.palantir.remoting3:tracing-okhttp3", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:tracing-okhttp3" ] }, "com.palantir.remoting3:tracing-okhttp3": { @@ -767,13 +476,7 @@ "transitive": [ "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", - "com.palantir.remoting3:jersey-servers", - "com.palantir.remoting3:tracing", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" + "com.palantir.remoting3:tracing" ] }, "com.palantir.tokens:auth-tokens": { @@ -782,52 +485,6 @@ "com.palantir.remoting-api:service-config" ] }, - "com.palantir.tritium:tritium-api": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.palantir.tritium:tritium-core": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing" - ] - }, - "com.palantir.tritium:tritium-lib": { - "locked": "0.8.3" - }, - "com.palantir.tritium:tritium-metrics": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-proxy": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-slf4j": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, - "com.palantir.tritium:tritium-tracing": { - "locked": "0.8.3", - "transitive": [ - "com.palantir.tritium:tritium-lib" - ] - }, "com.squareup.okhttp3:logging-interceptor": { "locked": "3.8.1", "transitive": [ @@ -835,31 +492,9 @@ ] }, "io.dropwizard.metrics:metrics-core": { - "locked": "3.2.5", - "transitive": [ - "com.palantir.remoting3:okhttp-clients", - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" - ] - }, - "javax.annotation:javax.annotation-api": { - "locked": "1.2", + "locked": "3.2.3", "transitive": [ - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server" - ] - }, - "javax.inject:javax.inject": { - "locked": "1", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-utils" - ] - }, - "javax.validation:validation-api": { - "locked": "1.1.0.Final", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" + "com.palantir.remoting3:okhttp-clients" ] }, "javax.ws.rs:javax.ws.rs-api": { @@ -868,105 +503,7 @@ "com.palantir.atlasdb:qos-service-api", "com.palantir.remoting-api:errors", "com.palantir.remoting3:error-handling", - "com.palantir.remoting3:jaxrs-clients", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.glassfish.hk2.external:aopalliance-repackaged": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-locator" - ] - }, - "org.glassfish.hk2.external:javax.inject": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-locator", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-api": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-locator", - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-locator": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.hk2:hk2-utils": { - "locked": "2.5.0-b05", - "transitive": [ - "org.glassfish.hk2:hk2-api", - "org.glassfish.hk2:hk2-locator" - ] - }, - "org.glassfish.hk2:osgi-resource-locator": { - "locked": "1.0.1", - "transitive": [ - "org.glassfish.jersey.core:jersey-common", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.jersey.bundles.repackaged:jersey-guava": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-common" - ] - }, - "org.glassfish.jersey.core:jersey-client": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.glassfish.jersey.core:jersey-common": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-client", - "org.glassfish.jersey.core:jersey-server", - "org.glassfish.jersey.media:jersey-media-jaxb" - ] - }, - "org.glassfish.jersey.core:jersey-server": { - "locked": "2.23.2", - "transitive": [ - "com.palantir.remoting3:jersey-servers" - ] - }, - "org.glassfish.jersey.media:jersey-media-jaxb": { - "locked": "2.23.2", - "transitive": [ - "org.glassfish.jersey.core:jersey-server" - ] - }, - "org.hdrhistogram:HdrHistogram": { - "locked": "2.1.10", - "transitive": [ - "com.palantir.tritium:tritium-metrics", - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir" - ] - }, - "org.javassist:javassist": { - "locked": "3.20.0-GA", - "transitive": [ - "org.glassfish.hk2:hk2-locator" + "com.palantir.remoting3:jaxrs-clients" ] }, "org.jvnet:animal-sniffer-annotation": { @@ -975,14 +512,8 @@ "com.netflix.feign:feign-core" ] }, - "org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir": { - "locked": "1.1.2", - "transitive": [ - "com.palantir.tritium:tritium-metrics" - ] - }, "org.slf4j:slf4j-api": { - "locked": "1.7.21", + "locked": "1.7.25", "transitive": [ "com.netflix.feign:feign-slf4j", "com.palantir.remoting3:error-handling", @@ -990,11 +521,6 @@ "com.palantir.remoting3:okhttp-clients", "com.palantir.remoting3:tracing", "com.palantir.tokens:auth-tokens", - "com.palantir.tritium:tritium-core", - "com.palantir.tritium:tritium-lib", - "com.palantir.tritium:tritium-metrics", - "com.palantir.tritium:tritium-slf4j", - "com.palantir.tritium:tritium-tracing", "io.dropwizard.metrics:metrics-core" ] }