From d5cc1e9c293acef55722bdd85e6974ec3504f8d5 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 12:57:25 +0000 Subject: [PATCH 01/53] copy pasta --- .../api/CandidateCellForSweeping.java | 4 + .../palantir/atlasdb/keyvalue/api/Cell.java | 4 + .../atlasdb/keyvalue/api/RowResult.java | 4 + .../atlasdb/keyvalue/api/TableReference.java | 9 ++ .../palantir/atlasdb/keyvalue/api/Value.java | 5 + .../api/CandidateCellForSweepingTest.java | 111 ++++++++++++++++++ .../atlasdb/keyvalue/api/CellTest.java | 34 ++++++ .../keyvalue/api/TableReferenceTest.java | 37 ++++++ .../atlasdb/keyvalue/api/ValueTest.java | 50 ++++++++ 9 files changed, 258 insertions(+) create mode 100644 atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweepingTest.java create mode 100644 atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/ValueTest.java diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweeping.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweeping.java index 10203ede228..c18c089588a 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweeping.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweeping.java @@ -41,4 +41,8 @@ public interface CandidateCellForSweeping { * Otherwise, the return value is undefined and depends on the implementation. */ boolean isLatestValueEmpty(); + + default long sizeInBytes() { + return cell().sizeInBytes() + ((long) sortedTimestamps().size()) * Long.BYTES; + } } diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Cell.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Cell.java index e1add1e5215..e472b7086fc 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Cell.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Cell.java @@ -116,6 +116,10 @@ public byte[] getColumnName() { return columnName; } + public long sizeInBytes() { + return Long.sum(rowName.length, columnName.length); + } + @Override public int compareTo(Cell other) { int cmp = UnsignedBytes.lexicographicalComparator().compare(rowName, other.rowName); diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/RowResult.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/RowResult.java index 46cf40dbdc9..b7897188122 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/RowResult.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/RowResult.java @@ -68,6 +68,10 @@ public byte[] getRowName() { return row.clone(); } + public long getRowNameSize() { + return row.length; + } + public NavigableMap getColumns() { return columns; } diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/TableReference.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/TableReference.java index a32f83d0c69..f4e4dc2f814 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/TableReference.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/TableReference.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.nio.charset.StandardCharsets; import java.util.Objects; import org.apache.commons.lang3.StringUtils; @@ -146,6 +147,14 @@ public String toString() { return getQualifiedName(); } + public long sizeInBytes() { + return stringSizeInBytes(tableName) + stringSizeInBytes(namespace.getName()); + } + + private static long stringSizeInBytes(String string) { + return Character.BYTES * ((long) string.getBytes(StandardCharsets.UTF_8).length); + } + public static TableReference fromString(String tableReferenceAsString) { int dotCount = StringUtils.countMatches(tableReferenceAsString, "."); if (dotCount == 0) { diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Value.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Value.java index 601de82155d..e658af135e8 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Value.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/keyvalue/api/Value.java @@ -82,6 +82,11 @@ private Value(byte[] contents, long timestamp) { public static final Function GET_VALUE = Value::getContents; + public long sizeInBytes() { + // one byte added for the timestamp + return Long.sum(Long.BYTES, contents.length); + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweepingTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweepingTest.java new file mode 100644 index 00000000000..b2723846b62 --- /dev/null +++ b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CandidateCellForSweepingTest.java @@ -0,0 +1,111 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class CandidateCellForSweepingTest { + private static final byte BYTE = (byte) 0xa; + private static final long TIMESTAMP = 1977; + private static final ImmutableSet THREE_CELL_NAME_SIZES = + ImmutableSet.of(1, Cell.MAX_NAME_LENGTH / 2, Cell.MAX_NAME_LENGTH); + private static final ImmutableList EXAMPLE_CELLS = + Sets.cartesianProduct(THREE_CELL_NAME_SIZES, THREE_CELL_NAME_SIZES).stream() + .map(pair -> Cell.create(spawnBytes(pair.get(0)), spawnBytes(pair.get(1)))) + .collect(ImmutableList.toImmutableList()); + + private static final ImmutableList SORTED_TIMESTAMPS_SIZES = ImmutableList.of(0, 1, 2, 100, 1000); + + @Mock + private List MOCK_TIMESTAMPS; + + @Test + public void candidateCellSizeWithLargerTimestampCollectionIsBigger() { + EXAMPLE_CELLS.forEach(cell -> { + CandidateCellForSweeping withOneTimestamp = createCandidateCell(cell, ImmutableSet.of(TIMESTAMP), true); + CandidateCellForSweeping withTwoTimestamps = + createCandidateCell(cell, ImmutableSet.of(TIMESTAMP, TIMESTAMP + 1), false); + assertThat(withOneTimestamp.sizeInBytes()).isLessThan(withTwoTimestamps.sizeInBytes()); + }); + } + + @Test + public void candidateCellSizeIsCorrectForDifferentSortedTimestampSizes() { + SORTED_TIMESTAMPS_SIZES.forEach(sortedTimestampsSize -> { + for (CandidateCellForSweeping candidate : createCandidateCells(sortedTimestampsSize)) { + assertThat(candidate.sizeInBytes()) + .isEqualTo(Long.sum(candidate.cell().sizeInBytes(), (long) sortedTimestampsSize * Long.BYTES)); + } + }); + } + + @Test + public void noOverflowFromCollectionSize() { + // Mocking because otherwise we OOM. + when(MOCK_TIMESTAMPS.size()).thenReturn(Integer.MAX_VALUE); + Cell exampleCell = EXAMPLE_CELLS.get(0); + for (boolean isLatestValueEmpty : new boolean[] {true, false}) { + assertThat(createCandidateCell(exampleCell, MOCK_TIMESTAMPS, isLatestValueEmpty) + .sizeInBytes()) + .isEqualTo(Long.sum(Integer.MAX_VALUE * 8L, exampleCell.sizeInBytes())); + } + } + + private static ImmutableSet createCandidateCells(int sortedTimestampsSize) { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (boolean isLatestValueEmpty : new boolean[] {true, false}) { + builder.addAll(EXAMPLE_CELLS.stream() + .map(cell -> createCandidateCell( + cell, spawnCollectionOfTimestamps(sortedTimestampsSize), isLatestValueEmpty)) + .iterator()); + } + return builder.build(); + } + + private static CandidateCellForSweeping createCandidateCell( + Cell cell, Collection sortedTimestamps, boolean isLatestValueEmpty) { + return ImmutableCandidateCellForSweeping.builder() + .cell(cell) + .sortedTimestamps(sortedTimestamps) + .isLatestValueEmpty(isLatestValueEmpty) + .build(); + } + + private static List spawnCollectionOfTimestamps(int size) { + return Collections.nCopies(size, TIMESTAMP); + } + + private static byte[] spawnBytes(int size) { + byte[] bytes = new byte[size]; + Arrays.fill(bytes, BYTE); + return bytes; + } +} diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CellTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CellTest.java index 50e6d01e47b..a21d33a1637 100644 --- a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CellTest.java +++ b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/CellTest.java @@ -18,12 +18,21 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import com.palantir.logsafe.exceptions.SafeIllegalArgumentException; import com.palantir.logsafe.exceptions.SafeNullPointerException; +import com.palantir.util.Pair; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; import org.junit.Test; public final class CellTest { + private static final ImmutableSet THREE_CELL_NAME_SIZES = + ImmutableSet.of(1, Cell.MAX_NAME_LENGTH / 2, Cell.MAX_NAME_LENGTH); + private static final ImmutableSet TWO_BYTES = ImmutableSet.of((byte) 0xa, (byte) 0xb); @Test public void testCreate() { @@ -80,6 +89,31 @@ public void testHashCode() { .hasSameHashCodeAs(Cell.create(bytes("col"), bytes("row"))); } + @Test + public void testSizeInBytes() { + for (Pair sizes : allPairs(THREE_CELL_NAME_SIZES)) { + for (Pair bytes : allPairs(TWO_BYTES)) { + assertThat(Cell.create( + spawnBytes(sizes.getLhSide(), bytes.getLhSide()), + spawnBytes(sizes.getRhSide(), bytes.getRhSide())) + .sizeInBytes()) + .isEqualTo(sizes.getLhSide() + sizes.getRhSide()); + } + } + } + + private static Set> allPairs(Set set) { + return Sets.cartesianProduct(set, set).stream() + .map(list -> Pair.create(list.get(0), list.get(1))) + .collect(Collectors.toSet()); + } + + private static byte[] spawnBytes(int size, byte element) { + byte[] bytes = new byte[size]; + Arrays.fill(bytes, element); + return bytes; + } + private static byte[] bytes(String value) { return value.getBytes(StandardCharsets.UTF_8); } diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/TableReferenceTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/TableReferenceTest.java index 43bef742067..1cb52610646 100644 --- a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/TableReferenceTest.java +++ b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/TableReferenceTest.java @@ -56,4 +56,41 @@ public int hashCode() { }) .doesNotHaveSameHashCodeAs(TableReference.create(Namespace.create("table"), "test")); } + + @Test + public void sizeInBytesForTableReferenceWithEmptyNamespaceIsSizeOfAsciiTableName() { + assertThat(TableReference.createWithEmptyNamespace("").sizeInBytes()).isEqualTo(0); + assertThat(TableReference.createWithEmptyNamespace("FOO").sizeInBytes()).isEqualTo(3 * Character.BYTES); + assertThat(TableReference.createWithEmptyNamespace("FOOBA").sizeInBytes()) + .isEqualTo(5 * Character.BYTES); + } + + @Test + public void sizeInBytesForTableReferenceWithAsciiNamespaceAndTableNameIsCorrect() { + assertThat(TableReference.create(Namespace.create("FOO"), "").sizeInBytes()) + .isEqualTo(3 * Character.BYTES); + assertThat(TableReference.create(Namespace.create("FOO"), "BAR").sizeInBytes()) + .isEqualTo(6 * Character.BYTES); + assertThat(TableReference.create(Namespace.create("FOO"), "BABAZ").sizeInBytes()) + .isEqualTo(8 * Character.BYTES); + assertThat(TableReference.create(Namespace.create("FOOBAR"), "BAZ").sizeInBytes()) + .isEqualTo(9 * Character.BYTES); + } + + @Test + public void orderOfSizeInBytesOfValuesWithSameNamespaceFollowsTableNameSizeOrder() { + Namespace namespace = Namespace.create("TestNameSpace"); + assertThat(TableReference.create(namespace, "smallerTableName").sizeInBytes()) + .isLessThan(TableReference.create(namespace, "largerTableNamePadding") + .sizeInBytes()); + } + + @Test + public void orderOfSizeInBytesOfValuesWithSameTableNameFollowsNamespaceOrder() { + String tableName = "tableName"; + assertThat(TableReference.create(Namespace.create("smallerNamespace"), tableName) + .sizeInBytes()) + .isLessThan(TableReference.create(Namespace.create("largerNamespacePadding"), tableName) + .sizeInBytes()); + } } diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/ValueTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/ValueTest.java new file mode 100644 index 00000000000..2cb08dfc959 --- /dev/null +++ b/atlasdb-api/src/test/java/com/palantir/atlasdb/keyvalue/api/ValueTest.java @@ -0,0 +1,50 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.keyvalue.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.palantir.atlasdb.encoding.PtBytes; +import java.util.Arrays; +import org.junit.Test; + +public class ValueTest { + private static final int SMALLER = 100; + private static final int LARGER = 200; + private static final byte BYTE = (byte) 0xa; + + @Test + public void sizeInBytesOfValueWithNoContentsIsSizeOfLong() { + assertThat(Value.create(PtBytes.EMPTY_BYTE_ARRAY, Value.INVALID_VALUE_TIMESTAMP) + .sizeInBytes()) + .isEqualTo(Long.BYTES); + } + + @Test + public void sizeInBytesOfValueOrderFollowsContentsSizeOrder() { + assertThat(Value.create(spawnBytes(SMALLER), Value.INVALID_VALUE_TIMESTAMP) + .sizeInBytes()) + .isLessThan(Value.create(spawnBytes(LARGER), Value.INVALID_VALUE_TIMESTAMP) + .sizeInBytes()); + } + + private static byte[] spawnBytes(int size) { + byte[] bytes = new byte[size]; + Arrays.fill(bytes, BYTE); + return bytes; + } +} From 8a3eef096248f1902ff09f511e4333ba5da15742 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 13:30:17 +0000 Subject: [PATCH 02/53] adding expectations measuring utilities & tests --- .../ExpectationsMeasuringUtils.java | 92 ++++++++ .../ExpectationsMeasuringUtilsTest.java | 211 ++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java new file mode 100644 index 00000000000..f248682e29b --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java @@ -0,0 +1,92 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.google.common.collect.Multimap; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.RangeRequest; +import com.palantir.atlasdb.keyvalue.api.RowResult; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.api.Value; +import com.palantir.util.paging.TokenBackedBasicResultsPage; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.Function; + +public final class ExpectationsMeasuringUtils { + private ExpectationsMeasuringUtils() {} + + public static long sizeInBytes(Entry entry) { + return entry.getKey().sizeInBytes() + entry.getValue().sizeInBytes(); + } + + public static long sizeInBytes(Multimap valueByCell) { + return valueByCell.keys().stream() + .mapToLong(cell -> cell.sizeInBytes() + Long.BYTES) + .sum(); + } + + public static long sizeInBytes(Map longByCell) { + return longByCell.keySet().stream() + .mapToLong(cell -> cell.sizeInBytes() + Long.BYTES) + .sum(); + } + + public static long sizeInBytes(RowResult rowResult) { + return sizeInBytes(rowResult, Value::sizeInBytes); + } + + private static long sizeInBytes(RowResult rowResult, Function measurer) { + return rowResult.getRowNameSize() + + rowResult.getColumns().entrySet().stream() + .mapToLong(entry -> entry.getKey().length + measurer.apply(entry.getValue())) + .sum(); + } + + public static long arrayByRefSizeInBytes(Map arrayByRef) { + return arrayByRef.entrySet().stream() + .mapToLong(entry -> entry.getKey().sizeInBytes() + entry.getValue().length) + .sum(); + } + + public static long valueByCellSizeInBytes(Map valueByCell) { + return valueByCell.entrySet().stream() + .mapToLong(ExpectationsMeasuringUtils::sizeInBytes) + .sum(); + } + + /** + * Ignoring the size of token for next page because the interface method + * {@link TokenBackedBasicResultsPage#getTokenForNextPage} might have side effects (e.g. interact with the kvs). + * The interface specification is ambiguous. Also, the current token is not exposed. + * todo(aalouane): SG search + */ + public static long pageByRequestSizeInBytes( + Map, byte[]>> pageByRange) { + return pageByRange.values().stream() + .map(TokenBackedBasicResultsPage::getResults) + .flatMap(Collection::stream) + .mapToLong(ExpectationsMeasuringUtils::sizeInBytes) + .sum(); + } + + public static long setResultSizeInBytes(RowResult> rowResult) { + return sizeInBytes(rowResult, set -> (long) set.size() * Long.BYTES); + } +} diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java new file mode 100644 index 00000000000..fddacc200ef --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java @@ -0,0 +1,211 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Multimap; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.RangeRequest; +import com.palantir.atlasdb.keyvalue.api.RowResult; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.api.Value; +import com.palantir.util.paging.SimpleTokenBackedResultsPage; +import com.palantir.util.paging.TokenBackedBasicResultsPage; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; + +public class ExpectationsMeasuringUtilsTest { + private static final byte BYTE = (byte) 0xa; + private static final int SIZE_1 = 123; + private static final int SIZE_2 = 214; + private static final int SIZE_3 = 329; + private static final int TIMESTAMP = 100; + + @Test + public void emptyValueByCellMapSizeIsZero() { + assertThat(ExpectationsMeasuringUtils.valueByCellSizeInBytes(Map.of())) + .isEqualTo(0); + } + + @Test + public void emptyLongByCellMapSizeInBytesIsZero() { + assertThat(ExpectationsMeasuringUtils.sizeInBytes(Map.of())).isEqualTo(0); + } + + @Test + public void emptyLongByCellMultimapSizeInBytesIsZero() { + assertThat(ExpectationsMeasuringUtils.sizeInBytes(ArrayListMultimap.create())) + .isEqualTo(0); + } + + @Test + public void emptyByteArrayByTableReferenceMapSizeInBytesIsZero() { + assertThat(ExpectationsMeasuringUtils.arrayByRefSizeInBytes(Map.of())) + .isEqualTo(0); + } + + @Test + public void emptyTokenPageByRangeRequestMapSizeInBytesIsZero() { + assertThat(ExpectationsMeasuringUtils.pageByRequestSizeInBytes( + Map., byte[]>>of())) + .isEqualTo(0); + } + + @Test + public void testLongByCellMapSize() { + Map longByCell = IntStream.range(1, SIZE_1 + 1) + .boxed() + .collect(Collectors.toUnmodifiableMap(ExpectationsMeasuringUtilsTest::createCell, Long::valueOf)); + + // expected value follows from https://en.wikipedia.org/wiki/1_%2B_2_%2B_3_%2B_4_%2B_%E2%8B%AF + assertThat(ExpectationsMeasuringUtils.sizeInBytes(longByCell)) + .isEqualTo(Long.sum(Long.BYTES, (long) SIZE_1 + 1) * SIZE_1); + } + + @Test + public void testValueByCellMapSize() { + Map valueByCell = IntStream.range(1, SIZE_1 + 1) + .boxed() + .collect(Collectors.toUnmodifiableMap( + ExpectationsMeasuringUtilsTest::createCell, size -> createValue(2 * size))); + + assertThat(ExpectationsMeasuringUtils.valueByCellSizeInBytes(valueByCell)) + .isEqualTo((2L * SIZE_1 + Long.BYTES + 2) * SIZE_1); + } + + @Test + public void testValueByLongMultimapSize() { + Multimap valueByLong = IntStream.range(0, SIZE_2 * SIZE_1) + .boxed() + .collect(ImmutableSetMultimap.toImmutableSetMultimap( + valueInt -> createCell(1 + (valueInt % SIZE_1)), Long::valueOf)); + + assertThat(ExpectationsMeasuringUtils.sizeInBytes(valueByLong)) + .isEqualTo(SIZE_2 * SIZE_1 * (SIZE_1 + Long.BYTES + 1L)); + } + + @Test + public void testArrayByTableRefSize() { + Map arrayByTableRef = IntStream.range(1, 2 * SIZE_1 + 1) + .boxed() + .collect(Collectors.toUnmodifiableMap( + ExpectationsMeasuringUtilsTest::createTableReference, + ExpectationsMeasuringUtilsTest::spawnBytes)); + + assertThat(ExpectationsMeasuringUtils.arrayByRefSizeInBytes(arrayByTableRef)) + .isEqualTo((Character.BYTES + 1L) * (2L * SIZE_1 + 1L) * SIZE_1); + } + + @Test + public void testValueRowResult() { + assertThat(ExpectationsMeasuringUtils.sizeInBytes(RowResult.of(createCell(SIZE_1), createValue(SIZE_1)))) + .isEqualTo(3L * SIZE_1 + Long.BYTES); + + assertThat(ExpectationsMeasuringUtils.sizeInBytes(RowResult.of(createCell(SIZE_2), createValue(SIZE_2)))) + .isEqualTo(3L * SIZE_2 + Long.BYTES); + } + + @Test + public void testLongSetRowResultSize() { + assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of()))) + .isEqualTo(2L * SIZE_1); + + assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of(1L)))) + .isEqualTo(2L * SIZE_1 + Long.BYTES); + + assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of(1L, 2L)))) + .isEqualTo(2L * SIZE_1 + 2L * Long.BYTES); + + assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of( + createCell(SIZE_1), LongStream.range(0, SIZE_1).boxed().collect(Collectors.toSet())))) + .isEqualTo(SIZE_1 * (Long.BYTES + 2L)); + } + + @Test + public void testValueByCellEntrySize() { + assertThat(ExpectationsMeasuringUtils.sizeInBytes( + new SimpleImmutableEntry<>(createCell(SIZE_1), createValue(SIZE_1)))) + .isEqualTo(3L * SIZE_1 + Long.BYTES); + + assertThat(ExpectationsMeasuringUtils.sizeInBytes( + new SimpleImmutableEntry<>(createCell(SIZE_2), createValue(SIZE_2)))) + .isEqualTo(3L * SIZE_2 + Long.BYTES); + } + + @Test + public void testPageByRequestSize() { + assertThat(ExpectationsMeasuringUtils.pageByRequestSizeInBytes( + ImmutableMap., byte[]>>builder() + .put(RangeRequest.all(), createPage(SIZE_2, SIZE_2, SIZE_2, 0)) + .put( + RangeRequest.builder() + .startRowInclusive(spawnBytes(SIZE_3)) + .build(), + createPage(SIZE_2, SIZE_2, SIZE_2, 1)) + .put( + RangeRequest.builder() + .endRowExclusive(spawnBytes(SIZE_3)) + .build(), + createPage(SIZE_3, SIZE_2, SIZE_2, SIZE_3)) + .put( + RangeRequest.builder() + .startRowInclusive(spawnBytes(SIZE_2)) + .build(), + createPage(SIZE_3, SIZE_3, SIZE_3, SIZE_3)) + .buildOrThrow())) + .isEqualTo((SIZE_3 + 1L) * (3L * SIZE_2 + Long.BYTES) + SIZE_3 * (3L * SIZE_3 + Long.BYTES)); + } + + private static SimpleTokenBackedResultsPage, byte[]> createPage( + int tokenSize, int cellNameSize, int valueNameSize, int resultsCount) { + return SimpleTokenBackedResultsPage.create( + spawnBytes(tokenSize), + Stream.generate(() -> RowResult.of(createCell(cellNameSize), createValue(valueNameSize))) + .limit(resultsCount) + .collect(Collectors.toUnmodifiableList())); + } + + private static TableReference createTableReference(int size) { + return TableReference.createWithEmptyNamespace(StringUtils.repeat('A', size)); + } + + private static Cell createCell(int nameSize) { + return Cell.create(spawnBytes(nameSize), spawnBytes(nameSize)); + } + + private static Value createValue(int size) { + return Value.create(spawnBytes(size), TIMESTAMP); + } + + private static byte[] spawnBytes(int size) { + byte[] bytes = new byte[size]; + Arrays.fill(bytes, BYTE); + return bytes; + } +} From 037777e5bade3e82aad6f814959130362905287e Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 14:12:37 +0000 Subject: [PATCH 03/53] adding keyvalueservice data tracker & tests --- .../KeyValueServiceDataTracker.java | 128 +++++++ .../KeyValueServiceDataTrackerTest.java | 344 ++++++++++++++++++ 2 files changed, 472 insertions(+) create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTracker.java create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTrackerTest.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTracker.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTracker.java new file mode 100644 index 00000000000..7bee7b3efce --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTracker.java @@ -0,0 +1,128 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.google.common.collect.Comparators; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AtomicLongMap; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableKvsCallReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.KvsCallReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import java.util.Comparator; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; + +public final class KeyValueServiceDataTracker { + private final AtomicLongMap bytesReadByTable = AtomicLongMap.create(); + private final LongAdder bytesReadOverall = new LongAdder(); + private final AtomicLongMap kvsCallByTable = AtomicLongMap.create(); + private final LongAdder kvsCallsOverall = new LongAdder(); + private final ConcurrentMap maximumBytesKvsCallInfoByTable = + new ConcurrentHashMap<>(); + private final AtomicReference> maximumBytesKvsCallInfoOverall = + new AtomicReference<>(Optional.empty()); + + public TransactionReadInfo getReadInfo() { + return ImmutableTransactionReadInfo.builder() + .bytesRead(bytesReadOverall.longValue()) + .kvsCalls(kvsCallsOverall.longValue()) + .maximumBytesKvsCallInfo(maximumBytesKvsCallInfoOverall.get()) + .build(); + } + + /** + * This is un-synchronized as it is called after task completion and the abort/commit stage. + * For returned statistics, no consistency guarantees are provided if the user interacts with the transaction + * outside the context of {@link com.palantir.atlasdb.transaction.api.TransactionTask} or after commit/abort + * stages (e.g. this can happen with futures/iterators). + */ + public ImmutableMap getReadInfoByTable() { + Set tableRefs = Sets.intersection( + bytesReadByTable.asMap().keySet(), kvsCallByTable.asMap().keySet()); + + return tableRefs.stream() + .collect(ImmutableMap.toImmutableMap( + Function.identity(), tableRef -> ImmutableTransactionReadInfo.builder() + .bytesRead(bytesReadByTable.get(tableRef)) + .kvsCalls(kvsCallByTable.get(tableRef)) + .maximumBytesKvsCallInfo( + Optional.ofNullable(maximumBytesKvsCallInfoByTable.get(tableRef))) + .build())); + } + + /** + * Tracks an effectively completed kvs read method call for some {@link TableReference}. + * Effectively completed refers to an eager call (i.e. does not spawn futures or iterators for later consumption). + */ + public void readForTable(TableReference tableRef, String methodName, long bytesRead) { + KvsCallReadInfo callInfo = ImmutableKvsCallReadInfo.builder() + .bytesRead(bytesRead) + .methodName(methodName) + .build(); + updateKvsMethodOverallTallies(callInfo); + updateKvsMethodByTableTallies(tableRef, callInfo); + } + + /** + * Track a lazy kvs read method call for some {@link TableReference}. + */ + public void partialReadForTable(TableReference tableRef, long bytesRead) { + bytesReadOverall.add(bytesRead); + bytesReadByTable.addAndGet(tableRef, bytesRead); + } + + /** + * Track that a kvs read method was called for some {@link TableReference}. + */ + public void callForTable(TableReference tableRef) { + kvsCallsOverall.increment(); + kvsCallByTable.incrementAndGet(tableRef); + } + + /** + * Track an effectively completed kvs read method call with no {@link TableReference} information. + * Effectively completed refers to an eager call (i.e. does not spawn futures or iterators for later consumption). + */ + public void tableAgnosticRead(String methodName, long bytesRead) { + KvsCallReadInfo callInfo = ImmutableKvsCallReadInfo.builder() + .bytesRead(bytesRead) + .methodName(methodName) + .build(); + updateKvsMethodOverallTallies(callInfo); + } + + private void updateKvsMethodOverallTallies(KvsCallReadInfo callInfo) { + bytesReadOverall.add(callInfo.bytesRead()); + kvsCallsOverall.increment(); + maximumBytesKvsCallInfoOverall.updateAndGet(currentMaybeCall -> Comparators.max( + currentMaybeCall, Optional.of(callInfo), Comparators.emptiesFirst(Comparator.naturalOrder()))); + } + + private void updateKvsMethodByTableTallies(TableReference tableRef, KvsCallReadInfo callInfo) { + bytesReadByTable.addAndGet(tableRef, callInfo.bytesRead()); + kvsCallByTable.incrementAndGet(tableRef); + maximumBytesKvsCallInfoByTable.merge(tableRef, callInfo, Comparators::max); + } +} diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTrackerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTrackerTest.java new file mode 100644 index 00000000000..340daa2f893 --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/KeyValueServiceDataTrackerTest.java @@ -0,0 +1,344 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableMap; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableKvsCallReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import org.junit.Test; + +public class KeyValueServiceDataTrackerTest { + private static final TableReference TABLE_1 = TableReference.createWithEmptyNamespace("Table1"); + private static final TableReference TABLE_2 = TableReference.createWithEmptyNamespace("Table2"); + private static final TableReference TABLE_3 = TableReference.createWithEmptyNamespace("Table3"); + private static final String KVS_METHOD_NAME_1 = "getRows"; + private static final String KVS_METHOD_NAME_2 = "getAsync"; + private static final String KVS_METHOD_NAME_3 = "get"; + private static final String KVS_METHOD_NAME_4 = "getMetadataForTables"; + private static final String KVS_METHOD_NAME_5 = "getMetadataForTable"; + private static final long NO_BYTES_READ = 0L; + private static final long BYTES_READ_1 = 83L; + private static final long BYTES_READ_2 = 103L; + private static final long BYTES_READ_3 = 971L; + + private final KeyValueServiceDataTracker tracker = new KeyValueServiceDataTracker(); + + @Test + public void noReadsTracksNothing() { + assertEquals( + ImmutableTransactionReadInfo.builder().bytesRead(0).kvsCalls(0).build(), tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void oneReadForTableIsTracked() { + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_1, BYTES_READ_1); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1) + .kvsCalls(1) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_1) + .methodName(KVS_METHOD_NAME_1) + .build()) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(TABLE_1, readInfo), tracker.getReadInfoByTable()); + } + + @Test + public void multipleReadsForTableAreTracked() { + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_1, BYTES_READ_1); + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_2, BYTES_READ_2); + tracker.readForTable(TABLE_2, KVS_METHOD_NAME_2, BYTES_READ_2); + tracker.readForTable(TABLE_3, KVS_METHOD_NAME_3, BYTES_READ_3); + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_3, NO_BYTES_READ); + tracker.readForTable(TABLE_2, KVS_METHOD_NAME_2, BYTES_READ_3); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + 2 * BYTES_READ_2 + 2 * BYTES_READ_3) + .kvsCalls(6) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_2) + .build()) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + + ImmutableMap readInfoByTable = + ImmutableMap.builder() + .put( + TABLE_1, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + BYTES_READ_2) + .kvsCalls(3) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_2) + .methodName(KVS_METHOD_NAME_2) + .build()) + .build()) + .put( + TABLE_2, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_2 + BYTES_READ_3) + .kvsCalls(2) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_2) + .build()) + .build()) + .put( + TABLE_3, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_3) + .kvsCalls(1) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_3) + .build()) + .build()) + .buildOrThrow(); + + assertEquals(readInfoByTable, tracker.getReadInfoByTable()); + } + + @Test + public void onePartialReadForTableIsTracked() { + tracker.callForTable(TABLE_2); + tracker.partialReadForTable(TABLE_2, BYTES_READ_2); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_2) + .kvsCalls(1) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(TABLE_2, readInfo), tracker.getReadInfoByTable()); + } + + @Test + public void multiplePartialReadForTableAreTracked() { + tracker.callForTable(TABLE_1); + tracker.partialReadForTable(TABLE_1, BYTES_READ_1); + tracker.partialReadForTable(TABLE_1, NO_BYTES_READ); + tracker.callForTable(TABLE_1); + tracker.partialReadForTable(TABLE_1, BYTES_READ_3); + tracker.partialReadForTable(TABLE_1, BYTES_READ_2); + tracker.partialReadForTable(TABLE_1, BYTES_READ_3); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + BYTES_READ_2 + 2 * BYTES_READ_3) + .kvsCalls(2) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(TABLE_1, readInfo), tracker.getReadInfoByTable()); + } + + @Test + public void multiplePartialReadForTableOnMultipleTablesAreTracked() { + tracker.callForTable(TABLE_1); + tracker.callForTable(TABLE_2); + tracker.callForTable(TABLE_2); + tracker.partialReadForTable(TABLE_1, BYTES_READ_1); + tracker.callForTable(TABLE_3); + tracker.partialReadForTable(TABLE_3, NO_BYTES_READ); + tracker.partialReadForTable(TABLE_1, NO_BYTES_READ); + tracker.callForTable(TABLE_3); + tracker.partialReadForTable(TABLE_3, BYTES_READ_3); + tracker.callForTable(TABLE_3); + tracker.partialReadForTable(TABLE_3, BYTES_READ_3); + tracker.partialReadForTable(TABLE_2, BYTES_READ_1); + tracker.partialReadForTable(TABLE_1, BYTES_READ_2); + tracker.partialReadForTable(TABLE_2, BYTES_READ_3); + tracker.partialReadForTable(TABLE_3, BYTES_READ_1); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(3 * BYTES_READ_1 + BYTES_READ_2 + 3 * BYTES_READ_3) + .kvsCalls(6) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + + ImmutableMap readInfoByTable = + ImmutableMap.builder() + .put( + TABLE_1, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + BYTES_READ_2) + .kvsCalls(1) + .build()) + .put( + TABLE_2, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + BYTES_READ_3) + .kvsCalls(2) + .build()) + .put( + TABLE_3, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + 2 * BYTES_READ_3) + .kvsCalls(3) + .build()) + .buildOrThrow(); + + assertEquals(readInfoByTable, tracker.getReadInfoByTable()); + } + + @Test + public void oneCallForTableIsTracked() { + tracker.callForTable(TABLE_2); + TransactionReadInfo readInfo = + ImmutableTransactionReadInfo.builder().bytesRead(0).kvsCalls(1).build(); + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void multipleCallForTableAreTracked() { + tracker.callForTable(TABLE_2); + tracker.callForTable(TABLE_2); + tracker.callForTable(TABLE_2); + TransactionReadInfo readInfo = + ImmutableTransactionReadInfo.builder().bytesRead(0).kvsCalls(3).build(); + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void multipleCallForTableOnMultipleTablesAreTracked() { + tracker.callForTable(TABLE_2); + tracker.callForTable(TABLE_1); + tracker.callForTable(TABLE_2); + tracker.callForTable(TABLE_3); + tracker.callForTable(TABLE_2); + TransactionReadInfo readInfo = + ImmutableTransactionReadInfo.builder().bytesRead(0).kvsCalls(5).build(); + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void oneTableAgnosticReadIsTracked() { + tracker.tableAgnosticRead(KVS_METHOD_NAME_4, BYTES_READ_3); + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_3) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_4) + .build()) + .kvsCalls(1) + .build(); + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void multipleTableAgnosticReadAreTracked() { + tracker.tableAgnosticRead(KVS_METHOD_NAME_4, BYTES_READ_3); + tracker.tableAgnosticRead(KVS_METHOD_NAME_5, BYTES_READ_3); + tracker.tableAgnosticRead(KVS_METHOD_NAME_4, BYTES_READ_1); + tracker.tableAgnosticRead(KVS_METHOD_NAME_5, BYTES_READ_2); + tracker.tableAgnosticRead(KVS_METHOD_NAME_5, BYTES_READ_1); + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(2 * BYTES_READ_1 + BYTES_READ_2 + 2 * BYTES_READ_3) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_4) + .build()) + .kvsCalls(5) + .build(); + assertEquals(readInfo, tracker.getReadInfo()); + assertEquals(ImmutableMap.of(), tracker.getReadInfoByTable()); + } + + @Test + public void multipleTrackingCallMultipleTablesTest() { + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_1, BYTES_READ_1); + tracker.tableAgnosticRead(KVS_METHOD_NAME_4, BYTES_READ_1); + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_3, BYTES_READ_3); + tracker.callForTable(TABLE_1); + tracker.readForTable(TABLE_1, KVS_METHOD_NAME_2, BYTES_READ_3); + tracker.callForTable(TABLE_2); + tracker.readForTable(TABLE_2, KVS_METHOD_NAME_2, BYTES_READ_2); + tracker.partialReadForTable(TABLE_1, BYTES_READ_3); + tracker.tableAgnosticRead(KVS_METHOD_NAME_5, BYTES_READ_2); + tracker.callForTable(TABLE_1); + tracker.partialReadForTable(TABLE_2, BYTES_READ_1); + tracker.callForTable(TABLE_3); + tracker.partialReadForTable(TABLE_2, BYTES_READ_2); + tracker.partialReadForTable(TABLE_3, BYTES_READ_2); + tracker.partialReadForTable(TABLE_1, BYTES_READ_2); + tracker.readForTable(TABLE_2, KVS_METHOD_NAME_3, BYTES_READ_3); + tracker.readForTable(TABLE_3, KVS_METHOD_NAME_1, BYTES_READ_2); + tracker.partialReadForTable(TABLE_3, BYTES_READ_1); + tracker.partialReadForTable(TABLE_3, BYTES_READ_1); + + TransactionReadInfo readInfo = ImmutableTransactionReadInfo.builder() + .bytesRead(5 * BYTES_READ_1 + 6 * BYTES_READ_2 + 4 * BYTES_READ_3) + .kvsCalls(12) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_2) + .build()) + .build(); + + assertEquals(readInfo, tracker.getReadInfo()); + + ImmutableMap readInfoByTable = + ImmutableMap.builder() + .put( + TABLE_1, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + BYTES_READ_2 + 3 * BYTES_READ_3) + .kvsCalls(5) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_2) + .build()) + .build()) + .put( + TABLE_2, + ImmutableTransactionReadInfo.builder() + .bytesRead(BYTES_READ_1 + 2 * BYTES_READ_2 + BYTES_READ_3) + .kvsCalls(3) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_3) + .methodName(KVS_METHOD_NAME_3) + .build()) + .build()) + .put( + TABLE_3, + ImmutableTransactionReadInfo.builder() + .bytesRead(2 * BYTES_READ_1 + 2 * BYTES_READ_2) + .kvsCalls(2) + .maximumBytesKvsCallInfo(ImmutableKvsCallReadInfo.builder() + .bytesRead(BYTES_READ_2) + .methodName(KVS_METHOD_NAME_1) + .build()) + .build()) + .buildOrThrow(); + + assertEquals(readInfoByTable, tracker.getReadInfoByTable()); + } +} From b0f6609d4f43545c98e0fdaabbd37db3c2830dae Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 22:22:50 +0000 Subject: [PATCH 04/53] adding iterators --- .../TrackingClosableIterator.java | 34 +++++++++ .../impl/expectations/TrackingIterator.java | 46 ++++++++++++ .../TrackingRowColumnRangeIterator.java | 32 ++++++++ .../TrackingClosableIteratorTest.java | 51 +++++++++++++ .../expectations/TrackingIteratorTest.java | 73 +++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIterator.java create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIterator.java create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingRowColumnRangeIterator.java create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIteratorTest.java create mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIteratorTest.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIterator.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIterator.java new file mode 100644 index 00000000000..1adc345334b --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIterator.java @@ -0,0 +1,34 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.palantir.common.base.ClosableIterator; +import java.util.function.Consumer; +import java.util.function.Function; + +public class TrackingClosableIterator extends TrackingIterator> + implements ClosableIterator { + + public TrackingClosableIterator(ClosableIterator delegate, Consumer tracker, Function measurer) { + super(delegate, tracker, measurer); + } + + @Override + public void close() { + delegate().close(); + } +} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIterator.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIterator.java new file mode 100644 index 00000000000..c05de15a78c --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIterator.java @@ -0,0 +1,46 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.google.common.collect.ForwardingIterator; +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.function.Function; + +public class TrackingIterator> extends ForwardingIterator { + I delegate; + Function measurer; + Consumer tracker; + + public TrackingIterator(I delegate, Consumer tracker, Function measurer) { + this.delegate = delegate; + this.tracker = tracker; + this.measurer = measurer; + } + + @Override + protected I delegate() { + return delegate; + } + + @Override + public T next() { + T result = delegate.next(); + tracker.accept(measurer.apply(result)); + return result; + } +} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingRowColumnRangeIterator.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingRowColumnRangeIterator.java new file mode 100644 index 00000000000..f90b9aa8120 --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingRowColumnRangeIterator.java @@ -0,0 +1,32 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.RowColumnRangeIterator; +import com.palantir.atlasdb.keyvalue.api.Value; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +public class TrackingRowColumnRangeIterator extends TrackingIterator, RowColumnRangeIterator> + implements RowColumnRangeIterator { + public TrackingRowColumnRangeIterator( + RowColumnRangeIterator delegate, Consumer tracker, Function, Long> measurer) { + super(delegate, tracker, measurer); + } +} diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIteratorTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIteratorTest.java new file mode 100644 index 00000000000..6d8305b8771 --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingClosableIteratorTest.java @@ -0,0 +1,51 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableList; +import com.palantir.common.base.ClosableIterator; +import com.palantir.common.base.ClosableIterators; +import java.util.function.Consumer; +import java.util.function.Function; +import org.junit.Test; + +public class TrackingClosableIteratorTest { + private static final ImmutableList STRINGS = + ImmutableList.of("test1", "test200", "composite", "", "t", "tt"); + private static final Function MEASURER = Functions.compose(Long::valueOf, String::length); + + @Test + public void trackingClosableStringIteratorDelegatesClose() { + ClosableIterator iterator = spy(spawnClosableIterator()); + TrackingClosableIterator trackingIterator = new TrackingClosableIterator<>(iterator, noOp(), MEASURER); + trackingIterator.close(); + verify(iterator, times(1)).close(); + } + + private static ClosableIterator spawnClosableIterator() { + return ClosableIterators.wrapWithEmptyClose(STRINGS.stream().iterator()); + } + + private static Consumer noOp() { + return t -> {}; + } +} diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIteratorTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIteratorTest.java new file mode 100644 index 00000000000..fb4c14a0a74 --- /dev/null +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingIteratorTest.java @@ -0,0 +1,73 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableList; +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.function.Function; +import org.junit.Test; + +public class TrackingIteratorTest { + private static final ImmutableList STRINGS = + ImmutableList.of("test1", "test200", "composite", "", "t", "tt"); + private static final Function MEASURER = Functions.compose(Long::valueOf, String::length); + + @Test + public void trackingStringIteratorForwardsData() { + Iterator iterator = spawnIterator(); + TrackingIterator> trackingIterator = + new TrackingIterator<>(spawnIterator(), noOp(), MEASURER); + + trackingIterator.forEachRemaining(string -> { + assertThat(iterator.hasNext()).isTrue(); + assertThat(string).isEqualTo(iterator.next()); + }); + + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void trackingStringIteratorTracksData() { + Iterator iterator = spawnIterator(); + + TrackingIterator> trackingIterator = new TrackingIterator<>( + spawnIterator(), + new Consumer() { + final Iterator baseIterator = spawnIterator(); + + @Override + public void accept(Long bytes) { + assertThat(bytes).isEqualTo(MEASURER.apply(baseIterator.next())); + } + }, + MEASURER); + + trackingIterator.forEachRemaining(noOp()); + } + + private static Iterator spawnIterator() { + return STRINGS.stream().iterator(); + } + + private static Consumer noOp() { + return t -> {}; + } +} From 0dbad633558d537bb765e0bc103754a41724c55c Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 22:36:34 +0000 Subject: [PATCH 05/53] putting it all together --- .../TrackingKeyValueServiceImpl.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java new file mode 100644 index 00000000000..12ffa0d8dfd --- /dev/null +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java @@ -0,0 +1,223 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl.expectations; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.palantir.atlasdb.keyvalue.api.BatchColumnRangeSelection; +import com.palantir.atlasdb.keyvalue.api.CandidateCellForSweeping; +import com.palantir.atlasdb.keyvalue.api.CandidateCellForSweepingRequest; +import com.palantir.atlasdb.keyvalue.api.Cell; +import com.palantir.atlasdb.keyvalue.api.ColumnRangeSelection; +import com.palantir.atlasdb.keyvalue.api.ColumnSelection; +import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; +import com.palantir.atlasdb.keyvalue.api.KeyValueService; +import com.palantir.atlasdb.keyvalue.api.RangeRequest; +import com.palantir.atlasdb.keyvalue.api.RowColumnRangeIterator; +import com.palantir.atlasdb.keyvalue.api.RowResult; +import com.palantir.atlasdb.keyvalue.api.TableReference; +import com.palantir.atlasdb.keyvalue.api.Value; +import com.palantir.atlasdb.keyvalue.impl.ForwardingKeyValueService; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import com.palantir.common.base.ClosableIterator; +import com.palantir.common.exception.AtlasDbDependencyException; +import com.palantir.util.paging.TokenBackedBasicResultsPage; +import java.lang.reflect.Array; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +public class TrackingKeyValueServiceImpl extends ForwardingKeyValueService implements TrackingKeyValueService { + KeyValueService delegate; + KeyValueServiceDataTracker tracker; + + private TrackingKeyValueServiceImpl(KeyValueService delegate) { + this.delegate = delegate; + } + + @Override + public KeyValueService delegate() { + return delegate; + } + + @Override + public TransactionReadInfo getOverallReadInfo() { + return tracker.getReadInfo(); + } + + @Override + public ImmutableMap getReadInfoByTable() { + return tracker.getReadInfoByTable(); + } + + @Override + public ListenableFuture> getAsync(TableReference tableRef, Map timestampByCell) { + return Futures.transform( + delegate.getAsync(tableRef, timestampByCell), + valueByCell -> { + tracker.readForTable( + tableRef, "getAsync", ExpectationsMeasuringUtils.valueByCellSizeInBytes(valueByCell)); + return valueByCell; + }, + MoreExecutors.directExecutor()); + } + + @Override + public Map getRows( + TableReference tableRef, Iterable rows, ColumnSelection columnSelection, long timestamp) { + Map result = delegate.getRows(tableRef, rows, columnSelection, timestamp); + tracker.readForTable(tableRef, "getRows", ExpectationsMeasuringUtils.valueByCellSizeInBytes(result)); + return result; + } + + @Override + public Map getRowsColumnRange( + TableReference tableRef, + Iterable rows, + BatchColumnRangeSelection batchColumnRangeSelection, + long timestamp) { + tracker.callForTable(tableRef); + Map result = + delegate.getRowsColumnRange(tableRef, rows, batchColumnRangeSelection, timestamp); + result.replaceAll((rowsRead, iterator) -> new TrackingRowColumnRangeIterator( + iterator, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes)); + return result; + } + + @Override + public RowColumnRangeIterator getRowsColumnRange( + TableReference tableRef, + Iterable rows, + ColumnRangeSelection columnRangeSelection, + int cellBatchHint, + long timestamp) { + tracker.callForTable(tableRef); + RowColumnRangeIterator result = + delegate.getRowsColumnRange(tableRef, rows, columnRangeSelection, cellBatchHint, timestamp); + + return new TrackingRowColumnRangeIterator( + result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes); + } + + @Override + public Map get(TableReference tableRef, Map timestampByCell) { + Map result = delegate.get(tableRef, timestampByCell); + tracker.readForTable(tableRef, "get", ExpectationsMeasuringUtils.valueByCellSizeInBytes(result)); + return result; + } + + @Override + public Map getLatestTimestamps(TableReference tableRef, Map timestampByCell) { + Map result = delegate.getLatestTimestamps(tableRef, timestampByCell); + tracker.readForTable(tableRef, "getLatestTimestamps", ExpectationsMeasuringUtils.sizeInBytes(result)); + return result; + } + + @Override + public ClosableIterator> getRange( + TableReference tableRef, RangeRequest rangeRequest, long timestamp) { + tracker.callForTable(tableRef); + try (ClosableIterator> result = delegate.getRange(tableRef, rangeRequest, timestamp)) { + return new TrackingClosableIterator<>( + result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes); + } + } + + @Override + public ClosableIterator>> getRangeOfTimestamps( + TableReference tableRef, RangeRequest rangeRequest, long timestamp) + throws InsufficientConsistencyException { + tracker.callForTable(tableRef); + try (ClosableIterator>> result = + delegate.getRangeOfTimestamps(tableRef, rangeRequest, timestamp)) { + return new TrackingClosableIterator<>( + result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::setResultSizeInBytes); + } + } + + @Override + public ClosableIterator> getCandidateCellsForSweeping( + TableReference tableRef, CandidateCellForSweepingRequest request) { + tracker.callForTable(tableRef); + try (ClosableIterator> result = + delegate.getCandidateCellsForSweeping(tableRef, request)) { + return new TrackingClosableIterator<>( + result, partialReadForTableConsumer(tableRef), candidates -> candidates.stream() + .mapToLong(CandidateCellForSweeping::sizeInBytes) + .sum()); + } + } + + @Override + public Map, byte[]>> getFirstBatchForRanges( + TableReference tableRef, Iterable rangeRequests, long timestamp) { + Map, byte[]>> result = + delegate.getFirstBatchForRanges(tableRef, rangeRequests, timestamp); + tracker.readForTable( + tableRef, "getFirstBatchForRanges", ExpectationsMeasuringUtils.pageByRequestSizeInBytes(result)); + return result; + } + + @Override + public Set getAllTableNames() { + Set result = delegate.getAllTableNames(); + tracker.tableAgnosticRead( + "getAllTableNames", + result.stream().mapToLong(TableReference::sizeInBytes).sum()); + return result; + } + + @Override + public byte[] getMetadataForTable(TableReference tableRef) { + byte[] result = delegate.getMetadataForTable(tableRef); + tracker.tableAgnosticRead("getMetadataForTable", result.length); + return result; + } + + @Override + public Map getMetadataForTables() { + Map result = delegate.getMetadataForTables(); + tracker.tableAgnosticRead("getMetadataForTables", ExpectationsMeasuringUtils.arrayByRefSizeInBytes(result)); + return result; + } + + @Override + public Multimap getAllTimestamps(TableReference tableRef, Set cells, long timestamp) + throws AtlasDbDependencyException { + Multimap result = delegate.getAllTimestamps(tableRef, cells, timestamp); + tracker.readForTable(tableRef, "getAllTimestamps", ExpectationsMeasuringUtils.sizeInBytes(result)); + return result; + } + + @Override + public List getRowKeysInRange(TableReference tableRef, byte[] startRow, byte[] endRow, int maxResults) { + List result = delegate.getRowKeysInRange(tableRef, startRow, endRow, maxResults); + tracker.readForTable( + tableRef, + "getRowKeysInRange", + result.stream().mapToLong(Array::getLength).sum()); + return result; + } + + private Consumer partialReadForTableConsumer(TableReference tableRef) { + return bytes -> tracker.partialReadForTable(tableRef, bytes); + } +} From 49f106ee89a4ac079646c8e6bdef1946c198fe4f Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Sun, 30 Oct 2022 23:07:55 +0000 Subject: [PATCH 06/53] fix constructor --- .../impl/expectations/TrackingKeyValueServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java index 12ffa0d8dfd..7d42b18b2bf 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java @@ -49,7 +49,7 @@ public class TrackingKeyValueServiceImpl extends ForwardingKeyValueService imple KeyValueService delegate; KeyValueServiceDataTracker tracker; - private TrackingKeyValueServiceImpl(KeyValueService delegate) { + public TrackingKeyValueServiceImpl(KeyValueService delegate) { this.delegate = delegate; } From 89ae9f0380382bef398796824f5ec5a4ea5cde29 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 31 Oct 2022 14:29:03 +0000 Subject: [PATCH 07/53] progress --- .../transaction/api/OpenTransaction.java | 3 +- .../ExpectationsAwareTransaction.java | 7 +- .../expectations/ExpectationsViolation.java | 2 +- .../impl/CallbackAwareTransaction.java | 4 +- .../ForwardingCallbackAwareTransaction.java | 2 +- ...orwardingExpectationsAwareTransaction.java | 65 ++++++++++++++++++ .../transaction/impl/SnapshotTransaction.java | 68 ++++++++++++++++++- .../impl/SnapshotTransactionManager.java | 50 +++++++++++++- .../expectations/ExpectationsManager.java | 2 + .../expectations/ExpectationsManagerImpl.java | 1 + .../impl/expectations/ExpectationsTask.java | 2 + .../TrackingKeyValueServiceImpl.java | 2 +- .../src/main/metrics/expectations.yml | 19 ++++++ .../expectations/ExpectationsManagerTest.java | 1 + 14 files changed, 213 insertions(+), 15 deletions(-) rename {atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl => atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api}/expectations/ExpectationsAwareTransaction.java (81%) rename {atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl => atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api}/expectations/ExpectationsViolation.java (94%) create mode 100644 atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java create mode 100644 atlasdb-impl-shared/src/main/metrics/expectations.yml diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java index a477f5e5a95..54cf1e3f30c 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java @@ -17,8 +17,9 @@ package com.palantir.atlasdb.transaction.api; import com.palantir.atlasdb.metrics.Timed; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -public interface OpenTransaction extends Transaction { +public interface OpenTransaction extends ExpectationsAwareTransaction { /** * Runs a provided task, commits the transaction, and performs cleanup. If no further work needs to be done with the diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java similarity index 81% rename from atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java rename to atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 4657c32acf3..e86b00b1b8c 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -14,12 +14,9 @@ * limitations under the License. */ -package com.palantir.atlasdb.transaction.impl.expectations; +package com.palantir.atlasdb.transaction.api.expectations; import com.palantir.atlasdb.transaction.api.Transaction; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; -import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; import java.util.Set; /** @@ -39,4 +36,6 @@ public interface ExpectationsAwareTransaction extends Transaction { void runExpectationsCallbacks(); Set checkAndGetViolations(); + + void reportExpectationsCollectedData(); } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsViolation.java similarity index 94% rename from atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java rename to atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsViolation.java index f72e23d6b22..f45b409fb42 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsViolation.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.palantir.atlasdb.transaction.impl.expectations; +package com.palantir.atlasdb.transaction.api.expectations; import org.derive4j.Data; diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java index 59afd8bae10..32c02bc9c52 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java @@ -17,8 +17,8 @@ package com.palantir.atlasdb.transaction.impl; import com.palantir.atlasdb.transaction.api.PreCommitCondition; -import com.palantir.atlasdb.transaction.api.Transaction; import com.palantir.atlasdb.transaction.api.TransactionFailedException; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.service.TransactionService; /** @@ -26,7 +26,7 @@ * are run. This can be used to e.g run {@link PreCommitCondition#cleanup()} before the onSuccess callbacks * are run. */ -public interface CallbackAwareTransaction extends Transaction { +public interface CallbackAwareTransaction extends ExpectationsAwareTransaction { void commitWithoutCallbacks() throws TransactionFailedException; void commitWithoutCallbacks(TransactionService transactionService) throws TransactionFailedException; diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingCallbackAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingCallbackAwareTransaction.java index fd106669d73..71c4c8f8ff1 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingCallbackAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingCallbackAwareTransaction.java @@ -19,7 +19,7 @@ import com.palantir.atlasdb.transaction.api.TransactionFailedException; import com.palantir.atlasdb.transaction.service.TransactionService; -public abstract class ForwardingCallbackAwareTransaction extends ForwardingTransaction +public abstract class ForwardingCallbackAwareTransaction extends ForwardingExpectationsAwareTransaction implements CallbackAwareTransaction { @Override diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java new file mode 100644 index 00000000000..986c099478b --- /dev/null +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -0,0 +1,65 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.impl; + +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import java.util.Set; + +public abstract class ForwardingExpectationsAwareTransaction extends ForwardingTransaction + implements ExpectationsAwareTransaction { + @Override + public abstract ExpectationsAwareTransaction delegate(); + + @Override + public ExpectationsConfig expectationsConfig() { + return delegate().expectationsConfig(); + } + + @Override + public long getAgeMillis() { + return delegate().getAgeMillis(); + } + + @Override + public TransactionReadInfo getReadInfo() { + return delegate().getReadInfo(); + } + + @Override + public ExpectationsStatistics getCallbackStatistics() { + return delegate().getCallbackStatistics(); + } + + @Override + public void runExpectationsCallbacks() { + delegate().runExpectationsCallbacks(); + } + + @Override + public Set checkAndGetViolations() { + return delegate().checkAndGetViolations(); + } + + @Override + public void reportExpectationsCollectedData() { + delegate().reportExpectationsCollectedData(); + } +} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 4cfecde9fc1..f4c709b8d03 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -25,6 +25,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Collections2; import com.google.common.collect.FluentIterable; @@ -96,6 +97,14 @@ import com.palantir.atlasdb.transaction.api.TransactionLockAcquisitionTimeoutException; import com.palantir.atlasdb.transaction.api.TransactionLockTimeoutException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableExpectationsStatistics; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; +import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; +import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.TransactionOutcomeMetrics; import com.palantir.atlasdb.transaction.knowledge.TransactionKnowledgeComponents; @@ -174,6 +183,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nullable; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.tuple.Pair; @@ -216,7 +226,7 @@ private enum State { protected final TimelockService timelockService; protected final LockWatchManagerInternal lockWatchManager; - final KeyValueService keyValueService; + final TrackingKeyValueService keyValueService; final AsyncKeyValueService immediateKeyValueService; final TransactionService defaultTransactionService; private final AsyncTransactionService immediateTransactionService; @@ -265,6 +275,8 @@ private enum State { protected final TimestampCache timestampCache; protected final TransactionKnowledgeComponents knowledge; + protected final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; + protected final Stopwatch stopwatch = Stopwatch.createStarted(); /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read @@ -273,7 +285,7 @@ private enum State { */ /* package */ SnapshotTransaction( MetricsManager metricsManager, - KeyValueService keyValueService, + KeyValueService tmKeyValueService, TimelockService timelockService, LockWatchManagerInternal lockWatchManager, TransactionService transactionService, @@ -302,7 +314,7 @@ private enum State { this.lockWatchManager = lockWatchManager; this.conflictTracer = conflictTracer; this.transactionTimerContext = getTimer("transactionMillis").time(); - this.keyValueService = keyValueService; + this.keyValueService = new TrackingKeyValueServiceImpl(tmKeyValueService); this.immediateKeyValueService = KeyValueServices.synchronousAsAsyncKeyValueService(keyValueService); this.timelockService = timelockService; this.defaultTransactionService = transactionService; @@ -338,6 +350,8 @@ private enum State { timelockService, immutableTimestamp, knowledge); + this.expectationsDataCollectionMetrics = + ExpectationsDataCollectionMetrics.of(metricsManager.getTaggedRegistry()); } protected TransactionScopedCache getCache() { @@ -2576,6 +2590,54 @@ private Multimap getCellsToScrubByTable(State expectedStat return tableRefToCells; } + // todo(aalouane) + @Override + public ExpectationsConfig expectationsConfig() { + throw new NotImplementedException(); + } + + @Override + public long getAgeMillis() { + stopwatch.stop(); + return stopwatch.elapsed(TimeUnit.MILLISECONDS); + } + + @Override + public TransactionReadInfo getReadInfo() { + return keyValueService.getOverallReadInfo(); + } + + @Override + public ExpectationsStatistics getCallbackStatistics() { + return ImmutableExpectationsStatistics.builder() + .transactionAgeMillis(getAgeMillis()) + .readInfoByTable(keyValueService.getReadInfoByTable()) + .build(); + } + + // todo(aalouane) + @Override + public void runExpectationsCallbacks() { + throw new NotImplementedException(); + } + + // todo(aalouane) + @Override + public Set checkAndGetViolations() { + throw new NotImplementedException(); + } + + @Override + public void reportExpectationsCollectedData() { + expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); + TransactionReadInfo info = getReadInfo(); + expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); + expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); + info.maximumBytesKvsCallInfo() + .ifPresent(kvsReadInfo -> + expectationsDataCollectionMetrics.worstKvsBytesRead().update(kvsReadInfo.bytesRead())); + } + private Timer getTimer(String name) { return metricsManager.registerOrGetTimer(SnapshotTransaction.class, name); } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index 38168edfceb..415f63adfdf 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -45,6 +45,10 @@ import com.palantir.atlasdb.transaction.api.TransactionFailedRetriableException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.TransactionTask; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; import com.palantir.atlasdb.transaction.impl.metrics.MemoizingTableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.MetricsFilterEvaluationContext; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -68,6 +72,7 @@ import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; @@ -183,8 +188,16 @@ public T runTaskWithCondi condition.cleanup(); throw e; } - return openTransaction.finishWithCallback( - transaction -> task.execute(transaction, condition), condition::cleanup); + + T result; + try { + result = openTransaction.finishWithCallback( + transaction -> task.execute(transaction, condition), condition::cleanup); + } finally { + openTransaction.reportExpectationsCollectedData(); + } + + return result; } @Override @@ -278,6 +291,39 @@ public T finishWithCallback(TransactionTask task, postTaskContext.stop(); return result; } + + @Override + public ExpectationsConfig expectationsConfig() { + return delegate.expectationsConfig(); + } + + @Override + public long getAgeMillis() { + return delegate.getAgeMillis(); + } + + @Override + public TransactionReadInfo getReadInfo() { + return delegate.getReadInfo(); + } + + @Override + public ExpectationsStatistics getCallbackStatistics() { + return delegate.getCallbackStatistics(); + } + + @Override + public void runExpectationsCallbacks() {} + + @Override + public Set checkAndGetViolations() { + return delegate.checkAndGetViolations(); + } + + @Override + public void reportExpectationsCollectedData() { + delegate.reportExpectationsCollectedData(); + } } private void scrubForAggressiveHardDelete(SnapshotTransaction tx) { diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java index bee3d8b6ce8..6b748ea699d 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java @@ -16,6 +16,8 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; + public interface ExpectationsManager extends AutoCloseable { void register(ExpectationsAwareTransaction transaction); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java index 2e7fe261319..23052ddb347 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import java.time.Duration; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java index 913cf6829d4..e4a40d7e554 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java @@ -16,6 +16,8 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java index 7d42b18b2bf..60ab7d25b26 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java @@ -47,7 +47,7 @@ public class TrackingKeyValueServiceImpl extends ForwardingKeyValueService implements TrackingKeyValueService { KeyValueService delegate; - KeyValueServiceDataTracker tracker; + KeyValueServiceDataTracker tracker = new KeyValueServiceDataTracker(); public TrackingKeyValueServiceImpl(KeyValueService delegate) { this.delegate = delegate; diff --git a/atlasdb-impl-shared/src/main/metrics/expectations.yml b/atlasdb-impl-shared/src/main/metrics/expectations.yml new file mode 100644 index 00000000000..bdb5654df21 --- /dev/null +++ b/atlasdb-impl-shared/src/main/metrics/expectations.yml @@ -0,0 +1,19 @@ +options: + javaPackage: 'com.palantir.atlasdb.transaction.expectations' + +namespaces: + expectationsDataCollection: + docs: Data collection on transaction patterns + metrics: + bytesRead: + docs: Bytes read by a transaction through kvs read calls + type: histogram + kvsCalls: + docs: Number of kvs read calls made by a transaction + type: histogram + ageMillis: + docs: Elapsed duration in milliseconds for a transaction + type: histogram + worstKvsBytesRead: + docs: Most bytes read by a transaction in one kvs read call + type: histogram diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java index eea190c98ed..9fa58f6cae6 100644 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import java.util.Set; import java.util.concurrent.TimeUnit; import org.jmock.lib.concurrent.DeterministicScheduler; From 2c97dbfe7e3f67426b40b119eabb371467039c77 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 31 Oct 2022 14:55:19 +0000 Subject: [PATCH 08/53] progress --- .../api/expectations/ExpectationsAwareTransaction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index e86b00b1b8c..778d553a68f 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -27,6 +27,7 @@ public interface ExpectationsAwareTransaction extends Transaction { ExpectationsConfig expectationsConfig(); + // todo(aalouane): pass boolean to indicate whether this should freeze any time keeping long getAgeMillis(); TransactionReadInfo getReadInfo(); From 4c7fb19815da8ccb45f141a8db8da75736f8ea3c Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 31 Oct 2022 17:04:02 +0000 Subject: [PATCH 09/53] checkpoint --- .../atlasdb/transaction/impl/SnapshotTransactionManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index 415f63adfdf..e69b6805272 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -45,6 +45,7 @@ import com.palantir.atlasdb.transaction.api.TransactionFailedRetriableException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.TransactionTask; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; @@ -248,7 +249,7 @@ public List startTransactions(List Date: Tue, 1 Nov 2022 10:57:21 +0000 Subject: [PATCH 10/53] more wiring --- .../impl/SnapshotTransactionManager.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index e69b6805272..6f3d4c3a626 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -421,10 +421,16 @@ private T runTaskWithCond conflictTracer, tableLevelMetricsController, knowledge); - return runTaskThrowOnConflictWithCallback( - txn -> task.execute(txn, condition), - new ReadTransaction(transaction, sweepStrategyManager), - condition::cleanup); + T result; + try { + result = runTaskThrowOnConflictWithCallback( + txn -> task.execute(txn, condition), + new ReadTransaction(transaction, sweepStrategyManager), + condition::cleanup); + } finally { + transaction.reportExpectationsCollectedData(); + } + return result; } @Override From f08fd070ae17fa3e9a4e16791f6398dbcd6d34ad Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 1 Nov 2022 13:07:39 +0000 Subject: [PATCH 11/53] stuff --- .../expectations/ExpectationsAwareTransaction.java | 3 ++- .../impl/ForwardingExpectationsAwareTransaction.java | 5 +++++ .../atlasdb/transaction/impl/SnapshotTransaction.java | 11 ++++++++--- .../transaction/impl/SnapshotTransactionManager.java | 5 +++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 778d553a68f..a32b90d9fa8 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -27,9 +27,10 @@ public interface ExpectationsAwareTransaction extends Transaction { ExpectationsConfig expectationsConfig(); - // todo(aalouane): pass boolean to indicate whether this should freeze any time keeping long getAgeMillis(); + long getAgeMillisAndFreezeTimer(); + TransactionReadInfo getReadInfo(); ExpectationsStatistics getCallbackStatistics(); diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index 986c099478b..de67211c784 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -38,6 +38,11 @@ public long getAgeMillis() { return delegate().getAgeMillis(); } + @Override + public long getAgeMillisAndFreezeTimer() { + return delegate().getAgeMillisAndFreezeTimer(); + } + @Override public TransactionReadInfo getReadInfo() { return delegate().getReadInfo(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index f4c709b8d03..792a79ab808 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2598,10 +2598,15 @@ public ExpectationsConfig expectationsConfig() { @Override public long getAgeMillis() { - stopwatch.stop(); return stopwatch.elapsed(TimeUnit.MILLISECONDS); } + @Override + public long getAgeMillisAndFreezeTimer() { + stopwatch.stop(); + return getAgeMillis(); + } + @Override public TransactionReadInfo getReadInfo() { return keyValueService.getOverallReadInfo(); @@ -2610,7 +2615,7 @@ public TransactionReadInfo getReadInfo() { @Override public ExpectationsStatistics getCallbackStatistics() { return ImmutableExpectationsStatistics.builder() - .transactionAgeMillis(getAgeMillis()) + .transactionAgeMillis(getAgeMillisAndFreezeTimer()) .readInfoByTable(keyValueService.getReadInfoByTable()) .build(); } @@ -2629,7 +2634,7 @@ public Set checkAndGetViolations() { @Override public void reportExpectationsCollectedData() { - expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); + expectationsDataCollectionMetrics.ageMillis().update(getAgeMillisAndFreezeTimer()); TransactionReadInfo info = getReadInfo(); expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index 6f3d4c3a626..f6a39dc69c1 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -303,6 +303,11 @@ public long getAgeMillis() { return delegate.getAgeMillis(); } + @Override + public long getAgeMillisAndFreezeTimer() { + return delegate.getAgeMillisAndFreezeTimer(); + } + @Override public TransactionReadInfo getReadInfo() { return delegate.getReadInfo(); From 5e2fc66a6e830bc095a1e380c4f88bd9ca749c55 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 11 Nov 2022 20:11:58 +0000 Subject: [PATCH 12/53] removing obsolete code --- .../ExpectationsMeasuringUtils.java | 92 -------- .../TrackingKeyValueServiceImpl.java | 176 --------------- .../ExpectationsMeasuringUtilsTest.java | 211 ------------------ 3 files changed, 479 deletions(-) delete mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java delete mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java deleted file mode 100644 index f248682e29b..00000000000 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtils.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import com.google.common.collect.Multimap; -import com.palantir.atlasdb.keyvalue.api.Cell; -import com.palantir.atlasdb.keyvalue.api.RangeRequest; -import com.palantir.atlasdb.keyvalue.api.RowResult; -import com.palantir.atlasdb.keyvalue.api.TableReference; -import com.palantir.atlasdb.keyvalue.api.Value; -import com.palantir.util.paging.TokenBackedBasicResultsPage; -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.function.Function; - -public final class ExpectationsMeasuringUtils { - private ExpectationsMeasuringUtils() {} - - public static long sizeInBytes(Entry entry) { - return entry.getKey().sizeInBytes() + entry.getValue().sizeInBytes(); - } - - public static long sizeInBytes(Multimap valueByCell) { - return valueByCell.keys().stream() - .mapToLong(cell -> cell.sizeInBytes() + Long.BYTES) - .sum(); - } - - public static long sizeInBytes(Map longByCell) { - return longByCell.keySet().stream() - .mapToLong(cell -> cell.sizeInBytes() + Long.BYTES) - .sum(); - } - - public static long sizeInBytes(RowResult rowResult) { - return sizeInBytes(rowResult, Value::sizeInBytes); - } - - private static long sizeInBytes(RowResult rowResult, Function measurer) { - return rowResult.getRowNameSize() - + rowResult.getColumns().entrySet().stream() - .mapToLong(entry -> entry.getKey().length + measurer.apply(entry.getValue())) - .sum(); - } - - public static long arrayByRefSizeInBytes(Map arrayByRef) { - return arrayByRef.entrySet().stream() - .mapToLong(entry -> entry.getKey().sizeInBytes() + entry.getValue().length) - .sum(); - } - - public static long valueByCellSizeInBytes(Map valueByCell) { - return valueByCell.entrySet().stream() - .mapToLong(ExpectationsMeasuringUtils::sizeInBytes) - .sum(); - } - - /** - * Ignoring the size of token for next page because the interface method - * {@link TokenBackedBasicResultsPage#getTokenForNextPage} might have side effects (e.g. interact with the kvs). - * The interface specification is ambiguous. Also, the current token is not exposed. - * todo(aalouane): SG search - */ - public static long pageByRequestSizeInBytes( - Map, byte[]>> pageByRange) { - return pageByRange.values().stream() - .map(TokenBackedBasicResultsPage::getResults) - .flatMap(Collection::stream) - .mapToLong(ExpectationsMeasuringUtils::sizeInBytes) - .sum(); - } - - public static long setResultSizeInBytes(RowResult> rowResult) { - return sizeInBytes(rowResult, set -> (long) set.size() * Long.BYTES); - } -} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java index 60ab7d25b26..0efdf9f4d7a 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java @@ -17,33 +17,10 @@ package com.palantir.atlasdb.transaction.impl.expectations; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import com.palantir.atlasdb.keyvalue.api.BatchColumnRangeSelection; -import com.palantir.atlasdb.keyvalue.api.CandidateCellForSweeping; -import com.palantir.atlasdb.keyvalue.api.CandidateCellForSweepingRequest; -import com.palantir.atlasdb.keyvalue.api.Cell; -import com.palantir.atlasdb.keyvalue.api.ColumnRangeSelection; -import com.palantir.atlasdb.keyvalue.api.ColumnSelection; -import com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException; import com.palantir.atlasdb.keyvalue.api.KeyValueService; -import com.palantir.atlasdb.keyvalue.api.RangeRequest; -import com.palantir.atlasdb.keyvalue.api.RowColumnRangeIterator; -import com.palantir.atlasdb.keyvalue.api.RowResult; import com.palantir.atlasdb.keyvalue.api.TableReference; -import com.palantir.atlasdb.keyvalue.api.Value; import com.palantir.atlasdb.keyvalue.impl.ForwardingKeyValueService; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.common.base.ClosableIterator; -import com.palantir.common.exception.AtlasDbDependencyException; -import com.palantir.util.paging.TokenBackedBasicResultsPage; -import java.lang.reflect.Array; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; public class TrackingKeyValueServiceImpl extends ForwardingKeyValueService implements TrackingKeyValueService { KeyValueService delegate; @@ -67,157 +44,4 @@ public TransactionReadInfo getOverallReadInfo() { public ImmutableMap getReadInfoByTable() { return tracker.getReadInfoByTable(); } - - @Override - public ListenableFuture> getAsync(TableReference tableRef, Map timestampByCell) { - return Futures.transform( - delegate.getAsync(tableRef, timestampByCell), - valueByCell -> { - tracker.readForTable( - tableRef, "getAsync", ExpectationsMeasuringUtils.valueByCellSizeInBytes(valueByCell)); - return valueByCell; - }, - MoreExecutors.directExecutor()); - } - - @Override - public Map getRows( - TableReference tableRef, Iterable rows, ColumnSelection columnSelection, long timestamp) { - Map result = delegate.getRows(tableRef, rows, columnSelection, timestamp); - tracker.readForTable(tableRef, "getRows", ExpectationsMeasuringUtils.valueByCellSizeInBytes(result)); - return result; - } - - @Override - public Map getRowsColumnRange( - TableReference tableRef, - Iterable rows, - BatchColumnRangeSelection batchColumnRangeSelection, - long timestamp) { - tracker.callForTable(tableRef); - Map result = - delegate.getRowsColumnRange(tableRef, rows, batchColumnRangeSelection, timestamp); - result.replaceAll((rowsRead, iterator) -> new TrackingRowColumnRangeIterator( - iterator, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes)); - return result; - } - - @Override - public RowColumnRangeIterator getRowsColumnRange( - TableReference tableRef, - Iterable rows, - ColumnRangeSelection columnRangeSelection, - int cellBatchHint, - long timestamp) { - tracker.callForTable(tableRef); - RowColumnRangeIterator result = - delegate.getRowsColumnRange(tableRef, rows, columnRangeSelection, cellBatchHint, timestamp); - - return new TrackingRowColumnRangeIterator( - result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes); - } - - @Override - public Map get(TableReference tableRef, Map timestampByCell) { - Map result = delegate.get(tableRef, timestampByCell); - tracker.readForTable(tableRef, "get", ExpectationsMeasuringUtils.valueByCellSizeInBytes(result)); - return result; - } - - @Override - public Map getLatestTimestamps(TableReference tableRef, Map timestampByCell) { - Map result = delegate.getLatestTimestamps(tableRef, timestampByCell); - tracker.readForTable(tableRef, "getLatestTimestamps", ExpectationsMeasuringUtils.sizeInBytes(result)); - return result; - } - - @Override - public ClosableIterator> getRange( - TableReference tableRef, RangeRequest rangeRequest, long timestamp) { - tracker.callForTable(tableRef); - try (ClosableIterator> result = delegate.getRange(tableRef, rangeRequest, timestamp)) { - return new TrackingClosableIterator<>( - result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::sizeInBytes); - } - } - - @Override - public ClosableIterator>> getRangeOfTimestamps( - TableReference tableRef, RangeRequest rangeRequest, long timestamp) - throws InsufficientConsistencyException { - tracker.callForTable(tableRef); - try (ClosableIterator>> result = - delegate.getRangeOfTimestamps(tableRef, rangeRequest, timestamp)) { - return new TrackingClosableIterator<>( - result, partialReadForTableConsumer(tableRef), ExpectationsMeasuringUtils::setResultSizeInBytes); - } - } - - @Override - public ClosableIterator> getCandidateCellsForSweeping( - TableReference tableRef, CandidateCellForSweepingRequest request) { - tracker.callForTable(tableRef); - try (ClosableIterator> result = - delegate.getCandidateCellsForSweeping(tableRef, request)) { - return new TrackingClosableIterator<>( - result, partialReadForTableConsumer(tableRef), candidates -> candidates.stream() - .mapToLong(CandidateCellForSweeping::sizeInBytes) - .sum()); - } - } - - @Override - public Map, byte[]>> getFirstBatchForRanges( - TableReference tableRef, Iterable rangeRequests, long timestamp) { - Map, byte[]>> result = - delegate.getFirstBatchForRanges(tableRef, rangeRequests, timestamp); - tracker.readForTable( - tableRef, "getFirstBatchForRanges", ExpectationsMeasuringUtils.pageByRequestSizeInBytes(result)); - return result; - } - - @Override - public Set getAllTableNames() { - Set result = delegate.getAllTableNames(); - tracker.tableAgnosticRead( - "getAllTableNames", - result.stream().mapToLong(TableReference::sizeInBytes).sum()); - return result; - } - - @Override - public byte[] getMetadataForTable(TableReference tableRef) { - byte[] result = delegate.getMetadataForTable(tableRef); - tracker.tableAgnosticRead("getMetadataForTable", result.length); - return result; - } - - @Override - public Map getMetadataForTables() { - Map result = delegate.getMetadataForTables(); - tracker.tableAgnosticRead("getMetadataForTables", ExpectationsMeasuringUtils.arrayByRefSizeInBytes(result)); - return result; - } - - @Override - public Multimap getAllTimestamps(TableReference tableRef, Set cells, long timestamp) - throws AtlasDbDependencyException { - Multimap result = delegate.getAllTimestamps(tableRef, cells, timestamp); - tracker.readForTable(tableRef, "getAllTimestamps", ExpectationsMeasuringUtils.sizeInBytes(result)); - return result; - } - - @Override - public List getRowKeysInRange(TableReference tableRef, byte[] startRow, byte[] endRow, int maxResults) { - List result = delegate.getRowKeysInRange(tableRef, startRow, endRow, maxResults); - tracker.readForTable( - tableRef, - "getRowKeysInRange", - result.stream().mapToLong(Array::getLength).sum()); - return result; - } - - private Consumer partialReadForTableConsumer(TableReference tableRef) { - return bytes -> tracker.partialReadForTable(tableRef, bytes); - } } diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java deleted file mode 100644 index fddacc200ef..00000000000 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsMeasuringUtilsTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.collect.Multimap; -import com.palantir.atlasdb.keyvalue.api.Cell; -import com.palantir.atlasdb.keyvalue.api.RangeRequest; -import com.palantir.atlasdb.keyvalue.api.RowResult; -import com.palantir.atlasdb.keyvalue.api.TableReference; -import com.palantir.atlasdb.keyvalue.api.Value; -import com.palantir.util.paging.SimpleTokenBackedResultsPage; -import com.palantir.util.paging.TokenBackedBasicResultsPage; -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.LongStream; -import java.util.stream.Stream; -import org.apache.commons.lang3.StringUtils; -import org.junit.Test; - -public class ExpectationsMeasuringUtilsTest { - private static final byte BYTE = (byte) 0xa; - private static final int SIZE_1 = 123; - private static final int SIZE_2 = 214; - private static final int SIZE_3 = 329; - private static final int TIMESTAMP = 100; - - @Test - public void emptyValueByCellMapSizeIsZero() { - assertThat(ExpectationsMeasuringUtils.valueByCellSizeInBytes(Map.of())) - .isEqualTo(0); - } - - @Test - public void emptyLongByCellMapSizeInBytesIsZero() { - assertThat(ExpectationsMeasuringUtils.sizeInBytes(Map.of())).isEqualTo(0); - } - - @Test - public void emptyLongByCellMultimapSizeInBytesIsZero() { - assertThat(ExpectationsMeasuringUtils.sizeInBytes(ArrayListMultimap.create())) - .isEqualTo(0); - } - - @Test - public void emptyByteArrayByTableReferenceMapSizeInBytesIsZero() { - assertThat(ExpectationsMeasuringUtils.arrayByRefSizeInBytes(Map.of())) - .isEqualTo(0); - } - - @Test - public void emptyTokenPageByRangeRequestMapSizeInBytesIsZero() { - assertThat(ExpectationsMeasuringUtils.pageByRequestSizeInBytes( - Map., byte[]>>of())) - .isEqualTo(0); - } - - @Test - public void testLongByCellMapSize() { - Map longByCell = IntStream.range(1, SIZE_1 + 1) - .boxed() - .collect(Collectors.toUnmodifiableMap(ExpectationsMeasuringUtilsTest::createCell, Long::valueOf)); - - // expected value follows from https://en.wikipedia.org/wiki/1_%2B_2_%2B_3_%2B_4_%2B_%E2%8B%AF - assertThat(ExpectationsMeasuringUtils.sizeInBytes(longByCell)) - .isEqualTo(Long.sum(Long.BYTES, (long) SIZE_1 + 1) * SIZE_1); - } - - @Test - public void testValueByCellMapSize() { - Map valueByCell = IntStream.range(1, SIZE_1 + 1) - .boxed() - .collect(Collectors.toUnmodifiableMap( - ExpectationsMeasuringUtilsTest::createCell, size -> createValue(2 * size))); - - assertThat(ExpectationsMeasuringUtils.valueByCellSizeInBytes(valueByCell)) - .isEqualTo((2L * SIZE_1 + Long.BYTES + 2) * SIZE_1); - } - - @Test - public void testValueByLongMultimapSize() { - Multimap valueByLong = IntStream.range(0, SIZE_2 * SIZE_1) - .boxed() - .collect(ImmutableSetMultimap.toImmutableSetMultimap( - valueInt -> createCell(1 + (valueInt % SIZE_1)), Long::valueOf)); - - assertThat(ExpectationsMeasuringUtils.sizeInBytes(valueByLong)) - .isEqualTo(SIZE_2 * SIZE_1 * (SIZE_1 + Long.BYTES + 1L)); - } - - @Test - public void testArrayByTableRefSize() { - Map arrayByTableRef = IntStream.range(1, 2 * SIZE_1 + 1) - .boxed() - .collect(Collectors.toUnmodifiableMap( - ExpectationsMeasuringUtilsTest::createTableReference, - ExpectationsMeasuringUtilsTest::spawnBytes)); - - assertThat(ExpectationsMeasuringUtils.arrayByRefSizeInBytes(arrayByTableRef)) - .isEqualTo((Character.BYTES + 1L) * (2L * SIZE_1 + 1L) * SIZE_1); - } - - @Test - public void testValueRowResult() { - assertThat(ExpectationsMeasuringUtils.sizeInBytes(RowResult.of(createCell(SIZE_1), createValue(SIZE_1)))) - .isEqualTo(3L * SIZE_1 + Long.BYTES); - - assertThat(ExpectationsMeasuringUtils.sizeInBytes(RowResult.of(createCell(SIZE_2), createValue(SIZE_2)))) - .isEqualTo(3L * SIZE_2 + Long.BYTES); - } - - @Test - public void testLongSetRowResultSize() { - assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of()))) - .isEqualTo(2L * SIZE_1); - - assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of(1L)))) - .isEqualTo(2L * SIZE_1 + Long.BYTES); - - assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of(createCell(SIZE_1), Set.of(1L, 2L)))) - .isEqualTo(2L * SIZE_1 + 2L * Long.BYTES); - - assertThat(ExpectationsMeasuringUtils.setResultSizeInBytes(RowResult.of( - createCell(SIZE_1), LongStream.range(0, SIZE_1).boxed().collect(Collectors.toSet())))) - .isEqualTo(SIZE_1 * (Long.BYTES + 2L)); - } - - @Test - public void testValueByCellEntrySize() { - assertThat(ExpectationsMeasuringUtils.sizeInBytes( - new SimpleImmutableEntry<>(createCell(SIZE_1), createValue(SIZE_1)))) - .isEqualTo(3L * SIZE_1 + Long.BYTES); - - assertThat(ExpectationsMeasuringUtils.sizeInBytes( - new SimpleImmutableEntry<>(createCell(SIZE_2), createValue(SIZE_2)))) - .isEqualTo(3L * SIZE_2 + Long.BYTES); - } - - @Test - public void testPageByRequestSize() { - assertThat(ExpectationsMeasuringUtils.pageByRequestSizeInBytes( - ImmutableMap., byte[]>>builder() - .put(RangeRequest.all(), createPage(SIZE_2, SIZE_2, SIZE_2, 0)) - .put( - RangeRequest.builder() - .startRowInclusive(spawnBytes(SIZE_3)) - .build(), - createPage(SIZE_2, SIZE_2, SIZE_2, 1)) - .put( - RangeRequest.builder() - .endRowExclusive(spawnBytes(SIZE_3)) - .build(), - createPage(SIZE_3, SIZE_2, SIZE_2, SIZE_3)) - .put( - RangeRequest.builder() - .startRowInclusive(spawnBytes(SIZE_2)) - .build(), - createPage(SIZE_3, SIZE_3, SIZE_3, SIZE_3)) - .buildOrThrow())) - .isEqualTo((SIZE_3 + 1L) * (3L * SIZE_2 + Long.BYTES) + SIZE_3 * (3L * SIZE_3 + Long.BYTES)); - } - - private static SimpleTokenBackedResultsPage, byte[]> createPage( - int tokenSize, int cellNameSize, int valueNameSize, int resultsCount) { - return SimpleTokenBackedResultsPage.create( - spawnBytes(tokenSize), - Stream.generate(() -> RowResult.of(createCell(cellNameSize), createValue(valueNameSize))) - .limit(resultsCount) - .collect(Collectors.toUnmodifiableList())); - } - - private static TableReference createTableReference(int size) { - return TableReference.createWithEmptyNamespace(StringUtils.repeat('A', size)); - } - - private static Cell createCell(int nameSize) { - return Cell.create(spawnBytes(nameSize), spawnBytes(nameSize)); - } - - private static Value createValue(int size) { - return Value.create(spawnBytes(size), TIMESTAMP); - } - - private static byte[] spawnBytes(int size) { - byte[] bytes = new byte[size]; - Arrays.fill(bytes, BYTE); - return bytes; - } -} From b51745b600327b674681a37f9ef900d1b6c05102 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 14 Nov 2022 10:41:21 +0000 Subject: [PATCH 13/53] removing clock and impls --- .../transaction/impl/SnapshotTransaction.java | 32 +++++++------------ ...a => TrackingKeyValueServiceNoOpImpl.java} | 4 +-- 2 files changed, 14 insertions(+), 22 deletions(-) rename atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/{TrackingKeyValueServiceImpl.java => TrackingKeyValueServiceNoOpImpl.java} (88%) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 792a79ab808..3fde63f3d34 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -25,7 +25,6 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Stopwatch; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Collections2; import com.google.common.collect.FluentIterable; @@ -100,11 +99,10 @@ import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; -import com.palantir.atlasdb.transaction.api.expectations.ImmutableExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; -import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceImpl; +import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceNoOpImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.TransactionOutcomeMetrics; import com.palantir.atlasdb.transaction.knowledge.TransactionKnowledgeComponents; @@ -276,7 +274,6 @@ private enum State { protected final TransactionKnowledgeComponents knowledge; protected final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; - protected final Stopwatch stopwatch = Stopwatch.createStarted(); /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read @@ -314,7 +311,7 @@ private enum State { this.lockWatchManager = lockWatchManager; this.conflictTracer = conflictTracer; this.transactionTimerContext = getTimer("transactionMillis").time(); - this.keyValueService = new TrackingKeyValueServiceImpl(tmKeyValueService); + this.keyValueService = new TrackingKeyValueServiceNoOpImpl(tmKeyValueService); this.immediateKeyValueService = KeyValueServices.synchronousAsAsyncKeyValueService(keyValueService); this.timelockService = timelockService; this.defaultTransactionService = transactionService; @@ -2596,28 +2593,28 @@ public ExpectationsConfig expectationsConfig() { throw new NotImplementedException(); } + // todo(aalouane) @Override public long getAgeMillis() { - return stopwatch.elapsed(TimeUnit.MILLISECONDS); + throw new NotImplementedException(); } + // todo(aalouane) @Override public long getAgeMillisAndFreezeTimer() { - stopwatch.stop(); - return getAgeMillis(); + throw new NotImplementedException(); } + // todo(aalouane) @Override public TransactionReadInfo getReadInfo() { - return keyValueService.getOverallReadInfo(); + throw new NotImplementedException(); } + // todo(aalouane) @Override public ExpectationsStatistics getCallbackStatistics() { - return ImmutableExpectationsStatistics.builder() - .transactionAgeMillis(getAgeMillisAndFreezeTimer()) - .readInfoByTable(keyValueService.getReadInfoByTable()) - .build(); + throw new NotImplementedException(); } // todo(aalouane) @@ -2632,15 +2629,10 @@ public Set checkAndGetViolations() { throw new NotImplementedException(); } + // todo(aalouane) @Override public void reportExpectationsCollectedData() { - expectationsDataCollectionMetrics.ageMillis().update(getAgeMillisAndFreezeTimer()); - TransactionReadInfo info = getReadInfo(); - expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); - expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); - info.maximumBytesKvsCallInfo() - .ifPresent(kvsReadInfo -> - expectationsDataCollectionMetrics.worstKvsBytesRead().update(kvsReadInfo.bytesRead())); + throw new NotImplementedException(); } private Timer getTimer(String name) { diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java similarity index 88% rename from atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java rename to atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java index 0efdf9f4d7a..1194ac7da9b 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java @@ -22,11 +22,11 @@ import com.palantir.atlasdb.keyvalue.impl.ForwardingKeyValueService; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -public class TrackingKeyValueServiceImpl extends ForwardingKeyValueService implements TrackingKeyValueService { +public class TrackingKeyValueServiceNoOpImpl extends ForwardingKeyValueService implements TrackingKeyValueService { KeyValueService delegate; KeyValueServiceDataTracker tracker = new KeyValueServiceDataTracker(); - public TrackingKeyValueServiceImpl(KeyValueService delegate) { + public TrackingKeyValueServiceNoOpImpl(KeyValueService delegate) { this.delegate = delegate; } From c72a8f1193179efefa2d43616d78675472cd8749 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 14 Nov 2022 10:47:18 +0000 Subject: [PATCH 14/53] removing metrics definition --- .../transaction/impl/SnapshotTransaction.java | 8 +------- .../src/main/metrics/expectations.yml | 19 ------------------- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 atlasdb-impl-shared/src/main/metrics/expectations.yml diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 3fde63f3d34..757a4faa089 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -100,7 +100,6 @@ import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceNoOpImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -273,7 +272,6 @@ private enum State { protected final TimestampCache timestampCache; protected final TransactionKnowledgeComponents knowledge; - protected final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read @@ -347,8 +345,6 @@ private enum State { timelockService, immutableTimestamp, knowledge); - this.expectationsDataCollectionMetrics = - ExpectationsDataCollectionMetrics.of(metricsManager.getTaggedRegistry()); } protected TransactionScopedCache getCache() { @@ -2631,9 +2627,7 @@ public Set checkAndGetViolations() { // todo(aalouane) @Override - public void reportExpectationsCollectedData() { - throw new NotImplementedException(); - } + public void reportExpectationsCollectedData() {} private Timer getTimer(String name) { return metricsManager.registerOrGetTimer(SnapshotTransaction.class, name); diff --git a/atlasdb-impl-shared/src/main/metrics/expectations.yml b/atlasdb-impl-shared/src/main/metrics/expectations.yml deleted file mode 100644 index bdb5654df21..00000000000 --- a/atlasdb-impl-shared/src/main/metrics/expectations.yml +++ /dev/null @@ -1,19 +0,0 @@ -options: - javaPackage: 'com.palantir.atlasdb.transaction.expectations' - -namespaces: - expectationsDataCollection: - docs: Data collection on transaction patterns - metrics: - bytesRead: - docs: Bytes read by a transaction through kvs read calls - type: histogram - kvsCalls: - docs: Number of kvs read calls made by a transaction - type: histogram - ageMillis: - docs: Elapsed duration in milliseconds for a transaction - type: histogram - worstKvsBytesRead: - docs: Most bytes read by a transaction in one kvs read call - type: histogram From 76c4e62b536d4149bbd16a11101b8e794ba1a058 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 14 Nov 2022 14:33:25 +0000 Subject: [PATCH 15/53] removing getAgeMillisAndFreezeStopwatch --- .../api/expectations/ExpectationsAwareTransaction.java | 2 -- .../impl/ForwardingExpectationsAwareTransaction.java | 5 ----- .../atlasdb/transaction/impl/SnapshotTransaction.java | 6 ------ .../transaction/impl/SnapshotTransactionManager.java | 5 ----- 4 files changed, 18 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index a32b90d9fa8..e86b00b1b8c 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -29,8 +29,6 @@ public interface ExpectationsAwareTransaction extends Transaction { long getAgeMillis(); - long getAgeMillisAndFreezeTimer(); - TransactionReadInfo getReadInfo(); ExpectationsStatistics getCallbackStatistics(); diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index de67211c784..986c099478b 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -38,11 +38,6 @@ public long getAgeMillis() { return delegate().getAgeMillis(); } - @Override - public long getAgeMillisAndFreezeTimer() { - return delegate().getAgeMillisAndFreezeTimer(); - } - @Override public TransactionReadInfo getReadInfo() { return delegate().getReadInfo(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 757a4faa089..8cff580641b 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2595,12 +2595,6 @@ public long getAgeMillis() { throw new NotImplementedException(); } - // todo(aalouane) - @Override - public long getAgeMillisAndFreezeTimer() { - throw new NotImplementedException(); - } - // todo(aalouane) @Override public TransactionReadInfo getReadInfo() { diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index f6a39dc69c1..6f3d4c3a626 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -303,11 +303,6 @@ public long getAgeMillis() { return delegate.getAgeMillis(); } - @Override - public long getAgeMillisAndFreezeTimer() { - return delegate.getAgeMillisAndFreezeTimer(); - } - @Override public TransactionReadInfo getReadInfo() { return delegate.getReadInfo(); From 86b882a5cbc62743befb3a9d533973e24e5117d1 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 15 Nov 2022 11:33:11 +0000 Subject: [PATCH 16/53] staging --- .../ExpectationsConfigurations.java | 23 +++------- .../expectations/ExpectationsStatistics.java | 2 + .../TransactionViolationFlags.java | 42 +++++++++++++++++++ .../ExpectationsAwareTransaction.java | 4 +- .../impl/expectations/ExpectationsTask.java | 24 ++++++++--- .../expectations/ExpectationsManagerTest.java | 29 +++++++------ 6 files changed, 85 insertions(+), 39 deletions(-) rename atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java => atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java (63%) create mode 100644 atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java similarity index 63% rename from atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java rename to atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java index f72e23d6b22..e6d1c082c3f 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsViolation.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java @@ -14,24 +14,11 @@ * limitations under the License. */ -package com.palantir.atlasdb.transaction.impl.expectations; +package com.palantir.atlasdb.transaction.api.expectations; -import org.derive4j.Data; +public final class ExpectationsConfigurations { + private ExpectationsConfigurations() {} -@Data -public interface ExpectationsViolation { - interface Cases { - R ranForTooLong(); - - R readTooMuch(); - - R readTooMuchInOneKvsCall(); - - R queriedKvsTooManyTimes(); - } - - R match(Cases cases); - - @Override - boolean equals(Object other); + public static final ExpectationsConfig DEFAULT = + ImmutableExpectationsConfig.builder().build(); } diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java index 59a8f668971..932d9ba44cc 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java @@ -25,7 +25,9 @@ */ @Value.Immutable public interface ExpectationsStatistics { + @Value.Parameter long transactionAgeMillis(); + @Value.Parameter ImmutableMap readInfoByTable(); } diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java new file mode 100644 index 00000000000..3fa0f1b6266 --- /dev/null +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java @@ -0,0 +1,42 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.transaction.api.expectations; + +import org.immutables.value.Value; + +@Value.Immutable +public interface TransactionViolationFlags { + @Value.Default + default boolean ranForTooLong() { + return false; + } + + @Value.Default + default boolean readTooMuch() { + return false; + } + + @Value.Default + default boolean readTooMuchInOneKvsCall() { + return false; + } + + @Value.Default + default boolean queriedKvsTooMuch() { + return false; + } +} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java index 4657c32acf3..bf3b8d8290b 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java @@ -20,7 +20,7 @@ import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import java.util.Set; +import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; /** * A consistent view of {@link ExpectationsStatistics} is not guaranteed if the user interacts with the transaction @@ -38,5 +38,5 @@ public interface ExpectationsAwareTransaction extends Transaction { void runExpectationsCallbacks(); - Set checkAndGetViolations(); + TransactionViolationFlags checkAndGetViolations(); } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java index 913cf6829d4..c5764483df7 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java @@ -16,11 +16,13 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.google.common.collect.ImmutableSet; +import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionViolationFlags; +import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; import java.util.Set; -import java.util.stream.Collectors; public final class ExpectationsTask implements Runnable { private static final SafeLogger log = SafeLoggerFactory.get(ExpectationsTask.class); @@ -30,13 +32,23 @@ public ExpectationsTask(Set transactions) { this.transactions = transactions; } + // todo(aalouane) test this class after impl is done @Override public void run() { try { - Set violations = transactions.stream() - .flatMap(transaction -> transaction.checkAndGetViolations().stream()) - .collect(Collectors.toSet()); - updateMetrics(violations); + + Set flagSet = transactions.stream() + .map(ExpectationsAwareTransaction::checkAndGetViolations) + .collect(ImmutableSet.toImmutableSet()); + + updateMetrics(ImmutableTransactionViolationFlags.builder() + .ranForTooLong(flagSet.stream().anyMatch(TransactionViolationFlags::ranForTooLong)) + .readTooMuch(flagSet.stream().anyMatch(TransactionViolationFlags::readTooMuch)) + .readTooMuchInOneKvsCall( + flagSet.stream().anyMatch(TransactionViolationFlags::readTooMuchInOneKvsCall)) + .queriedKvsTooMuch(flagSet.stream().anyMatch(TransactionViolationFlags::queriedKvsTooMuch)) + .build()); + } catch (Throwable throwable) { log.warn( "Transactional Expectations task failed", @@ -46,5 +58,5 @@ public void run() { } // todo(aalouane) wire metrics and for each violation type mark violationOccurred-equivalent metric as 1 or 0 - private void updateMetrics(Set violations) {} + private void updateMetrics(TransactionViolationFlags violationFlags) {} } diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java index eea190c98ed..36ab05a3b31 100644 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java @@ -18,14 +18,12 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.jmock.lib.concurrent.DeterministicScheduler; import org.junit.Test; @@ -34,7 +32,7 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ExpectationsManagerTest { +public final class ExpectationsManagerTest { private final DeterministicScheduler executorService = spy(new DeterministicScheduler()); private final ExpectationsManager manager = ExpectationsManagerImpl.createStarted(executorService); @@ -43,17 +41,14 @@ public class ExpectationsManagerTest { @Test public void runnableWasScheduled() { - verify(executorService, times(1)).scheduleWithFixedDelay(any(), anyLong(), anyLong(), any(TimeUnit.class)); + verify(executorService).scheduleWithFixedDelay(any(), anyLong(), anyLong(), any(TimeUnit.class)); } @Test - public void scheduledTaskIsNotInterruptedByException() { + public void scheduledTaskInteractsWithTransaction() { manager.register(transaction); - when(transaction.checkAndGetViolations()) - .thenThrow(new RuntimeException()) - .thenReturn(Set.of()); - executorService.tick(2 * ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction, atLeast(2)).checkAndGetViolations(); + executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); + verify(transaction).checkAndGetViolations(); } @Test @@ -61,14 +56,22 @@ public void doubleRegisterIsRedundant() { manager.register(transaction); manager.register(transaction); executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction, times(1)).checkAndGetViolations(); + verify(transaction).checkAndGetViolations(); + } + + @Test + public void scheduledTaskIsNotInterruptedByException() { + manager.register(transaction); + when(transaction.checkAndGetViolations()).thenThrow(new RuntimeException()); + executorService.tick(2 * ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); + verify(transaction, times(2)).checkAndGetViolations(); } @Test public void unregisterLeadsToNoMoreInteractionsWithTransaction() { manager.register(transaction); executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction, times(1)).checkAndGetViolations(); + verify(transaction).checkAndGetViolations(); manager.unregister(transaction); executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); verifyNoMoreInteractions(transaction); @@ -78,7 +81,7 @@ public void unregisterLeadsToNoMoreInteractionsWithTransaction() { public void doubleUnregisterIsRedundant() { manager.register(transaction); executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction, times(1)).checkAndGetViolations(); + verify(transaction).checkAndGetViolations(); manager.unregister(transaction); manager.unregister(transaction); executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); From 02717265a35491ef79f09b0ae4f27ce0b0b998f4 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 15 Nov 2022 13:04:45 +0000 Subject: [PATCH 17/53] moving ExpectationsAwareTransaction --- .../api}/expectations/ExpectationsAwareTransaction.java | 6 +----- .../transaction/impl/expectations/ExpectationsManager.java | 2 ++ .../impl/expectations/ExpectationsManagerImpl.java | 1 + .../transaction/impl/expectations/ExpectationsTask.java | 1 + .../impl/expectations/ExpectationsManagerTest.java | 1 + 5 files changed, 6 insertions(+), 5 deletions(-) rename {atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl => atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api}/expectations/ExpectationsAwareTransaction.java (77%) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java similarity index 77% rename from atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java rename to atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index bf3b8d8290b..e5bf245b097 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -14,13 +14,9 @@ * limitations under the License. */ -package com.palantir.atlasdb.transaction.impl.expectations; +package com.palantir.atlasdb.transaction.api.expectations; import com.palantir.atlasdb.transaction.api.Transaction; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; -import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; /** * A consistent view of {@link ExpectationsStatistics} is not guaranteed if the user interacts with the transaction diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java index bee3d8b6ce8..6b748ea699d 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java @@ -16,6 +16,8 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; + public interface ExpectationsManager extends AutoCloseable { void register(ExpectationsAwareTransaction transaction); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java index 2e7fe261319..23052ddb347 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java @@ -16,6 +16,7 @@ package com.palantir.atlasdb.transaction.impl.expectations; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import java.time.Duration; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java index c5764483df7..abb1334b919 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java @@ -17,6 +17,7 @@ package com.palantir.atlasdb.transaction.impl.expectations; import com.google.common.collect.ImmutableSet; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionViolationFlags; import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; import com.palantir.logsafe.SafeArg; diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java index 36ab05a3b31..a6a30b8db48 100644 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java +++ b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import java.util.concurrent.TimeUnit; import org.jmock.lib.concurrent.DeterministicScheduler; import org.junit.Test; From ca029036d4d7d0498bfd7f933d2a74f4500611b1 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 15 Nov 2022 13:15:05 +0000 Subject: [PATCH 18/53] fix --- .../transaction/impl/expectations/ExpectationsTask.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java index dec446bd042..abb1334b919 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java @@ -16,15 +16,10 @@ package com.palantir.atlasdb.transaction.impl.expectations; -<<<<<<< HEAD -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; -======= import com.google.common.collect.ImmutableSet; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionViolationFlags; import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; ->>>>>>> tex-pr-6 import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; From e346fe771304f6747fbebbd4f2af635167336e68 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 15 Nov 2022 13:34:25 +0000 Subject: [PATCH 19/53] adding tracking data to ExpectationsAwareTransaction interface --- .../api/expectations/ExpectationsAwareTransaction.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index e5bf245b097..ef716f534cf 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -35,4 +35,6 @@ public interface ExpectationsAwareTransaction extends Transaction { void runExpectationsCallbacks(); TransactionViolationFlags checkAndGetViolations(); + + void reportExpectationsCollectedData(); } From 167f461b3042538cba0ae2e0d5fb7e77347ad095 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 15 Nov 2022 13:42:50 +0000 Subject: [PATCH 20/53] fixes --- .../impl/ForwardingExpectationsAwareTransaction.java | 5 ++--- .../atlasdb/transaction/impl/SnapshotTransaction.java | 4 ++-- .../atlasdb/transaction/impl/SnapshotTransactionManager.java | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index 986c099478b..25e02a21c83 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -19,9 +19,8 @@ import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import java.util.Set; +import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; public abstract class ForwardingExpectationsAwareTransaction extends ForwardingTransaction implements ExpectationsAwareTransaction { @@ -54,7 +53,7 @@ public void runExpectationsCallbacks() { } @Override - public Set checkAndGetViolations() { + public TransactionViolationFlags checkAndGetViolations() { return delegate().checkAndGetViolations(); } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 8cff580641b..b7ea4ef3dae 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -98,8 +98,8 @@ import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceNoOpImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -2615,7 +2615,7 @@ public void runExpectationsCallbacks() { // todo(aalouane) @Override - public Set checkAndGetViolations() { + public TransactionViolationFlags checkAndGetViolations() { throw new NotImplementedException(); } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index 6f3d4c3a626..c144d6596a1 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -48,8 +48,8 @@ import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsViolation; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; import com.palantir.atlasdb.transaction.impl.metrics.MemoizingTableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.MetricsFilterEvaluationContext; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -73,7 +73,6 @@ import java.time.Duration; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; @@ -317,7 +316,7 @@ public ExpectationsStatistics getCallbackStatistics() { public void runExpectationsCallbacks() {} @Override - public Set checkAndGetViolations() { + public TransactionViolationFlags checkAndGetViolations() { return delegate.checkAndGetViolations(); } From cbbbfaf28d3cb71868fe7f2065cef7566e092430 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 18 Nov 2022 21:53:40 +0000 Subject: [PATCH 21/53] removing stuff --- .../ExpectationsAwareTransaction.java | 13 +-- .../ExpectationsConfigurations.java | 3 + .../expectations/ExpectationsStatistics.java | 33 ------- .../TransactionViolationFlags.java | 42 --------- .../expectations/ExpectationsManager.java | 37 -------- .../expectations/ExpectationsManagerImpl.java | 77 ---------------- .../impl/expectations/ExpectationsTask.java | 63 ------------- .../expectations/ExpectationsManagerTest.java | 91 ------------------- 8 files changed, 10 insertions(+), 349 deletions(-) delete mode 100644 atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java delete mode 100644 atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java delete mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java delete mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java delete mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java delete mode 100644 atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index ef716f534cf..571184d2f49 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -28,13 +28,14 @@ public interface ExpectationsAwareTransaction extends Transaction { long getAgeMillis(); + /** + * Returns a point-in-time snapshot of transaction read information. + */ TransactionReadInfo getReadInfo(); - ExpectationsStatistics getCallbackStatistics(); - - void runExpectationsCallbacks(); - - TransactionViolationFlags checkAndGetViolations(); - + /** + * Update TEX data collection metrics for post-mortem transactions. + * Invoke only after the transaction committed or aborted. + */ void reportExpectationsCollectedData(); } diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java index e6d1c082c3f..7d1c8acb60a 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java @@ -16,6 +16,9 @@ package com.palantir.atlasdb.transaction.api.expectations; +/** + * Default/Presets in a separate utility class to avoid class loading deadlock. + */ public final class ExpectationsConfigurations { private ExpectationsConfigurations() {} diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java deleted file mode 100644 index 932d9ba44cc..00000000000 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsStatistics.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.api.expectations; - -import com.google.common.collect.ImmutableMap; -import com.palantir.atlasdb.keyvalue.api.TableReference; -import org.immutables.value.Value; - -/** - * Tracked data for an individual transaction. - */ -@Value.Immutable -public interface ExpectationsStatistics { - @Value.Parameter - long transactionAgeMillis(); - - @Value.Parameter - ImmutableMap readInfoByTable(); -} diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java deleted file mode 100644 index 3fa0f1b6266..00000000000 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/TransactionViolationFlags.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.api.expectations; - -import org.immutables.value.Value; - -@Value.Immutable -public interface TransactionViolationFlags { - @Value.Default - default boolean ranForTooLong() { - return false; - } - - @Value.Default - default boolean readTooMuch() { - return false; - } - - @Value.Default - default boolean readTooMuchInOneKvsCall() { - return false; - } - - @Value.Default - default boolean queriedKvsTooMuch() { - return false; - } -} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java deleted file mode 100644 index 6b748ea699d..00000000000 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManager.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; - -public interface ExpectationsManager extends AutoCloseable { - void register(ExpectationsAwareTransaction transaction); - - /* - * Stop tracking a given transaction without running expectations callbacks. - * Should not be used on transactions that are already in progress. - * Use case is for when transactions are created and registered but thrown away unused. - */ - void unregister(ExpectationsAwareTransaction transaction); - - /* - * Mark transaction as concluded (aborted/succeeded), run expectations callbacks and stop tracking the transaction. - * Unregistering transactions is done at best effort basis. - * Running callbacks is guaranteed. - */ - void markCompletion(ExpectationsAwareTransaction transaction); -} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java deleted file mode 100644 index 23052ddb347..00000000000 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerImpl.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import java.time.Duration; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -public final class ExpectationsManagerImpl implements ExpectationsManager { - public static final long SCHEDULER_DELAY_MILLIS = Duration.ofMinutes(5).toMillis(); - private final Set trackedTransactions = ConcurrentHashMap.newKeySet(); - private final AtomicBoolean taskIsScheduled = new AtomicBoolean(false); - private final ScheduledExecutorService executorService; - - private ExpectationsManagerImpl(ScheduledExecutorService executorService) { - this.executorService = executorService; - } - - public static ExpectationsManager createStarted(ScheduledExecutorService executorService) { - ExpectationsManagerImpl manager = new ExpectationsManagerImpl(executorService); - manager.scheduleExpectationsTask(); - return manager; - } - - private void scheduleExpectationsTask() { - if (!taskIsScheduled.compareAndExchange(false, true)) { - executorService.scheduleWithFixedDelay( - new ExpectationsTask(trackedTransactions), - SCHEDULER_DELAY_MILLIS, - SCHEDULER_DELAY_MILLIS, - TimeUnit.MILLISECONDS); - } - } - - @Override - public void register(ExpectationsAwareTransaction transaction) { - trackedTransactions.add(transaction); - } - - @Override - public void unregister(ExpectationsAwareTransaction transaction) { - trackedTransactions.remove(transaction); - } - - @Override - public void markCompletion(ExpectationsAwareTransaction transaction) { - try { - unregister(transaction); - } finally { - transaction.runExpectationsCallbacks(); - } - } - - @Override - public void close() throws Exception { - taskIsScheduled.set(true); - executorService.shutdown(); - } -} diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java deleted file mode 100644 index abb1334b919..00000000000 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsTask.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import com.google.common.collect.ImmutableSet; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import com.palantir.atlasdb.transaction.api.expectations.ImmutableTransactionViolationFlags; -import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; -import com.palantir.logsafe.SafeArg; -import com.palantir.logsafe.logger.SafeLogger; -import com.palantir.logsafe.logger.SafeLoggerFactory; -import java.util.Set; - -public final class ExpectationsTask implements Runnable { - private static final SafeLogger log = SafeLoggerFactory.get(ExpectationsTask.class); - private final Set transactions; - - public ExpectationsTask(Set transactions) { - this.transactions = transactions; - } - - // todo(aalouane) test this class after impl is done - @Override - public void run() { - try { - - Set flagSet = transactions.stream() - .map(ExpectationsAwareTransaction::checkAndGetViolations) - .collect(ImmutableSet.toImmutableSet()); - - updateMetrics(ImmutableTransactionViolationFlags.builder() - .ranForTooLong(flagSet.stream().anyMatch(TransactionViolationFlags::ranForTooLong)) - .readTooMuch(flagSet.stream().anyMatch(TransactionViolationFlags::readTooMuch)) - .readTooMuchInOneKvsCall( - flagSet.stream().anyMatch(TransactionViolationFlags::readTooMuchInOneKvsCall)) - .queriedKvsTooMuch(flagSet.stream().anyMatch(TransactionViolationFlags::queriedKvsTooMuch)) - .build()); - - } catch (Throwable throwable) { - log.warn( - "Transactional Expectations task failed", - SafeArg.of("trackedTransactionsCount", transactions.size()), - throwable); - } - } - - // todo(aalouane) wire metrics and for each violation type mark violationOccurred-equivalent metric as 1 or 0 - private void updateMetrics(TransactionViolationFlags violationFlags) {} -} diff --git a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java b/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java deleted file mode 100644 index a6a30b8db48..00000000000 --- a/atlasdb-impl-shared/src/test/java/com/palantir/atlasdb/transaction/impl/expectations/ExpectationsManagerTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import java.util.concurrent.TimeUnit; -import org.jmock.lib.concurrent.DeterministicScheduler; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public final class ExpectationsManagerTest { - private final DeterministicScheduler executorService = spy(new DeterministicScheduler()); - private final ExpectationsManager manager = ExpectationsManagerImpl.createStarted(executorService); - - @Mock - private ExpectationsAwareTransaction transaction; - - @Test - public void runnableWasScheduled() { - verify(executorService).scheduleWithFixedDelay(any(), anyLong(), anyLong(), any(TimeUnit.class)); - } - - @Test - public void scheduledTaskInteractsWithTransaction() { - manager.register(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction).checkAndGetViolations(); - } - - @Test - public void doubleRegisterIsRedundant() { - manager.register(transaction); - manager.register(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction).checkAndGetViolations(); - } - - @Test - public void scheduledTaskIsNotInterruptedByException() { - manager.register(transaction); - when(transaction.checkAndGetViolations()).thenThrow(new RuntimeException()); - executorService.tick(2 * ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction, times(2)).checkAndGetViolations(); - } - - @Test - public void unregisterLeadsToNoMoreInteractionsWithTransaction() { - manager.register(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction).checkAndGetViolations(); - manager.unregister(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verifyNoMoreInteractions(transaction); - } - - @Test - public void doubleUnregisterIsRedundant() { - manager.register(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verify(transaction).checkAndGetViolations(); - manager.unregister(transaction); - manager.unregister(transaction); - executorService.tick(ExpectationsManagerImpl.SCHEDULER_DELAY_MILLIS, TimeUnit.MILLISECONDS); - verifyNoMoreInteractions(transaction); - } -} From bfa1f8ddd30ff83a8cbd0143bed278fec5d5627f Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 18 Nov 2022 21:57:06 +0000 Subject: [PATCH 22/53] javadoc removal --- .../api/expectations/ExpectationsAwareTransaction.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 571184d2f49..434ed15b6ad 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -18,11 +18,6 @@ import com.palantir.atlasdb.transaction.api.Transaction; -/** - * A consistent view of {@link ExpectationsStatistics} is not guaranteed if the user interacts with the transaction - * post-commit/post-abort or outside the user task. - * todo(aalouane) move this javadoc to the user-exposed API when implemented - */ public interface ExpectationsAwareTransaction extends Transaction { ExpectationsConfig expectationsConfig(); From 81af9bd71777dae9d4d286286d43e22d00df3432 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 18 Nov 2022 21:58:22 +0000 Subject: [PATCH 23/53] javadoc removal --- .../api/expectations/ExpectationsAwareTransaction.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 571184d2f49..434ed15b6ad 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -18,11 +18,6 @@ import com.palantir.atlasdb.transaction.api.Transaction; -/** - * A consistent view of {@link ExpectationsStatistics} is not guaranteed if the user interacts with the transaction - * post-commit/post-abort or outside the user task. - * todo(aalouane) move this javadoc to the user-exposed API when implemented - */ public interface ExpectationsAwareTransaction extends Transaction { ExpectationsConfig expectationsConfig(); From 7f1e296fa4dd9bac52e0e4b3648401f19aa0e3c3 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 18 Nov 2022 22:07:48 +0000 Subject: [PATCH 24/53] removing config --- .../ExpectationsAwareTransaction.java | 2 - .../api/expectations/ExpectationsConfig.java | 78 ------------------- .../ExpectationsConfigurations.java | 27 ------- .../expectations/ExpectationsApiTest.java | 1 - 4 files changed, 108 deletions(-) delete mode 100644 atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfig.java delete mode 100644 atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 434ed15b6ad..234ab453782 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -19,8 +19,6 @@ import com.palantir.atlasdb.transaction.api.Transaction; public interface ExpectationsAwareTransaction extends Transaction { - ExpectationsConfig expectationsConfig(); - long getAgeMillis(); /** diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfig.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfig.java deleted file mode 100644 index 376b8333544..00000000000 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfig.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.api.expectations; - -import com.palantir.logsafe.Preconditions; -import com.palantir.logsafe.SafeArg; -import java.time.Duration; -import java.util.Optional; -import org.immutables.value.Value; - -@Value.Immutable -public interface ExpectationsConfig { - int MAXIMUM_NAME_SIZE = 255; - String DEFAULT_TRANSACTION_DISPLAY_NAME = "Unnamed"; - long ONE_MEBIBYTE = mebibytesToBytes(1); - - /** - * Length should not exceed {@value #MAXIMUM_NAME_SIZE}. - * This will be used for logging and will be expected to be safe to log. - */ - Optional transactionName(); - - @Value.Lazy - default String transactionDisplayName() { - return transactionName().orElse(DEFAULT_TRANSACTION_DISPLAY_NAME); - } - - @Value.Default - default long transactionAgeMillisLimit() { - return Duration.ofHours(24).toMillis(); - } - - @Value.Default - default long bytesReadLimit() { - return 50 * ONE_MEBIBYTE; - } - - @Value.Default - default long bytesReadInOneKvsCallLimit() { - return 10 * ONE_MEBIBYTE; - } - - @Value.Default - default long kvsReadCallCountLimit() { - return 100L; - } - - @Value.Check - default void check() { - transactionName().ifPresent(ExpectationsConfig::checkTransactionName); - } - - private static void checkTransactionName(String name) { - Preconditions.checkArgument( - name.length() <= MAXIMUM_NAME_SIZE, - "transactionName should be shorter", - SafeArg.of("transactionNameLength", name.length()), - SafeArg.of("maximumNameSize", MAXIMUM_NAME_SIZE)); - } - - static long mebibytesToBytes(long mebibytes) { - return 1024 * 1024 * mebibytes; - } -} diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java deleted file mode 100644 index 7d1c8acb60a..00000000000 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsConfigurations.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.api.expectations; - -/** - * Default/Presets in a separate utility class to avoid class loading deadlock. - */ -public final class ExpectationsConfigurations { - private ExpectationsConfigurations() {} - - public static final ExpectationsConfig DEFAULT = - ImmutableExpectationsConfig.builder().build(); -} diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java index 42a7c8b10ea..0d007afd4d5 100644 --- a/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java +++ b/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java @@ -20,7 +20,6 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ImmutableExpectationsConfig; import com.palantir.atlasdb.transaction.api.expectations.ImmutableKvsCallReadInfo; import com.palantir.atlasdb.transaction.api.expectations.KvsCallReadInfo; From e688370f4a5de9f9f98d193e8ab846879d2905b3 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Fri, 18 Nov 2022 22:21:47 +0000 Subject: [PATCH 25/53] impl --- ...orwardingExpectationsAwareTransaction.java | 23 --------- .../transaction/impl/SnapshotTransaction.java | 47 +++++++------------ .../impl/SnapshotTransactionManager.java | 21 --------- .../TrackingKeyValueServiceNoOpImpl.java | 1 + .../src/main/metrics/expectations.yml | 19 ++++++++ 5 files changed, 36 insertions(+), 75 deletions(-) create mode 100644 atlasdb-impl-shared/src/main/metrics/expectations.yml diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index 25e02a21c83..eb7d888f45f 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -17,21 +17,13 @@ package com.palantir.atlasdb.transaction.impl; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; public abstract class ForwardingExpectationsAwareTransaction extends ForwardingTransaction implements ExpectationsAwareTransaction { @Override public abstract ExpectationsAwareTransaction delegate(); - @Override - public ExpectationsConfig expectationsConfig() { - return delegate().expectationsConfig(); - } - @Override public long getAgeMillis() { return delegate().getAgeMillis(); @@ -42,21 +34,6 @@ public TransactionReadInfo getReadInfo() { return delegate().getReadInfo(); } - @Override - public ExpectationsStatistics getCallbackStatistics() { - return delegate().getCallbackStatistics(); - } - - @Override - public void runExpectationsCallbacks() { - delegate().runExpectationsCallbacks(); - } - - @Override - public TransactionViolationFlags checkAndGetViolations() { - return delegate().checkAndGetViolations(); - } - @Override public void reportExpectationsCollectedData() { delegate().reportExpectationsCollectedData(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index b7ea4ef3dae..ec91525a8eb 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -96,10 +96,8 @@ import com.palantir.atlasdb.transaction.api.TransactionLockAcquisitionTimeoutException; import com.palantir.atlasdb.transaction.api.TransactionLockTimeoutException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; +import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceNoOpImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -180,7 +178,6 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nullable; -import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.tuple.Pair; @@ -273,6 +270,7 @@ private enum State { protected final TransactionKnowledgeComponents knowledge; + private final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read * lock for it because we know that no writers exist. @@ -345,6 +343,8 @@ private enum State { timelockService, immutableTimestamp, knowledge); + this.expectationsDataCollectionMetrics = + ExpectationsDataCollectionMetrics.of(metricsManager.getTaggedRegistry()); } protected TransactionScopedCache getCache() { @@ -2583,45 +2583,30 @@ private Multimap getCellsToScrubByTable(State expectedStat return tableRefToCells; } - // todo(aalouane) - @Override - public ExpectationsConfig expectationsConfig() { - throw new NotImplementedException(); - } - - // todo(aalouane) @Override public long getAgeMillis() { - throw new NotImplementedException(); + return System.currentTimeMillis() - timeCreated; } - // todo(aalouane) @Override public TransactionReadInfo getReadInfo() { - throw new NotImplementedException(); + return keyValueService.getOverallReadInfo(); } - // todo(aalouane) @Override - public ExpectationsStatistics getCallbackStatistics() { - throw new NotImplementedException(); - } + public void reportExpectationsCollectedData() { + expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); - // todo(aalouane) - @Override - public void runExpectationsCallbacks() { - throw new NotImplementedException(); - } + TransactionReadInfo info = getReadInfo(); - // todo(aalouane) - @Override - public TransactionViolationFlags checkAndGetViolations() { - throw new NotImplementedException(); - } + expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); - // todo(aalouane) - @Override - public void reportExpectationsCollectedData() {} + expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); + + info.maximumBytesKvsCallInfo() + .ifPresent(kvsCallReadInfo -> + expectationsDataCollectionMetrics.worstKvsBytesRead().update(kvsCallReadInfo.bytesRead())); + } private Timer getTimer(String name) { return metricsManager.registerOrGetTimer(SnapshotTransaction.class, name); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index c144d6596a1..9cafeb55b2d 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -46,10 +46,7 @@ import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.TransactionTask; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.api.expectations.TransactionViolationFlags; import com.palantir.atlasdb.transaction.impl.metrics.MemoizingTableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.MetricsFilterEvaluationContext; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -292,11 +289,6 @@ public T finishWithCallback(TransactionTask task, return result; } - @Override - public ExpectationsConfig expectationsConfig() { - return delegate.expectationsConfig(); - } - @Override public long getAgeMillis() { return delegate.getAgeMillis(); @@ -307,19 +299,6 @@ public TransactionReadInfo getReadInfo() { return delegate.getReadInfo(); } - @Override - public ExpectationsStatistics getCallbackStatistics() { - return delegate.getCallbackStatistics(); - } - - @Override - public void runExpectationsCallbacks() {} - - @Override - public TransactionViolationFlags checkAndGetViolations() { - return delegate.checkAndGetViolations(); - } - @Override public void reportExpectationsCollectedData() { delegate.reportExpectationsCollectedData(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java index 1194ac7da9b..a5b26471bc2 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java @@ -22,6 +22,7 @@ import com.palantir.atlasdb.keyvalue.impl.ForwardingKeyValueService; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; +// todo(aalouane): remove this and use the real implementation after TEX4 merges public class TrackingKeyValueServiceNoOpImpl extends ForwardingKeyValueService implements TrackingKeyValueService { KeyValueService delegate; KeyValueServiceDataTracker tracker = new KeyValueServiceDataTracker(); diff --git a/atlasdb-impl-shared/src/main/metrics/expectations.yml b/atlasdb-impl-shared/src/main/metrics/expectations.yml new file mode 100644 index 00000000000..bdb5654df21 --- /dev/null +++ b/atlasdb-impl-shared/src/main/metrics/expectations.yml @@ -0,0 +1,19 @@ +options: + javaPackage: 'com.palantir.atlasdb.transaction.expectations' + +namespaces: + expectationsDataCollection: + docs: Data collection on transaction patterns + metrics: + bytesRead: + docs: Bytes read by a transaction through kvs read calls + type: histogram + kvsCalls: + docs: Number of kvs read calls made by a transaction + type: histogram + ageMillis: + docs: Elapsed duration in milliseconds for a transaction + type: histogram + worstKvsBytesRead: + docs: Most bytes read by a transaction in one kvs read call + type: histogram From 60a607e13bdf61c02911830ff8a3848868d0a498 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 21 Nov 2022 09:45:56 +0000 Subject: [PATCH 26/53] remove test --- .../expectations/ExpectationsApiTest.java | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java diff --git a/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java b/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java deleted file mode 100644 index 0d007afd4d5..00000000000 --- a/atlasdb-api/src/test/java/com/palantir/atlasdb/expectations/ExpectationsApiTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.expectations; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.palantir.atlasdb.transaction.api.expectations.ImmutableExpectationsConfig; -import com.palantir.atlasdb.transaction.api.expectations.ImmutableKvsCallReadInfo; -import com.palantir.atlasdb.transaction.api.expectations.KvsCallReadInfo; -import com.palantir.logsafe.exceptions.SafeIllegalArgumentException; -import org.apache.commons.lang3.StringUtils; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class ExpectationsApiTest { - public static final String SMALLER_NAME = "test1"; - public static final String LARGER_NAME = "test2"; - public static final KvsCallReadInfo SMALLER_NAME_10 = ImmutableKvsCallReadInfo.of(SMALLER_NAME, 10); - public static final KvsCallReadInfo SMALLER_NAME_20 = ImmutableKvsCallReadInfo.of(SMALLER_NAME, 20); - public static final KvsCallReadInfo LARGER_NAME_10 = ImmutableKvsCallReadInfo.of(LARGER_NAME, 10); - public static final KvsCallReadInfo LARGER_NAME_20 = ImmutableKvsCallReadInfo.of(LARGER_NAME, 20); - - public static final ExpectationsConfig DEFAULT_EXPECTATIONS_CONFIG = - ImmutableExpectationsConfig.builder().build(); - - @Test - public void testKvsCallReadInfoComparator() { - assertThat(SMALLER_NAME_10).isLessThan(SMALLER_NAME_20); - assertThat(SMALLER_NAME_20).isGreaterThan(SMALLER_NAME_10); - - assertThat(LARGER_NAME_10).isLessThan(LARGER_NAME_20); - assertThat(LARGER_NAME_20).isGreaterThan(LARGER_NAME_10); - - assertThat(LARGER_NAME_10).isLessThan(SMALLER_NAME_20); - assertThat(SMALLER_NAME_20).isGreaterThan(LARGER_NAME_10); - - assertThat(SMALLER_NAME_10).isLessThan(LARGER_NAME_20); - assertThat(LARGER_NAME_20).isGreaterThan(SMALLER_NAME_10); - - assertThat(SMALLER_NAME_10.compareTo( - ImmutableKvsCallReadInfo.builder().from(SMALLER_NAME_10).build())) - .isEqualTo(0); - assertThat(SMALLER_NAME_20.compareTo( - ImmutableKvsCallReadInfo.builder().from(SMALLER_NAME_20).build())) - .isEqualTo(0); - assertThat(LARGER_NAME_10.compareTo( - ImmutableKvsCallReadInfo.builder().from(LARGER_NAME_10).build())) - .isEqualTo(0); - assertThat(LARGER_NAME_20.compareTo( - ImmutableKvsCallReadInfo.builder().from(LARGER_NAME_20).build())) - .isEqualTo(0); - } - - @Test - public void testExpectationsConfigValidation() { - assertThatCode(() -> ImmutableExpectationsConfig.builder() - .from(DEFAULT_EXPECTATIONS_CONFIG) - .transactionName(StringUtils.repeat('t', ExpectationsConfig.MAXIMUM_NAME_SIZE)) - .build()) - .doesNotThrowAnyException(); - - assertThatThrownBy(() -> ImmutableExpectationsConfig.builder() - .from(DEFAULT_EXPECTATIONS_CONFIG) - .transactionName(StringUtils.repeat('t', ExpectationsConfig.MAXIMUM_NAME_SIZE + 1)) - .build()) - .isInstanceOf(SafeIllegalArgumentException.class); - } -} From e2892da2cb6f6410484bbb26aab2f378824037ca Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 21 Nov 2022 09:58:12 +0000 Subject: [PATCH 27/53] yml metrics spec --- atlasdb-impl-shared/src/main/metrics/expectations.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/atlasdb-impl-shared/src/main/metrics/expectations.yml b/atlasdb-impl-shared/src/main/metrics/expectations.yml index bdb5654df21..8406ebe5b2a 100644 --- a/atlasdb-impl-shared/src/main/metrics/expectations.yml +++ b/atlasdb-impl-shared/src/main/metrics/expectations.yml @@ -6,14 +6,14 @@ namespaces: docs: Data collection on transaction patterns metrics: bytesRead: - docs: Bytes read by a transaction through kvs read calls + docs: Bytes read by a transaction through Atlas kvs read calls type: histogram kvsCalls: - docs: Number of kvs read calls made by a transaction + docs: Number of Atlas kvs read calls made by a transaction type: histogram ageMillis: docs: Elapsed duration in milliseconds for a transaction type: histogram worstKvsBytesRead: - docs: Most bytes read by a transaction in one kvs read call + docs: Most bytes read by a transaction in one Atlas kvs read call type: histogram From 9c7728811d8315dc65b741c3f7bc2b744c1380a1 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 22 Nov 2022 10:06:55 +0000 Subject: [PATCH 28/53] flups --- .../api/expectations/ExpectationsAwareTransaction.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 234ab453782..d68fa307a7e 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -18,6 +18,11 @@ import com.palantir.atlasdb.transaction.api.Transaction; +/** + * Implementors of this interface provide methods useful for tracking transactional expectations and whether + * they were breached as well as relevant metrics and alerts. Transactional expectations represent transaction-level + * limits and rules for proper usage of AtlasDB transactions (e.g. reading too much data overall). + */ public interface ExpectationsAwareTransaction extends Transaction { long getAgeMillis(); @@ -28,7 +33,7 @@ public interface ExpectationsAwareTransaction extends Transaction { /** * Update TEX data collection metrics for post-mortem transactions. - * Invoke only after the transaction committed or aborted. + * Invoke only after the transaction committed or aborted, and only once. */ void reportExpectationsCollectedData(); } From 5552815858d48fee12fe7990c27e430afdbd4df8 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 22 Nov 2022 10:11:29 +0000 Subject: [PATCH 29/53] flup --- atlasdb-impl-shared/build.gradle | 1 + .../impl/SnapshotTransactionManager.java | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/atlasdb-impl-shared/build.gradle b/atlasdb-impl-shared/build.gradle index becdba0d022..62e0a779dee 100644 --- a/atlasdb-impl-shared/build.gradle +++ b/atlasdb-impl-shared/build.gradle @@ -12,6 +12,7 @@ license { exclude '**/PutUnlessExistsTableMetrics.java' exclude '**/TargetedSweepProgressMetrics.java' exclude '**/TimestampCorrectnessMetrics.java' + exclude '**/ExpectationsDataCollectionMetrics.java' } dependencies { diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index 9cafeb55b2d..acbdbaf7050 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -46,7 +46,6 @@ import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.TransactionTask; import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; import com.palantir.atlasdb.transaction.impl.metrics.MemoizingTableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.MetricsFilterEvaluationContext; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -288,21 +287,6 @@ public T finishWithCallback(TransactionTask task, postTaskContext.stop(); return result; } - - @Override - public long getAgeMillis() { - return delegate.getAgeMillis(); - } - - @Override - public TransactionReadInfo getReadInfo() { - return delegate.getReadInfo(); - } - - @Override - public void reportExpectationsCollectedData() { - delegate.reportExpectationsCollectedData(); - } } private void scrubForAggressiveHardDelete(SnapshotTransaction tx) { From ce4ef5b3cf27859e1aab0f41ccaabcf9bedd026a Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 22 Nov 2022 13:42:39 +0000 Subject: [PATCH 30/53] reportExpectationsCollectedData() no ops if called on in-progress transaction --- .../atlasdb/transaction/impl/SnapshotTransaction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index ec91525a8eb..ab43fae41ee 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2595,6 +2595,11 @@ public TransactionReadInfo getReadInfo() { @Override public void reportExpectationsCollectedData() { + if (!isDefinitivelyCommitted() && !isAborted()) { + log.error("reportExpectationsCollectedData is called on an in-progress transaction"); + return; + } + expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); TransactionReadInfo info = getReadInfo(); From ab77a450c67817a60d7e92e098899b7a8197ba35 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 22 Nov 2022 14:53:17 +0000 Subject: [PATCH 31/53] checkstyle --- .../palantir/atlasdb/transaction/impl/SnapshotTransaction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index ab43fae41ee..c8e070a8bb5 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -271,6 +271,7 @@ private enum State { protected final TransactionKnowledgeComponents knowledge; private final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; + /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read * lock for it because we know that no writers exist. From 9733e26011b7ccb1e617c8a699175123cf5247cb Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 22 Nov 2022 15:02:09 +0000 Subject: [PATCH 32/53] remove merge tokens --- .../api/expectations/ExpectationsAwareTransaction.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 5a3903d660d..337ad5906a4 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -33,16 +33,11 @@ public interface ExpectationsAwareTransaction extends Transaction { TransactionReadInfo getReadInfo(); /** - * <<<<<<< HEAD - * Update TEX data collection metrics for post-mortem transactions. - * Invoke only after the transaction committed or aborted, and only once. - * ======= * Update TEX data collection metrics for (post-mortem) transactions. * Expected usage is that this method is called once after the transaction has been committed or aborted. * This method won't report metrics if called on an in-progress transaction. * Calling this twice after the transaction has committed or aborted will result in duplication. * Clients should not call this method directly. - * >>>>>>> develop */ void reportExpectationsCollectedData(); } From 5033e1933defc997da2ebc2557f0c7af75bd9c7b Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Wed, 23 Nov 2022 14:57:58 +0000 Subject: [PATCH 33/53] merge develop --- .../ExpectationsAwareTransaction.java | 1 - .../transaction/impl/SnapshotTransaction.java | 4 +- .../TrackingKeyValueServiceNoOpImpl.java | 48 ------------------- 3 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java index 337ad5906a4..4fed762f8a4 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java @@ -22,7 +22,6 @@ * Implementors of this interface provide methods useful for tracking transactional expectations and whether * they were breached as well as relevant metrics and alerts. Transactional expectations represent transaction-level * limits and rules for proper usage of AtlasDB transactions (e.g. reading too much data overall). - * Todo(aalouane): move this out of API once part 4 is merged */ public interface ExpectationsAwareTransaction extends Transaction { long getAgeMillis(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index c8e070a8bb5..4091ec533d5 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -99,7 +99,7 @@ import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; -import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceNoOpImpl; +import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.TransactionOutcomeMetrics; import com.palantir.atlasdb.transaction.knowledge.TransactionKnowledgeComponents; @@ -308,7 +308,7 @@ private enum State { this.lockWatchManager = lockWatchManager; this.conflictTracer = conflictTracer; this.transactionTimerContext = getTimer("transactionMillis").time(); - this.keyValueService = new TrackingKeyValueServiceNoOpImpl(tmKeyValueService); + this.keyValueService = new TrackingKeyValueServiceImpl(tmKeyValueService); this.immediateKeyValueService = KeyValueServices.synchronousAsAsyncKeyValueService(keyValueService); this.timelockService = timelockService; this.defaultTransactionService = transactionService; diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java deleted file mode 100644 index a5b26471bc2..00000000000 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/expectations/TrackingKeyValueServiceNoOpImpl.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.palantir.atlasdb.transaction.impl.expectations; - -import com.google.common.collect.ImmutableMap; -import com.palantir.atlasdb.keyvalue.api.KeyValueService; -import com.palantir.atlasdb.keyvalue.api.TableReference; -import com.palantir.atlasdb.keyvalue.impl.ForwardingKeyValueService; -import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; - -// todo(aalouane): remove this and use the real implementation after TEX4 merges -public class TrackingKeyValueServiceNoOpImpl extends ForwardingKeyValueService implements TrackingKeyValueService { - KeyValueService delegate; - KeyValueServiceDataTracker tracker = new KeyValueServiceDataTracker(); - - public TrackingKeyValueServiceNoOpImpl(KeyValueService delegate) { - this.delegate = delegate; - } - - @Override - public KeyValueService delegate() { - return delegate; - } - - @Override - public TransactionReadInfo getOverallReadInfo() { - return tracker.getReadInfo(); - } - - @Override - public ImmutableMap getReadInfoByTable() { - return tracker.getReadInfoByTable(); - } -} From 7df9eca9d77febca92827cd0cbfcf3c56945da26 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 24 Nov 2022 14:46:40 +0000 Subject: [PATCH 34/53] pairing --- .../transaction/api/OpenTransaction.java | 3 +- .../impl/AbstractTransactionManager.java | 1 + .../transaction/impl/SnapshotTransaction.java | 4 +-- .../impl/SnapshotTransactionManager.java | 30 +++++-------------- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java index 54cf1e3f30c..a477f5e5a95 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java +++ b/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OpenTransaction.java @@ -17,9 +17,8 @@ package com.palantir.atlasdb.transaction.api; import com.palantir.atlasdb.metrics.Timed; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; -public interface OpenTransaction extends ExpectationsAwareTransaction { +public interface OpenTransaction extends Transaction { /** * Runs a provided task, commits the transaction, and performs cleanup. If no further work needs to be done with the diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/AbstractTransactionManager.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/AbstractTransactionManager.java index bd88f21abfc..17e6e0aeb1b 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/AbstractTransactionManager.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/AbstractTransactionManager.java @@ -81,6 +81,7 @@ protected final T runTaskThrowOnConflictWithCallback( } } finally { callback.run(); + txn.reportExpectationsCollectedData(); txn.runSuccessCallbacksIfDefinitivelyCommitted(); } } diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 4091ec533d5..7505bd23319 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -279,7 +279,7 @@ private enum State { */ /* package */ SnapshotTransaction( MetricsManager metricsManager, - KeyValueService tmKeyValueService, + KeyValueService delegateKeyValueService, TimelockService timelockService, LockWatchManagerInternal lockWatchManager, TransactionService transactionService, @@ -308,7 +308,7 @@ private enum State { this.lockWatchManager = lockWatchManager; this.conflictTracer = conflictTracer; this.transactionTimerContext = getTimer("transactionMillis").time(); - this.keyValueService = new TrackingKeyValueServiceImpl(tmKeyValueService); + this.keyValueService = new TrackingKeyValueServiceImpl(delegateKeyValueService); this.immediateKeyValueService = KeyValueServices.synchronousAsAsyncKeyValueService(keyValueService); this.timelockService = timelockService; this.defaultTransactionService = transactionService; diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java index acbdbaf7050..556a9328d18 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransactionManager.java @@ -45,7 +45,6 @@ import com.palantir.atlasdb.transaction.api.TransactionFailedRetriableException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.TransactionTask; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.impl.metrics.MemoizingTableLevelMetricsController; import com.palantir.atlasdb.transaction.impl.metrics.MetricsFilterEvaluationContext; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -185,15 +184,8 @@ public T runTaskWithCondi throw e; } - T result; - try { - result = openTransaction.finishWithCallback( - transaction -> task.execute(transaction, condition), condition::cleanup); - } finally { - openTransaction.reportExpectationsCollectedData(); - } - - return result; + return openTransaction.finishWithCallback( + transaction -> task.execute(transaction, condition), condition::cleanup); } @Override @@ -244,7 +236,7 @@ public List startTransactions(List T runTaskWithCond conflictTracer, tableLevelMetricsController, knowledge); - T result; - try { - result = runTaskThrowOnConflictWithCallback( - txn -> task.execute(txn, condition), - new ReadTransaction(transaction, sweepStrategyManager), - condition::cleanup); - } finally { - transaction.reportExpectationsCollectedData(); - } - return result; + return runTaskThrowOnConflictWithCallback( + txn -> task.execute(txn, condition), + new ReadTransaction(transaction, sweepStrategyManager), + condition::cleanup); } @Override From 11d4c4341477130c3669e412b25a1b35214aff79 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 24 Nov 2022 17:21:07 +0000 Subject: [PATCH 35/53] come one --- .../atlasdb/transaction/impl/CallbackAwareTransaction.java | 1 - .../transaction/impl}/ExpectationsAwareTransaction.java | 3 ++- .../impl/ForwardingExpectationsAwareTransaction.java | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) rename {atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations => atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl}/ExpectationsAwareTransaction.java (93%) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java index 32c02bc9c52..312e2352c1f 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/CallbackAwareTransaction.java @@ -18,7 +18,6 @@ import com.palantir.atlasdb.transaction.api.PreCommitCondition; import com.palantir.atlasdb.transaction.api.TransactionFailedException; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.service.TransactionService; /** diff --git a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ExpectationsAwareTransaction.java similarity index 93% rename from atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java rename to atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ExpectationsAwareTransaction.java index 4fed762f8a4..25b42fba391 100644 --- a/atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/expectations/ExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ExpectationsAwareTransaction.java @@ -14,9 +14,10 @@ * limitations under the License. */ -package com.palantir.atlasdb.transaction.api.expectations; +package com.palantir.atlasdb.transaction.impl; import com.palantir.atlasdb.transaction.api.Transaction; +import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; /** * Implementors of this interface provide methods useful for tracking transactional expectations and whether diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index eb7d888f45f..03ac3e0e0b1 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -16,7 +16,6 @@ package com.palantir.atlasdb.transaction.impl; -import com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; public abstract class ForwardingExpectationsAwareTransaction extends ForwardingTransaction From 5390c757158828a7cf12ed17637689bf5defbdc5 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 24 Nov 2022 17:39:26 +0000 Subject: [PATCH 36/53] yes --- .../impl/ForwardingExpectationsAwareTransaction.java | 1 + .../atlasdb/transaction/impl/SnapshotTransaction.java | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java index 03ac3e0e0b1..63a2e7cedf6 100644 --- a/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java +++ b/atlasdb-client/src/main/java/com/palantir/atlasdb/transaction/impl/ForwardingExpectationsAwareTransaction.java @@ -20,6 +20,7 @@ public abstract class ForwardingExpectationsAwareTransaction extends ForwardingTransaction implements ExpectationsAwareTransaction { + @Override public abstract ExpectationsAwareTransaction delegate(); diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 7505bd23319..4c3414735cf 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2596,7 +2596,7 @@ public TransactionReadInfo getReadInfo() { @Override public void reportExpectationsCollectedData() { - if (!isDefinitivelyCommitted() && !isAborted()) { + if (!(isDefinitivelyCommitted() || isAborted())) { log.error("reportExpectationsCollectedData is called on an in-progress transaction"); return; } @@ -2604,9 +2604,7 @@ public void reportExpectationsCollectedData() { expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); TransactionReadInfo info = getReadInfo(); - expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); - expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); info.maximumBytesKvsCallInfo() From dd465e3bd862ef9aca89627552ae856d62d7999e Mon Sep 17 00:00:00 2001 From: svc-changelog Date: Thu, 24 Nov 2022 17:46:30 +0000 Subject: [PATCH 37/53] Add generated changelog entries --- changelog/@unreleased/pr-6340.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/@unreleased/pr-6340.v2.yml diff --git a/changelog/@unreleased/pr-6340.v2.yml b/changelog/@unreleased/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/@unreleased/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 From ba63b23f7e272521ec9f54489b73cab194c1bb3f Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Thu, 24 Nov 2022 17:50:37 +0000 Subject: [PATCH 38/53] Autorelease 0.773.0-rc1 --- changelog/0.773.0-rc1/pr-6340.v2.yml | 7 +++++++ changelog/0.773.0-rc1/pr-6380.v2.yml | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 changelog/0.773.0-rc1/pr-6340.v2.yml create mode 100644 changelog/0.773.0-rc1/pr-6380.v2.yml diff --git a/changelog/0.773.0-rc1/pr-6340.v2.yml b/changelog/0.773.0-rc1/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.773.0-rc1/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 diff --git a/changelog/0.773.0-rc1/pr-6380.v2.yml b/changelog/0.773.0-rc1/pr-6380.v2.yml new file mode 100644 index 00000000000..720294fbcbe --- /dev/null +++ b/changelog/0.773.0-rc1/pr-6380.v2.yml @@ -0,0 +1,5 @@ +type: improvement +improvement: + description: Fix supplement for CoordinationAwareKnownConcludedTransactionsStore + links: + - https://github.com/palantir/atlasdb/pull/6380 From 15836f48ac3315bbc5506b9bd96378ebea15864e Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 24 Nov 2022 19:02:01 +0000 Subject: [PATCH 39/53] revapi --- .palantir/revapi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 738ca3a07fb..4626b8cffbf 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -13,3 +13,8 @@ acceptedBreaks: - code: "java.class.removed" old: "interface com.palantir.atlasdb.transaction.api.expectations.ExpectationsStatistics" justification: "removing TEX until ready" + "0.772.0": + com.palantir.atlasdb:atlasdb-api: + - code: "java.class.removed" + old: "interface com.palantir.atlasdb.transaction.api.expectations.ExpectationsAwareTransaction" + justification: "moving this out of API" From 0c1e1f4c64c23ef890d3cdb438d1ad3ba033c3ff Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Thu, 24 Nov 2022 20:26:51 +0000 Subject: [PATCH 40/53] Autorelease 0.773.0-rc2 --- changelog/0.773.0-rc2/pr-6340.v2.yml | 7 +++++++ changelog/0.773.0-rc2/pr-6380.v2.yml | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 changelog/0.773.0-rc2/pr-6340.v2.yml create mode 100644 changelog/0.773.0-rc2/pr-6380.v2.yml diff --git a/changelog/0.773.0-rc2/pr-6340.v2.yml b/changelog/0.773.0-rc2/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.773.0-rc2/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 diff --git a/changelog/0.773.0-rc2/pr-6380.v2.yml b/changelog/0.773.0-rc2/pr-6380.v2.yml new file mode 100644 index 00000000000..720294fbcbe --- /dev/null +++ b/changelog/0.773.0-rc2/pr-6380.v2.yml @@ -0,0 +1,5 @@ +type: improvement +improvement: + description: Fix supplement for CoordinationAwareKnownConcludedTransactionsStore + links: + - https://github.com/palantir/atlasdb/pull/6380 From 2f08a450e357bbcd81b9d0a9abf80adf907ddbb6 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 28 Nov 2022 10:59:31 +0000 Subject: [PATCH 41/53] debugging --- .../atlasdb/transaction/impl/SnapshotTransaction.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index 4c3414735cf..e08df8f1cf6 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2597,10 +2597,14 @@ public TransactionReadInfo getReadInfo() { @Override public void reportExpectationsCollectedData() { if (!(isDefinitivelyCommitted() || isAborted())) { - log.error("reportExpectationsCollectedData is called on an in-progress transaction"); + log.error( + "reportExpectationsCollectedData is called on an in-progress transaction", + SafeArg.of("state", state.get())); return; } + log.info("reportExpectationsCollectedData is running successfully", SafeArg.of("state", state)); + expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); TransactionReadInfo info = getReadInfo(); From dfab6fb56c892c05d45da41b9ce28c56c91d7aa3 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Mon, 28 Nov 2022 11:05:28 +0000 Subject: [PATCH 42/53] fix --- .../palantir/atlasdb/transaction/impl/SnapshotTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index e08df8f1cf6..d0c6895a437 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2603,7 +2603,7 @@ public void reportExpectationsCollectedData() { return; } - log.info("reportExpectationsCollectedData is running successfully", SafeArg.of("state", state)); + log.info("reportExpectationsCollectedData is running successfully", SafeArg.of("state", state.get())); expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); From 8c7fd1a53a2a341d22247f8795932e1e5061ea6b Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Mon, 28 Nov 2022 11:08:59 +0000 Subject: [PATCH 43/53] Autorelease 0.773.0-rc3 --- changelog/0.773.0-rc3/pr-6340.v2.yml | 7 +++++++ changelog/0.773.0-rc3/pr-6380.v2.yml | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 changelog/0.773.0-rc3/pr-6340.v2.yml create mode 100644 changelog/0.773.0-rc3/pr-6380.v2.yml diff --git a/changelog/0.773.0-rc3/pr-6340.v2.yml b/changelog/0.773.0-rc3/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.773.0-rc3/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 diff --git a/changelog/0.773.0-rc3/pr-6380.v2.yml b/changelog/0.773.0-rc3/pr-6380.v2.yml new file mode 100644 index 00000000000..720294fbcbe --- /dev/null +++ b/changelog/0.773.0-rc3/pr-6380.v2.yml @@ -0,0 +1,5 @@ +type: improvement +improvement: + description: Fix supplement for CoordinationAwareKnownConcludedTransactionsStore + links: + - https://github.com/palantir/atlasdb/pull/6380 From 480dae72716d70af0c67329a7519b61c6375f852 Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Tue, 6 Dec 2022 11:09:41 +0000 Subject: [PATCH 44/53] Autorelease 0.782.0-rc1 [skip ci] --- changelog/0.782.0-rc1/pr-6340.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/0.782.0-rc1/pr-6340.v2.yml diff --git a/changelog/0.782.0-rc1/pr-6340.v2.yml b/changelog/0.782.0-rc1/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.782.0-rc1/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 From 8fbe1223fb8fa37b56ccc02ce3bf50ed909f6953 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 8 Dec 2022 16:57:15 +0000 Subject: [PATCH 45/53] more logs --- .../transaction/impl/SnapshotTransaction.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index d0c6895a437..c6c31e7a702 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2605,15 +2605,28 @@ public void reportExpectationsCollectedData() { log.info("reportExpectationsCollectedData is running successfully", SafeArg.of("state", state.get())); - expectationsDataCollectionMetrics.ageMillis().update(getAgeMillis()); - + long ageMillis = getAgeMillis(); TransactionReadInfo info = getReadInfo(); + + log.info("ageMillis", SafeArg.of("age", ageMillis)); + log.info("txnInfo", SafeArg.of("info", info)); + + expectationsDataCollectionMetrics.ageMillis().update(ageMillis); expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); info.maximumBytesKvsCallInfo() .ifPresent(kvsCallReadInfo -> expectationsDataCollectionMetrics.worstKvsBytesRead().update(kvsCallReadInfo.bytesRead())); + + log.info( + "values in histogram", + SafeArg.of( + "kvsCalls", + expectationsDataCollectionMetrics + .kvsCalls() + .getSnapshot() + .getValues())); } private Timer getTimer(String name) { From 4238a7aaaded25ddee7557d789c64ffc894f4461 Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:00:50 +0000 Subject: [PATCH 46/53] Autorelease 0.782.0-rc2 [skip ci] --- changelog/0.782.0-rc2/pr-6340.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/0.782.0-rc2/pr-6340.v2.yml diff --git a/changelog/0.782.0-rc2/pr-6340.v2.yml b/changelog/0.782.0-rc2/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.782.0-rc2/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 From 1a1873f5670fdcb30e36adb753fd07606e4c9c98 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Tue, 3 Jan 2023 23:19:54 +0000 Subject: [PATCH 47/53] fixes --- .../transaction/impl/SnapshotTransaction.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index c6c31e7a702..e390179758b 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -2596,21 +2596,16 @@ public TransactionReadInfo getReadInfo() { @Override public void reportExpectationsCollectedData() { - if (!(isDefinitivelyCommitted() || isAborted())) { + if (!List.of(State.COMMITTED, State.ABORTED, State.FAILED).contains(state.get())) { log.error( "reportExpectationsCollectedData is called on an in-progress transaction", SafeArg.of("state", state.get())); return; } - log.info("reportExpectationsCollectedData is running successfully", SafeArg.of("state", state.get())); - long ageMillis = getAgeMillis(); TransactionReadInfo info = getReadInfo(); - log.info("ageMillis", SafeArg.of("age", ageMillis)); - log.info("txnInfo", SafeArg.of("info", info)); - expectationsDataCollectionMetrics.ageMillis().update(ageMillis); expectationsDataCollectionMetrics.bytesRead().update(info.bytesRead()); expectationsDataCollectionMetrics.kvsCalls().update(info.kvsCalls()); @@ -2618,15 +2613,6 @@ public void reportExpectationsCollectedData() { info.maximumBytesKvsCallInfo() .ifPresent(kvsCallReadInfo -> expectationsDataCollectionMetrics.worstKvsBytesRead().update(kvsCallReadInfo.bytesRead())); - - log.info( - "values in histogram", - SafeArg.of( - "kvsCalls", - expectationsDataCollectionMetrics - .kvsCalls() - .getSnapshot() - .getValues())); } private Timer getTimer(String name) { From 2786dee1a170bd14ce99d95e58e24fb585abbd97 Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Tue, 3 Jan 2023 23:26:53 +0000 Subject: [PATCH 48/53] Autorelease 0.784.0-rc1 [skip ci] --- changelog/0.784.0-rc1/pr-6340.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/0.784.0-rc1/pr-6340.v2.yml diff --git a/changelog/0.784.0-rc1/pr-6340.v2.yml b/changelog/0.784.0-rc1/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.784.0-rc1/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340 From c06bee3419dba9fa87485c6e11841923401e5dcc Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 12 Jan 2023 15:18:38 +0000 Subject: [PATCH 49/53] changing the metric namespace name --- atlasdb-impl-shared/src/main/metrics/expectations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-impl-shared/src/main/metrics/expectations.yml b/atlasdb-impl-shared/src/main/metrics/expectations.yml index 8406ebe5b2a..7cfe40b64ef 100644 --- a/atlasdb-impl-shared/src/main/metrics/expectations.yml +++ b/atlasdb-impl-shared/src/main/metrics/expectations.yml @@ -2,7 +2,7 @@ options: javaPackage: 'com.palantir.atlasdb.transaction.expectations' namespaces: - expectationsDataCollection: + expectations: docs: Data collection on transaction patterns metrics: bytesRead: From 642761c6032a275efe30c1d3e2c1c676e63f9448 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 12 Jan 2023 22:07:43 +0000 Subject: [PATCH 50/53] fix --- .../atlasdb/transaction/impl/SnapshotTransaction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index e390179758b..d768b322c06 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -97,7 +97,7 @@ import com.palantir.atlasdb.transaction.api.TransactionLockTimeoutException; import com.palantir.atlasdb.transaction.api.TransactionReadSentinelBehavior; import com.palantir.atlasdb.transaction.api.expectations.TransactionReadInfo; -import com.palantir.atlasdb.transaction.expectations.ExpectationsDataCollectionMetrics; +import com.palantir.atlasdb.transaction.expectations.ExpectationsMetrics; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueService; import com.palantir.atlasdb.transaction.impl.expectations.TrackingKeyValueServiceImpl; import com.palantir.atlasdb.transaction.impl.metrics.TableLevelMetricsController; @@ -270,7 +270,7 @@ private enum State { protected final TransactionKnowledgeComponents knowledge; - private final ExpectationsDataCollectionMetrics expectationsDataCollectionMetrics; + private final ExpectationsMetrics expectationsDataCollectionMetrics; /** * @param immutableTimestamp If we find a row written before the immutableTimestamp we don't need to grab a read From 756e438e0be444ad118d6d27e970c247c918dda2 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 12 Jan 2023 22:13:43 +0000 Subject: [PATCH 51/53] fix --- .../palantir/atlasdb/transaction/impl/SnapshotTransaction.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java index d768b322c06..5d42a7ccf86 100644 --- a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java +++ b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/SnapshotTransaction.java @@ -344,8 +344,7 @@ private enum State { timelockService, immutableTimestamp, knowledge); - this.expectationsDataCollectionMetrics = - ExpectationsDataCollectionMetrics.of(metricsManager.getTaggedRegistry()); + this.expectationsDataCollectionMetrics = ExpectationsMetrics.of(metricsManager.getTaggedRegistry()); } protected TransactionScopedCache getCache() { From 5598022ae0b365a5a2e801a331dd69c7ad94e8b4 Mon Sep 17 00:00:00 2001 From: Ahmed Alouane Date: Thu, 12 Jan 2023 22:46:06 +0000 Subject: [PATCH 52/53] exclude metrics file from license --- atlasdb-impl-shared/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlasdb-impl-shared/build.gradle b/atlasdb-impl-shared/build.gradle index d307057a38b..88f8467b9c1 100644 --- a/atlasdb-impl-shared/build.gradle +++ b/atlasdb-impl-shared/build.gradle @@ -12,7 +12,7 @@ license { exclude '**/PutUnlessExistsTableMetrics.java' exclude '**/TargetedSweepProgressMetrics.java' exclude '**/TimestampCorrectnessMetrics.java' - exclude '**/ExpectationsDataCollectionMetrics.java' + exclude '**/ExpectationsMetrics.java' exclude '**/VerificationModeMetrics.java' } From 908da7d44994626daccd123f8d189745ac281f6b Mon Sep 17 00:00:00 2001 From: aalouane <30903736+ergo14@users.noreply.github.com> Date: Thu, 12 Jan 2023 23:17:40 +0000 Subject: [PATCH 53/53] Autorelease 0.787.0-rc1 [skip ci] --- changelog/0.787.0-rc1/pr-6340.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/0.787.0-rc1/pr-6340.v2.yml diff --git a/changelog/0.787.0-rc1/pr-6340.v2.yml b/changelog/0.787.0-rc1/pr-6340.v2.yml new file mode 100644 index 00000000000..4b0aba4815f --- /dev/null +++ b/changelog/0.787.0-rc1/pr-6340.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: |- + Transactional expectations: enables metrics collections for key value service read calls on transactions post-mortem. + Metrics tracked per transaction: transaction age, number of bytes read, number of Atlas kvs method queries and number of bytes read for the worse Atlas kvs call (one with the most bytes read). + links: + - https://github.com/palantir/atlasdb/pull/6340