From 67065db60b9881dd55f92b18780d9679350bb045 Mon Sep 17 00:00:00 2001 From: Vadim Nabiev Date: Wed, 30 Mar 2022 16:14:51 +0300 Subject: [PATCH] [WIP] abstractions and API for querying to storages --- .../sections/RpcGeneratedSectionFactory.java | 5 +- .../sections/RpcSectionFactoryTest.java | 3 +- .../substrateclient/rpc/sections/State.java | 50 +++ .../rpc/sections/AuthorTests.java | 16 +- .../rpc/sections/ChainTests.java | 12 +- .../rpc/sections/StateTests.java | 130 ++++++- .../rpc/sections/SystemTests.java | 3 +- .../rpc/types/AccountIdReader.java | 21 + .../substrateclient/rpc/RpcImpl.java | 58 +++ settings.gradle | 2 + storage/build.gradle | 19 + .../substrateclient/storage/Arg.java | 17 + .../storage/Blake2B128Concat.java | 63 +++ .../storage/DiverseKeyValueCollection.java | 79 ++++ .../substrateclient/storage/Entry.java | 5 + .../HomogeneousKeyValueCollection.java | 57 +++ .../substrateclient/storage/Identity.java | 19 + .../storage/KeyCollection.java | 11 + .../storage/KeyCollectionImpl.java | 64 ++++ .../substrateclient/storage/KeyConsumer.java | 10 + .../substrateclient/storage/KeyHasher.java | 34 ++ .../storage/KeyHashingAlgorithm.java | 9 + .../storage/KeyValueCollection.java | 7 + .../storage/KeyValueConsumer.java | 9 + .../substrateclient/storage/MultiQuery.java | 7 + .../storage/PagedKeyCollection.java | 11 + .../storage/PagedKeyValueCollection.java | 11 + .../substrateclient/storage/QueryableKey.java | 22 ++ .../storage/QueryableKeyImpl.java | 103 +++++ .../storage/StorageChangeConsumer.java | 9 + .../storage/StorageDoubleMap.java | 12 + .../storage/StorageDoubleMapImpl.java | 36 ++ .../storage/StorageKeyProvider.java | 126 ++++++ .../substrateclient/storage/StorageMap.java | 17 + .../storage/StorageMapImpl.java | 41 ++ .../substrateclient/storage/StorageNMap.java | 43 +++ .../storage/StorageNMapImpl.java | 361 ++++++++++++++++++ .../substrateclient/storage/StorageValue.java | 18 + .../storage/StorageValueImpl.java | 48 +++ .../substrateclient/storage/TwoX64Concat.java | 87 +++++ .../storage/Blake2B128ConcatTests.java | 60 +++ .../storage/IdentityTests.java | 52 +++ .../storage/KeyHasherTests.java | 167 ++++++++ .../storage/StorageDoubleMapImplTests.java | 61 +++ .../storage/StorageKeyProviderTests.java | 253 ++++++++++++ .../storage/StorageMapImplTests.java | 50 +++ .../storage/StorageNMapImplTests.java | 273 +++++++++++++ .../storage/StorageValueImplTests.java | 73 ++++ .../storage/TwoX64ConcatTests.java | 73 ++++ 49 files changed, 2681 insertions(+), 36 deletions(-) create mode 100644 rpc/rpc-types/src/main/java/com/strategyobject/substrateclient/rpc/types/AccountIdReader.java create mode 100644 rpc/src/main/java/com/strategyobject/substrateclient/rpc/RpcImpl.java create mode 100644 storage/build.gradle create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/Arg.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/Blake2B128Concat.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/DiverseKeyValueCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/Entry.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/HomogeneousKeyValueCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/Identity.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollectionImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyConsumer.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHasher.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHashingAlgorithm.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueConsumer.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/MultiQuery.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyValueCollection.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKey.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKeyImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageChangeConsumer.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMap.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageKeyProvider.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMap.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMapImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMap.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMapImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValue.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValueImpl.java create mode 100644 storage/src/main/java/com/strategyobject/substrateclient/storage/TwoX64Concat.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/Blake2B128ConcatTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/IdentityTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/KeyHasherTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImplTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/StorageKeyProviderTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/StorageMapImplTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/StorageNMapImplTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/StorageValueImplTests.java create mode 100644 storage/src/test/java/com/strategyobject/substrateclient/storage/TwoX64ConcatTests.java diff --git a/rpc/rpc-codegen/src/main/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcGeneratedSectionFactory.java b/rpc/rpc-codegen/src/main/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcGeneratedSectionFactory.java index 73b3e4a4..6bf6e6dd 100644 --- a/rpc/rpc-codegen/src/main/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcGeneratedSectionFactory.java +++ b/rpc/rpc-codegen/src/main/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcGeneratedSectionFactory.java @@ -10,7 +10,10 @@ import static com.strategyobject.substrateclient.rpc.codegen.sections.Constants.CLASS_NAME_TEMPLATE; public class RpcGeneratedSectionFactory { - public T create(@NonNull Class interfaceClass, + private RpcGeneratedSectionFactory() { + } + + public static T create(@NonNull Class interfaceClass, @NonNull ProviderInterface provider) throws RpcInterfaceInitializationException { if (interfaceClass.getDeclaredAnnotationsByType(RpcInterface.class).length == 0) { throw new IllegalArgumentException( diff --git a/rpc/rpc-codegen/src/test/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcSectionFactoryTest.java b/rpc/rpc-codegen/src/test/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcSectionFactoryTest.java index dad4d714..12b92a2e 100644 --- a/rpc/rpc-codegen/src/test/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcSectionFactoryTest.java +++ b/rpc/rpc-codegen/src/test/java/com/strategyobject/substrateclient/rpc/codegen/sections/RpcSectionFactoryTest.java @@ -26,8 +26,7 @@ void createsRpcSectionAndCallsMethod() throws ExecutionException, InterruptedExc when(provider.send(anyString(), anyList())) .thenReturn(sendFuture); - val factory = new RpcGeneratedSectionFactory(); - val rpcSection = factory.create(TestSection.class, provider); + val rpcSection = RpcGeneratedSectionFactory.create(TestSection.class, provider); val actual = rpcSection.doNothing("some").get(); diff --git a/rpc/rpc-sections/src/main/java/com/strategyobject/substrateclient/rpc/sections/State.java b/rpc/rpc-sections/src/main/java/com/strategyobject/substrateclient/rpc/sections/State.java index 189272db..bccddaa6 100644 --- a/rpc/rpc-sections/src/main/java/com/strategyobject/substrateclient/rpc/sections/State.java +++ b/rpc/rpc-sections/src/main/java/com/strategyobject/substrateclient/rpc/sections/State.java @@ -2,11 +2,14 @@ import com.strategyobject.substrateclient.rpc.core.annotations.RpcCall; import com.strategyobject.substrateclient.rpc.core.annotations.RpcInterface; +import com.strategyobject.substrateclient.rpc.core.annotations.RpcSubscription; import com.strategyobject.substrateclient.rpc.types.*; import com.strategyobject.substrateclient.scale.annotations.Scale; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.Supplier; @RpcInterface(section = "state") public interface State { @@ -20,9 +23,56 @@ public interface State { @RpcCall(method = "getKeys") CompletableFuture> getKeys(StorageKey key); + @RpcCall(method = "getKeys") + CompletableFuture> getKeys(StorageKey key, @Scale BlockHash at); + + @RpcCall(method = "getKeysPaged") + CompletableFuture> getKeysPaged(StorageKey key, int count); + + @RpcCall(method = "getKeysPaged") + CompletableFuture> getKeysPaged(StorageKey key, int count, StorageKey startKey); + + @RpcCall(method = "getKeysPaged") + CompletableFuture> getKeysPaged(StorageKey key, + int count, + StorageKey startKey, + @Scale BlockHash at); + @RpcCall(method = "getStorage") CompletableFuture getStorage(StorageKey key); + @RpcCall(method = "getStorage") + CompletableFuture getStorage(StorageKey key, @Scale BlockHash at); + + @RpcCall(method = "getStorageHash") + @Scale + CompletableFuture getStorageHash(StorageKey key); + + @RpcCall(method = "getStorageHash") + @Scale + CompletableFuture getStorageHash(StorageKey key, @Scale BlockHash at); + + @RpcCall(method = "getStorageSize") + CompletableFuture getStorageSize(StorageKey key); + + @RpcCall(method = "getStorageSize") + CompletableFuture getStorageSize(StorageKey key, @Scale BlockHash at); + + @RpcCall(method = "queryStorage") + CompletableFuture> queryStorage(List keys, @Scale BlockHash fromBlock); + + @RpcCall(method = "queryStorage") + CompletableFuture> queryStorage(List keys, + @Scale BlockHash fromBlock, + @Scale BlockHash toBlock); + @RpcCall(method = "queryStorageAt") CompletableFuture> queryStorageAt(List keys); + + @RpcCall(method = "queryStorageAt") + CompletableFuture> queryStorageAt(List keys, @Scale BlockHash at); + + @RpcSubscription(type = "storage", subscribeMethod = "subscribeStorage", unsubscribeMethod = "unsubscribeStorage") + CompletableFuture>> subscribeStorage(List keys, + BiConsumer callback); } diff --git a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/AuthorTests.java b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/AuthorTests.java index a898bbfb..738ede2a 100644 --- a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/AuthorTests.java +++ b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/AuthorTests.java @@ -58,8 +58,7 @@ void hasKey() throws ExecutionException, InterruptedException, TimeoutException, .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - Author rpcSection = sectionFactory.create(Author.class, wsProvider); + Author rpcSection = RpcGeneratedSectionFactory.create(Author.class, wsProvider); val publicKey = PublicKey.fromBytes( HexConverter.toBytes("0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d")); @@ -83,8 +82,7 @@ void insertKey() throws ExecutionException, InterruptedException, TimeoutExcepti .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - Author rpcSection = sectionFactory.create(Author.class, wsProvider); + Author rpcSection = RpcGeneratedSectionFactory.create(Author.class, wsProvider); assertDoesNotThrow(() -> rpcSection.insertKey("aura", "alice", @@ -102,11 +100,10 @@ void submitExtrinsic() throws ExecutionException, InterruptedException, TimeoutE .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - Chain chainSection = sectionFactory.create(Chain.class, wsProvider); + Chain chainSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val genesis = chainSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); - Author authorSection = sectionFactory.create(Author.class, wsProvider); + Author authorSection = RpcGeneratedSectionFactory.create(Author.class, wsProvider); assertDoesNotThrow(() -> authorSection.submitExtrinsic(createBalanceTransferExtrinsic(genesis, NONCE.getAndIncrement())) .get(WAIT_TIMEOUT, TimeUnit.SECONDS)); @@ -121,11 +118,10 @@ void submitAndWatchExtrinsic() throws ExecutionException, InterruptedException, .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - Chain chainSection = sectionFactory.create(Chain.class, wsProvider); + Chain chainSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val genesis = chainSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); - Author authorSection = sectionFactory.create(Author.class, wsProvider); + Author authorSection = RpcGeneratedSectionFactory.create(Author.class, wsProvider); val updateCount = new AtomicInteger(0); val status = new AtomicReference(); diff --git a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/ChainTests.java b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/ChainTests.java index 687dd34a..dfa22c2e 100644 --- a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/ChainTests.java +++ b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/ChainTests.java @@ -40,8 +40,7 @@ void getFinalizedHead() throws ExecutionException, InterruptedException, Timeout .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - val rpcSection = sectionFactory.create(Chain.class, wsProvider); + val rpcSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val result = rpcSection.getFinalizedHead().get(WAIT_TIMEOUT, TimeUnit.SECONDS); @@ -57,8 +56,7 @@ void subscribeNewHeads() throws ExecutionException, InterruptedException, Timeou .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - val rpcSection = sectionFactory.create(Chain.class, wsProvider); + val rpcSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val blockCount = new AtomicInteger(0); val blockHash = new AtomicReference(null); @@ -91,8 +89,7 @@ void getBlockHash() throws ExecutionException, InterruptedException, TimeoutExce .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - val rpcSection = sectionFactory.create(Chain.class, wsProvider); + val rpcSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val result = rpcSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); @@ -108,8 +105,7 @@ void getBlock() throws ExecutionException, InterruptedException, TimeoutExceptio .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - Chain rpcSection = sectionFactory.create(Chain.class, wsProvider); + Chain rpcSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); val height = new AtomicInteger(0); rpcSection.subscribeNewHeads((e, header) -> { diff --git a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/StateTests.java b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/StateTests.java index d58dc705..6767c6b1 100644 --- a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/StateTests.java +++ b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/StateTests.java @@ -37,8 +37,7 @@ void getRuntimeVersion() throws ExecutionException, InterruptedException, Timeou .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); assertDoesNotThrow(() -> { rpcSection.getRuntimeVersion().get(WAIT_TIMEOUT, TimeUnit.SECONDS); @@ -54,8 +53,7 @@ void getMetadata() throws ExecutionException, InterruptedException, TimeoutExcep .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); assertDoesNotThrow(() -> { rpcSection.getMetadata().get(WAIT_TIMEOUT * 10, TimeUnit.SECONDS); @@ -71,8 +69,7 @@ void getKeys() throws ExecutionException, InterruptedException, TimeoutException .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 @@ -91,8 +88,7 @@ void getStorage() throws ExecutionException, InterruptedException, TimeoutExcept .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 @@ -112,8 +108,7 @@ void getStorageHandlesNullResponse() throws ExecutionException, InterruptedExcep .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); val emptyKey = new byte[32]; val storageData = rpcSection.getStorage(StorageKey.valueOf(emptyKey)).get(WAIT_TIMEOUT, TimeUnit.SECONDS); @@ -123,15 +118,124 @@ void getStorageHandlesNullResponse() throws ExecutionException, InterruptedExcep } @Test - void getStorageAt() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + void getStorageAtBlock() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { try (WsProvider wsProvider = WsProvider.builder() .setEndpoint(substrate.getWsAddress()) .disableAutoConnect() .build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - State rpcSection = sectionFactory.create(State.class, wsProvider); + val chainSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); + val blockHash = chainSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); + + // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f + // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 + val key = "0xc2261276cc9d1f8598ea4b6a74b15c2f308ce9615de0775a82f8a94dc3d285a1"; // TODO implement and use `xxhash` + val storageData = rpcSection.getStorage( + StorageKey.valueOf(HexConverter.toBytes(key)), + blockHash).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + assertNotNull(storageData); + assertTrue(storageData.getData().length > 0); + } + } + + @Test + void getStorageHash() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + try (WsProvider wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build()) { + wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); + + // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f + // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 + val key = "0xc2261276cc9d1f8598ea4b6a74b15c2f308ce9615de0775a82f8a94dc3d285a1"; // TODO implement and use `xxhash` + val hash = rpcSection.getStorageHash(StorageKey.valueOf(HexConverter.toBytes(key))).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + assertNotNull(hash); + assertTrue(hash.getData().length > 0); + } + } + + @Test + void getStorageHashAt() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + try (WsProvider wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build()) { + wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + val chainSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); + val blockHash = chainSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); + + // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f + // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 + val key = "0xc2261276cc9d1f8598ea4b6a74b15c2f308ce9615de0775a82f8a94dc3d285a1"; // TODO implement and use `xxhash` + val hash = rpcSection.getStorageHash( + StorageKey.valueOf(HexConverter.toBytes(key)), + blockHash).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + assertNotNull(hash); + assertTrue(hash.getData().length > 0); + } + } + + @Test + void getStorageSize() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + try (WsProvider wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build()) { + wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); + + // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f + // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 + val key = "0xc2261276cc9d1f8598ea4b6a74b15c2f308ce9615de0775a82f8a94dc3d285a1"; // TODO implement and use `xxhash` + val size = rpcSection.getStorageSize(StorageKey.valueOf(HexConverter.toBytes(key))).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + assertEquals(1, size); + } + } + + @Test + void getStorageSizeAt() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + try (WsProvider wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build()) { + wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + val chainSection = RpcGeneratedSectionFactory.create(Chain.class, wsProvider); + val blockHash = chainSection.getBlockHash(0).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); + + // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f + // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 + val key = "0xc2261276cc9d1f8598ea4b6a74b15c2f308ce9615de0775a82f8a94dc3d285a1"; // TODO implement and use `xxhash` + val size = rpcSection.getStorageSize( + StorageKey.valueOf(HexConverter.toBytes(key)), + blockHash).get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + assertEquals(1, size); + } + } + + @Test + void queryStorageAt() throws ExecutionException, InterruptedException, TimeoutException, RpcInterfaceInitializationException { + try (WsProvider wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build()) { + wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + State rpcSection = RpcGeneratedSectionFactory.create(State.class, wsProvider); // xxhash128("Balances") = 0xc2261276cc9d1f8598ea4b6a74b15c2f // xxhash128("StorageVersion") = 0x308ce9615de0775a82f8a94dc3d285a1 diff --git a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/SystemTests.java b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/SystemTests.java index c3dcb610..a727e590 100644 --- a/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/SystemTests.java +++ b/rpc/rpc-sections/src/test/java/com/strategyobject/substrateclient/rpc/sections/SystemTests.java @@ -32,8 +32,7 @@ void accountNextIndex() throws ExecutionException, InterruptedException, Timeout try (WsProvider wsProvider = WsProvider.builder().setEndpoint(substrate.getWsAddress()).disableAutoConnect().build()) { wsProvider.connect().get(WAIT_TIMEOUT, TimeUnit.SECONDS); - val sectionFactory = new RpcGeneratedSectionFactory(); - System rpcSection = sectionFactory.create(System.class, wsProvider); + System rpcSection = RpcGeneratedSectionFactory.create(System.class, wsProvider); val alicePublicKey = HexConverter.toBytes("0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d"); val result = rpcSection.accountNextIndex(AccountId.fromBytes(alicePublicKey)) diff --git a/rpc/rpc-types/src/main/java/com/strategyobject/substrateclient/rpc/types/AccountIdReader.java b/rpc/rpc-types/src/main/java/com/strategyobject/substrateclient/rpc/types/AccountIdReader.java new file mode 100644 index 00000000..9d7eff1d --- /dev/null +++ b/rpc/rpc-types/src/main/java/com/strategyobject/substrateclient/rpc/types/AccountIdReader.java @@ -0,0 +1,21 @@ +package com.strategyobject.substrateclient.rpc.types; + +import com.google.common.base.Preconditions; +import com.strategyobject.substrateclient.common.io.Streamer; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.annotations.AutoRegister; +import com.strategyobject.substrateclient.types.Size; +import lombok.NonNull; + +import java.io.IOException; +import java.io.InputStream; + +@AutoRegister(types = AccountId.class) +public class AccountIdReader implements ScaleReader { + @Override + public AccountId read(@NonNull InputStream stream, ScaleReader... readers) throws IOException { + Preconditions.checkArgument(readers == null || readers.length == 0); + + return AccountId.fromBytes(Streamer.readBytes(Size.of32.getValue(), stream)); + } +} diff --git a/rpc/src/main/java/com/strategyobject/substrateclient/rpc/RpcImpl.java b/rpc/src/main/java/com/strategyobject/substrateclient/rpc/RpcImpl.java new file mode 100644 index 00000000..c701c1c4 --- /dev/null +++ b/rpc/src/main/java/com/strategyobject/substrateclient/rpc/RpcImpl.java @@ -0,0 +1,58 @@ +package com.strategyobject.substrateclient.rpc; + +import com.strategyobject.substrateclient.rpc.codegen.sections.RpcGeneratedSectionFactory; +import com.strategyobject.substrateclient.rpc.codegen.sections.RpcInterfaceInitializationException; +import com.strategyobject.substrateclient.rpc.sections.Author; +import com.strategyobject.substrateclient.rpc.sections.Chain; +import com.strategyobject.substrateclient.rpc.sections.State; +import com.strategyobject.substrateclient.rpc.sections.System; +import com.strategyobject.substrateclient.transport.ProviderInterface; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class RpcImpl implements Rpc, AutoCloseable { + private final ProviderInterface providerInterface; + + @Override + public Author getAuthor() { + try { + return RpcGeneratedSectionFactory.create(Author.class, providerInterface); + } catch (RpcInterfaceInitializationException e) { + throw new RuntimeException(e); + } + } + + @Override + public Chain getChain() { + try { + return RpcGeneratedSectionFactory.create(Chain.class, providerInterface); + } catch (RpcInterfaceInitializationException e) { + throw new RuntimeException(e); + } + } + + @Override + public State getState() { + try { + return RpcGeneratedSectionFactory.create(State.class, providerInterface); + } catch (RpcInterfaceInitializationException e) { + throw new RuntimeException(e); + } + } + + @Override + public System getSystem() { + try { + return RpcGeneratedSectionFactory.create(System.class, providerInterface); + } catch (RpcInterfaceInitializationException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + if (providerInterface instanceof AutoCloseable) { + ((AutoCloseable) providerInterface).close(); + } + } +} diff --git a/settings.gradle b/settings.gradle index 8e765dce..ee9ac553 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,3 +14,5 @@ include 'scale:scale-codegen' include 'tests' include 'transport' include 'types' +include 'storage' + diff --git a/storage/build.gradle b/storage/build.gradle new file mode 100644 index 00000000..d69ffba5 --- /dev/null +++ b/storage/build.gradle @@ -0,0 +1,19 @@ +dependencies { + implementation project(':common') + implementation project(':scale') + implementation project(':crypto') + implementation project(':types') + implementation project(':rpc') + implementation project(':rpc:rpc-types') + implementation project(':rpc:rpc-sections') + + implementation 'org.bouncycastle:bcprov-jdk15on:1.70' + implementation 'net.openhft:zero-allocation-hashing:0.15' + + testImplementation project(':tests') + testCompileOnly project(':transport') + + testImplementation 'org.testcontainers:testcontainers:1.16.3' + testImplementation 'org.testcontainers:junit-jupiter:1.16.3' + testImplementation 'ch.qos.logback:logback-classic:1.2.6' +} \ No newline at end of file diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/Arg.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/Arg.java new file mode 100644 index 00000000..8f1bdd7b --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/Arg.java @@ -0,0 +1,17 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.Getter; +import lombok.NonNull; + +public class Arg { + @Getter + private final Object[] list; + + private Arg(Object[] list) { + this.list = list; + } + + public static Arg of(@NonNull Object... keys) { + return new Arg(keys); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/Blake2B128Concat.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/Blake2B128Concat.java new file mode 100644 index 00000000..61b0828c --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/Blake2B128Concat.java @@ -0,0 +1,63 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; +import lombok.val; +import lombok.var; +import org.bouncycastle.crypto.digests.Blake2bDigest; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.function.Consumer; + +public class Blake2B128Concat implements KeyHashingAlgorithm { + private static final int BLAKE_128_HASH_SIZE = 16; + private static volatile Blake2B128Concat instance; + + private static byte[] blake2_128(byte[] value) { + val digest = new Blake2bDigest(128); + digest.update(value, 0, value.length); + + val result = new byte[digest.getDigestSize()]; + digest.doFinal(result, 0); + return result; + } + + public static Blake2B128Concat getInstance() { + if (instance == null) { + synchronized (Blake2B128Concat.class) { + if (instance == null) { + instance = new Blake2B128Concat(); + } + } + } + return instance; + } + + @Override + public byte[] getHash(byte @NonNull [] encodedKey) { + return ByteBuffer.allocate(BLAKE_128_HASH_SIZE + encodedKey.length) + .put(blake2_128(encodedKey)) + .put(encodedKey) + .array(); + } + + @Override + public int hashSize() { + return BLAKE_128_HASH_SIZE; + } + + public byte[] deriveKey(byte @NonNull [] suffix, @NonNull Consumer consumer) { + val hash = Arrays.copyOfRange(suffix, 0, BLAKE_128_HASH_SIZE); + var key = new byte[0]; + var keyLength = 0; + + while (!Arrays.equals(hash, blake2_128(key))) { + keyLength++; + key = Arrays.copyOfRange(suffix, BLAKE_128_HASH_SIZE, BLAKE_128_HASH_SIZE + keyLength); + } + + consumer.accept(key); + + return Arrays.copyOfRange(suffix, BLAKE_128_HASH_SIZE + keyLength, suffix.length); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/DiverseKeyValueCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/DiverseKeyValueCollection.java new file mode 100644 index 00000000..d3a3e991 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/DiverseKeyValueCollection.java @@ -0,0 +1,79 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.StorageData; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; +import lombok.val; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +public class DiverseKeyValueCollection implements KeyValueCollection { + private final List> valueReaders; + private final List> pairs; + private final List>> keyExtractors; + + private DiverseKeyValueCollection(List> pairs, + List> valueReaders, + List>> keyExtractors) { + if (pairs.size() != valueReaders.size()) { + throw new IllegalArgumentException("Number of value readers doesn't match number of pairs."); + } + + if (pairs.size() != keyExtractors.size()) { + throw new IllegalArgumentException("Number of extractors doesn't match number of pairs."); + } + + this.valueReaders = valueReaders; + this.pairs = pairs; + this.keyExtractors = keyExtractors; + } + + public static KeyValueCollection with(@NonNull List> pairs, + List> valueReaders, + @NonNull List>> keyExtractors) { + return new DiverseKeyValueCollection(pairs, valueReaders, keyExtractors); + } + + @Override + public Iterator> iterator() { + return new Iterator>() { + private final Iterator> underlying = pairs.iterator(); + private final Iterator> readers = valueReaders.iterator(); + private final Iterator>> extractors = keyExtractors.iterator(); + + @Override + public boolean hasNext() { + return underlying.hasNext(); + } + + @Override + public Entry next() { + if (hasNext()) { + val pair = underlying.next(); + + return consumer -> { + try { + val value = pair.getValue1() == null ? + null : + readers.next().read(new ByteArrayInputStream(pair.getValue1().getData())); + val keys = extractors.next().apply(pair.getValue0()); + + consumer.accept(value, keys); + } catch (IOException e) { + throw new RuntimeException(); + } + }; + } + + throw new NoSuchElementException(); + } + }; + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/Entry.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/Entry.java new file mode 100644 index 00000000..31c1e7fb --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/Entry.java @@ -0,0 +1,5 @@ +package com.strategyobject.substrateclient.storage; + +public interface Entry { + void consume(KeyValueConsumer consumer); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/HomogeneousKeyValueCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/HomogeneousKeyValueCollection.java new file mode 100644 index 00000000..38c6ec6c --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/HomogeneousKeyValueCollection.java @@ -0,0 +1,57 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.StorageData; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.val; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +@RequiredArgsConstructor(staticName = "with") +class HomogeneousKeyValueCollection implements KeyValueCollection { + private final @NonNull List> pairs; + private final @NonNull ScaleReader valueReader; + private final @NonNull Function> keyExtractor; + + @Override + public Iterator> iterator() { + return new Iterator>() { + private final Iterator> underlying = pairs.iterator(); + + @Override + public boolean hasNext() { + return underlying.hasNext(); + } + + @Override + public Entry next() { + if (hasNext()) { + val pair = underlying.next(); + + return consumer -> { + try { + val value = pair.getValue1() == null ? + null : + valueReader.read(new ByteArrayInputStream(pair.getValue1().getData())); + val keys = keyExtractor.apply(pair.getValue0()); + + consumer.accept(value, keys); + } catch (IOException e) { + throw new RuntimeException(); + } + }; + } + + throw new NoSuchElementException(); + } + }; + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/Identity.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/Identity.java new file mode 100644 index 00000000..41512e62 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/Identity.java @@ -0,0 +1,19 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor(staticName = "sizeOf") +public class Identity implements KeyHashingAlgorithm { + private final int size; + + @Override + public byte[] getHash(byte @NonNull [] encodedKey) { + return encodedKey; + } + + @Override + public int hashSize() { + return 0; + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollection.java new file mode 100644 index 00000000..a46065bd --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollection.java @@ -0,0 +1,11 @@ +package com.strategyobject.substrateclient.storage; + +import java.util.Iterator; + +public interface KeyCollection { + int size(); + + MultiQuery multi(); + + Iterator> iterator(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollectionImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollectionImpl.java new file mode 100644 index 00000000..f63d8607 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyCollectionImpl.java @@ -0,0 +1,64 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.val; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor(staticName = "with") +public class KeyCollectionImpl implements KeyCollection { + private final @NonNull Rpc rpc; + private final @NonNull List storageKeys; + private final @NonNull ScaleReader valueReader; + private final @NonNull Function> keyExtractor; + + @Override + public int size() { + return storageKeys.size(); + } + + @Override + public MultiQuery multi() { + return () -> rpc + .getState() + .queryStorageAt(storageKeys) + .thenApplyAsync(changeSets -> HomogeneousKeyValueCollection.with( + changeSets.stream() + .flatMap(set -> set.getChanges().stream()) + .collect(Collectors.toList()), + valueReader, + keyExtractor + )); + } + + @Override + public Iterator> iterator() { + return new Iterator>() { + private final Iterator underlying = storageKeys.iterator(); + + @Override + public boolean hasNext() { + return underlying.hasNext(); + } + + @Override + public QueryableKey next() { + if (hasNext()) { + val key = underlying.next(); + + return QueryableKeyImpl.with(rpc, key, valueReader, keyExtractor); + } + + throw new NoSuchElementException(); + } + }; + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyConsumer.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyConsumer.java new file mode 100644 index 00000000..d908a05c --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyConsumer.java @@ -0,0 +1,10 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; + +import java.util.List; + +@FunctionalInterface +public interface KeyConsumer { + void accept(@NonNull List keys); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHasher.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHasher.java new file mode 100644 index 00000000..1530c9cf --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHasher.java @@ -0,0 +1,34 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.val; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +@RequiredArgsConstructor(staticName = "with") +public class KeyHasher { + private final @NonNull ScaleWriter keyWriter; + private final @NonNull ScaleReader keyReader; + private final @NonNull KeyHashingAlgorithm algorithm; + + public byte[] getHash(T key) throws IOException { + val buf = new ByteArrayOutputStream(); + keyWriter.write(key, buf); + + return algorithm.getHash(buf.toByteArray()); + } + + public T extractKey(@NonNull InputStream storageKeySuffix) throws IOException { + val skip = storageKeySuffix.skip(algorithm.hashSize()); + if (skip != algorithm.hashSize()) { + throw new RuntimeException("Stream couldn't skip the hash."); + } + + return keyReader.read(storageKeySuffix); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHashingAlgorithm.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHashingAlgorithm.java new file mode 100644 index 00000000..bd1fa492 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyHashingAlgorithm.java @@ -0,0 +1,9 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; + +public interface KeyHashingAlgorithm { + byte[] getHash(byte @NonNull [] encodedKey); + + int hashSize(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueCollection.java new file mode 100644 index 00000000..ac4e59af --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueCollection.java @@ -0,0 +1,7 @@ +package com.strategyobject.substrateclient.storage; + +import java.util.Iterator; + +public interface KeyValueCollection { + Iterator> iterator(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueConsumer.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueConsumer.java new file mode 100644 index 00000000..2a39de90 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/KeyValueConsumer.java @@ -0,0 +1,9 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; + +import java.util.List; + +public interface KeyValueConsumer { + void accept(V value, @NonNull List keys); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/MultiQuery.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/MultiQuery.java new file mode 100644 index 00000000..db2a599c --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/MultiQuery.java @@ -0,0 +1,7 @@ +package com.strategyobject.substrateclient.storage; + +import java.util.concurrent.CompletableFuture; + +public interface MultiQuery { + CompletableFuture> execute(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyCollection.java new file mode 100644 index 00000000..89720ebd --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyCollection.java @@ -0,0 +1,11 @@ +package com.strategyobject.substrateclient.storage; + +import java.util.concurrent.CompletableFuture; + +public interface PagedKeyCollection { + int number(); + + CompletableFuture moveNext(); + + KeyCollection current(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyValueCollection.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyValueCollection.java new file mode 100644 index 00000000..1561dffd --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/PagedKeyValueCollection.java @@ -0,0 +1,11 @@ +package com.strategyobject.substrateclient.storage; + +import java.util.concurrent.CompletableFuture; + +public interface PagedKeyValueCollection { + int number(); + + CompletableFuture moveNext(); + + KeyValueCollection current(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKey.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKey.java new file mode 100644 index 00000000..78248d59 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKey.java @@ -0,0 +1,22 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import lombok.NonNull; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public interface QueryableKey { + CompletableFuture execute(); + + StorageKey getKey(); + + ScaleReader getValueReader(); + + void consume(@NonNull KeyConsumer consumer); + + MultiQuery join(@NonNull QueryableKey... others); + + List extractKeys(@NonNull StorageKey fullKey); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKeyImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKeyImpl.java new file mode 100644 index 00000000..0791fa50 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/QueryableKeyImpl.java @@ -0,0 +1,103 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.val; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor(staticName = "with") +class QueryableKeyImpl implements QueryableKey{ + private final Rpc rpc; + private final StorageKey key; + private final ScaleReader valueReader; + private final Function> keyExtractor; + + @Override + public CompletableFuture execute() { + return rpc + .getState() + .getStorage(key) + .thenApplyAsync(d -> { + if (d == null) { + return null; + } + + try { + return valueReader.read(new ByteArrayInputStream(d.getData())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public StorageKey getKey() { + return key; + } + + @Override + public ScaleReader getValueReader() { + return valueReader; + } + + @Override + public void consume(@NonNull KeyConsumer consumer) { + val keys = keyExtractor.apply(key); + + consumer.accept(keys); + } + + @Override + public MultiQuery join(QueryableKey... others) { + return () -> rpc + .getState() + .queryStorageAt(concatKeys(others)) + .thenApplyAsync(changeSets -> DiverseKeyValueCollection.with( + changeSets.stream() + .flatMap(set -> set.getChanges().stream()) + .collect(Collectors.toList()), + concatReaders(others), + concatKeyExtractors(others) + + )); + } + + @Override + public List extractKeys(@NonNull StorageKey fullKey) { + return keyExtractor.apply(fullKey); + } + + private ArrayList concatKeys(QueryableKey[] others) { + val list = new ArrayList<>(Collections.singletonList(getKey())); + list.addAll(Arrays.stream(others).map(QueryableKey::getKey).collect(Collectors.toList())); + + return list; + } + + @SuppressWarnings("unchecked") + private List> concatReaders(QueryableKey[] others) { + val list = new ArrayList<>(Collections.singletonList((ScaleReader)getValueReader())); + list.addAll(Arrays.stream(others).map(q -> (ScaleReader)q.getValueReader()).collect(Collectors.toList())); + + return list; + } + + private ArrayList>> concatKeyExtractors(QueryableKey[] others) { + val list = new ArrayList>>(Collections.singletonList(this::extractKeys)); + list.addAll(Arrays.stream(others).map(q -> (Function>) q::extractKeys).collect(Collectors.toList())); + + return list; + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageChangeConsumer.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageChangeConsumer.java new file mode 100644 index 00000000..3052a972 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageChangeConsumer.java @@ -0,0 +1,9 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; + +import java.util.List; + +public interface StorageChangeConsumer { + void accept(Exception exception, BlockHash block, V value, List keys); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMap.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMap.java new file mode 100644 index 00000000..d104ba39 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMap.java @@ -0,0 +1,12 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import lombok.NonNull; + +import java.util.concurrent.CompletableFuture; + +public interface StorageDoubleMap { + CompletableFuture get(@NonNull F first, @NonNull S second); + + CompletableFuture at(@NonNull BlockHash block, @NonNull F first, @NonNull S second); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImpl.java new file mode 100644 index 00000000..074e094f --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImpl.java @@ -0,0 +1,36 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +import java.util.concurrent.CompletableFuture; + +@RequiredArgsConstructor(staticName = "with") +public class StorageDoubleMapImpl implements StorageDoubleMap { + private final StorageNMap underlying; + + private StorageDoubleMapImpl(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + underlying = StorageNMapImpl.with(rpc, scaleReader, storageKeyProvider); + } + + public static StorageDoubleMap with(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + return new StorageDoubleMapImpl<>(rpc, scaleReader, storageKeyProvider); + } + + @Override + public CompletableFuture get(@NonNull F first, @NonNull S second) { + return underlying.get(first, second); + } + + @Override + public CompletableFuture at(@NonNull BlockHash block, @NonNull F first, @NonNull S second) { + return underlying.at(block, first, second); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageKeyProvider.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageKeyProvider.java new file mode 100644 index 00000000..b11e787a --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageKeyProvider.java @@ -0,0 +1,126 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import lombok.*; +import net.openhft.hashing.LongHashFunction; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class StorageKeyProvider { + private static final int XX_HASH_SIZE = 16; + @Getter(AccessLevel.PACKAGE) + private final List> keyHashers = new ArrayList<>(); + private final ByteBuffer keyPrefix; + + private StorageKeyProvider(String palletName, String storageName) { + keyPrefix = ByteBuffer.allocate(XX_HASH_SIZE * 2); + + xxhash128(keyPrefix, palletName); + xxhash128(keyPrefix, storageName); + } + + private static void xxhash128(ByteBuffer buf, String value) { + val encodedValue = value.getBytes(StandardCharsets.UTF_8); + + final ByteOrder sourceOrder = buf.order(); // final ByteOrder instead of val because of checkstyle + buf.order(ByteOrder.LITTLE_ENDIAN); + + buf.asLongBuffer() + .put(LongHashFunction.xx(0).hashBytes(encodedValue)) + .put(LongHashFunction.xx(1).hashBytes(encodedValue)); + + buf.position(buf.position() + XX_HASH_SIZE); + buf.order(sourceOrder); + } + + public static StorageKeyProvider with(@NonNull String palletName, @NonNull String storageName) { + return new StorageKeyProvider(palletName, storageName); + } + + public StorageKeyProvider with(@NonNull KeyHasher... hashers) { + if (keyHashers.size() > 0) { + throw new IllegalStateException("Key hashers are already set."); + } + + Collections.addAll(keyHashers, hashers); + + return this; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + public StorageKey get(@NonNull Object... keys) { + if (keys.length > keyHashers.size()) { + throw new IndexOutOfBoundsException( + String.format("Number of keys mustn't exceed capacity. Passed: %s, capacity: %s.", + keys.length, + keyHashers.size())); + } + + val stream = new ByteArrayOutputStream(); + try { + stream.write(keyPrefix.array()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + for (var i = 0; i < keys.length; i++) { + try { + stream.write(((KeyHasher) keyHashers.get(i)).getHash(keys[i])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return StorageKey.valueOf(stream.toByteArray()); + } + + public int countOfKeys() { + return keyHashers.size(); + } + + public List extractKeys(StorageKey fullKey) { + val stream = new ByteArrayInputStream(Arrays.copyOfRange(fullKey.getData(), + keyPrefix.capacity(), + fullKey.getData().length)); + + return keyHashers + .stream() + .map(keyHasher -> { + try { + return keyHasher.extractKey(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + public List extractKeys(StorageKey fullKey, StorageKey queryKey, int countOfKeysInQuery) { + val stream = new ByteArrayInputStream(Arrays.copyOfRange(fullKey.getData(), + queryKey.getData().length, + fullKey.getData().length)); + + return keyHashers + .stream() + .skip(countOfKeysInQuery) + .map(keyHasher -> { + try { + return keyHasher.extractKey(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMap.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMap.java new file mode 100644 index 00000000..b6c9e6e4 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMap.java @@ -0,0 +1,17 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public interface StorageMap { + CompletableFuture get(@NonNull K key); + + CompletableFuture at(@NonNull BlockHash block, @NonNull K key); + + CompletableFuture>>> history(@NonNull K key); +} + diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMapImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMapImpl.java new file mode 100644 index 00000000..554cbf2b --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageMapImpl.java @@ -0,0 +1,41 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public class StorageMapImpl implements StorageMap { + private final StorageNMap underlying; + + private StorageMapImpl(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + underlying = StorageNMapImpl.with(rpc, scaleReader, storageKeyProvider); + } + + public static StorageMap with(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + return new StorageMapImpl<>(rpc, scaleReader, storageKeyProvider); + } + + @Override + public CompletableFuture get(@NonNull K key) { + return underlying.get(key); + } + + @Override + public CompletableFuture at(@NonNull BlockHash block, @NonNull K key) { + return underlying.at(block, key); + } + + @Override + public CompletableFuture>>> history(@NonNull K key) { + return underlying.history(key); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMap.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMap.java new file mode 100644 index 00000000..d47d9d42 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMap.java @@ -0,0 +1,43 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.rpc.types.Hash; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; + +public interface StorageNMap { + CompletableFuture get(@NonNull Object... keys); + + CompletableFuture at(@NonNull BlockHash block, @NonNull Object... keys); + + CompletableFuture hash(@NonNull Object... keys); + + CompletableFuture hashAt(@NonNull BlockHash block, @NonNull Object... keys); + + CompletableFuture size(@NonNull Object... keys); + + CompletableFuture sizeAt(@NonNull BlockHash block, @NonNull Object... keys); + + CompletableFuture> keys(@NonNull Object... keys); + + CompletableFuture> keysAt(@NonNull BlockHash block, @NonNull Object... keys); + + PagedKeyCollection keysPaged(int count, @NonNull Object... keys); + + QueryableKey query(@NonNull Object... keys); + + CompletableFuture>>> history(@NonNull Object... keys); + + CompletableFuture> entries(@NonNull Object... keys); + + PagedKeyValueCollection entriesPaged(int count, @NonNull Object... keys); + + CompletableFuture> multi(@NonNull Arg... args); + + CompletableFuture>> subscribe(@NonNull StorageChangeConsumer consumer, + @NonNull Arg... args); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMapImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMapImpl.java new file mode 100644 index 00000000..01aabd29 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageNMapImpl.java @@ -0,0 +1,361 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.rpc.types.Hash; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.val; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +@RequiredArgsConstructor(staticName = "with") +public class StorageNMapImpl implements StorageNMap { + private final @NonNull Rpc rpc; + private final @NonNull ScaleReader scaleReader; + private final @NonNull StorageKeyProvider storageKeyProvider; + + @Override + public CompletableFuture get(@NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorage(storageKeyProvider.get(keys)) + .thenApplyAsync(d -> { + if (d == null) { + return null; + } + + try { + return scaleReader.read(new ByteArrayInputStream(d.getData())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void ensureAllKeysWerePassed(@NonNull Object[] keys) { + if (keys.length != storageKeyProvider.countOfKeys()) { + throw new IndexOutOfBoundsException(String.format("Incorrect number of keys were passed. Passed: %s, expected: %s.", + keys.length, + storageKeyProvider.countOfKeys())); + } + } + + @Override + public CompletableFuture at(@NonNull BlockHash block, @NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorage(storageKeyProvider.get(keys), block) + .thenApplyAsync(d -> { + if (d == null) { + return null; + } + + try { + return scaleReader.read(new ByteArrayInputStream(d.getData())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public CompletableFuture hash(@NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorageHash(storageKeyProvider.get(keys)); + } + + @Override + public CompletableFuture hashAt(@NonNull BlockHash block, @NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorageHash(storageKeyProvider.get(keys), block); + } + + @Override + public CompletableFuture size(@NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorageSize(storageKeyProvider.get(keys)); + } + + @Override + public CompletableFuture sizeAt(@NonNull BlockHash block, @NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .getStorageSize(storageKeyProvider.get(keys), block); + } + + @Override + public CompletableFuture> keys(@NonNull Object... keys) { + val queryKey = storageKeyProvider.get(keys); + return rpc + .getState() + .getKeys(queryKey) + .thenApplyAsync(storageKeys -> KeyCollectionImpl.with(rpc, + storageKeys, + scaleReader, + key -> storageKeyProvider.extractKeys(key, queryKey, keys.length))); + } + + @Override + public CompletableFuture> keysAt(@NonNull BlockHash block, @NonNull Object... keys) { + val queryKey = storageKeyProvider.get(keys); + return rpc + .getState() + .getKeys(queryKey, block) + .thenApplyAsync(storageKeys -> KeyCollectionImpl.with(rpc, + storageKeys, + scaleReader, + key -> storageKeyProvider.extractKeys(key, queryKey, keys.length))); + } + + @Override + public PagedKeyCollection keysPaged(int count, @NonNull Object... keys) { + if (count <= 0) { + throw new IllegalArgumentException("Number of elements per page must be positive."); + } + + val queryKey = storageKeyProvider.get(keys); + + return new PagedKeyCollection() { + private int pageNumber = 0; + private int size = 0; + private boolean hasNext = false; + private StorageKey last; + private KeyCollection collection; + + @Override + public int number() { + if (!hasNext) { + throw new NoSuchElementException(); + } + + return pageNumber; + } + + @Override + public CompletableFuture moveNext() { + val task = pageNumber == 0 ? + rpc.getState().getKeysPaged(queryKey, count) : + size == count ? + rpc.getState().getKeysPaged(queryKey, count, last) : + CompletableFuture.completedFuture(Collections.emptyList()); + + return task + .thenApplyAsync(storageKeys -> { + if ((size = storageKeys.size()) > 0) { + hasNext = true; + pageNumber++; + last = storageKeys.get(storageKeys.size() - 1); + collection = KeyCollectionImpl.with(rpc, + storageKeys, + scaleReader, + key -> storageKeyProvider.extractKeys(key, queryKey, keys.length)); + } else { + hasNext = false; + } + + return hasNext; + }); + } + + @Override + public KeyCollection current() { + if (!hasNext) { + throw new NoSuchElementException(); + } + + return collection; + } + }; + } + + @Override + public QueryableKey query(@NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return QueryableKeyImpl.with(rpc, + storageKeyProvider.get(keys), + scaleReader, + storageKeyProvider::extractKeys); + } + + @Override + public CompletableFuture>>> history(@NonNull Object... keys) { + ensureAllKeysWerePassed(keys); + + return rpc + .getState() + .queryStorageAt(Collections.singletonList(storageKeyProvider.get(keys))) + .thenApplyAsync(set -> set + .stream() + .map(s -> Pair.of( + s.getBlock(), + s + .getChanges() + .stream() + .map(p -> Optional.of(p.getValue1()) + .map(v -> { + try { + return scaleReader.read(new ByteArrayInputStream(v.getData())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .orElse(null)) + .collect(Collectors.toList()))) + .collect(Collectors.toList())); + } + + @Override + public CompletableFuture> entries(@NonNull Object... keys) { + return keys(keys).thenComposeAsync(k -> k.multi().execute()); + } + + @Override + public PagedKeyValueCollection entriesPaged(int count, @NonNull Object... keys) { + return new PagedKeyValueCollection() { + private final PagedKeyCollection underlying = keysPaged(count, keys); + private KeyValueCollection collection; + + @Override + public int number() { + if (collection == null) { + throw new NoSuchElementException(); + } + + return underlying.number(); + } + + @Override + public CompletableFuture moveNext() { + val values = new CompletableFuture>(); + underlying.moveNext() + .whenCompleteAsync((next, throwable) -> { + if (throwable != null) { + values.completeExceptionally(throwable); + } else { + if (next) { + underlying.current().multi().execute() + .whenCompleteAsync((keyValues, e) -> { + if (e != null) { + values.completeExceptionally(e); + } else { + values.complete(keyValues); + } + }); + } else { + values.complete(null); + } + } + }); + + return values + .whenCompleteAsync((keyValues, throwable) -> { + if (throwable == null) { + collection = keyValues; + } + }).thenApplyAsync(Objects::nonNull); + } + + @Override + public KeyValueCollection current() { + if (collection == null) { + throw new NoSuchElementException(); + } + + return collection; + } + }; + } + + @Override + public CompletableFuture> multi(@NonNull Arg... args) { + if (args.length == 0) { + throw new IllegalArgumentException("Multi requests are not supported without arguments."); + } + + val keys = Arrays.stream(args) + .map(a -> { + ensureAllKeysWerePassed(a.getList()); + + return storageKeyProvider.get(a.getList()); + }) + .collect(Collectors.toList()); + + return rpc + .getState() + .queryStorageAt(keys) + .thenApplyAsync(set -> { + val pairs = set + .stream() + .flatMap(changeSet -> changeSet.getChanges().stream()) + .collect(Collectors.toList()); + + return HomogeneousKeyValueCollection.with(pairs, + scaleReader, + storageKeyProvider::extractKeys); + }); + } + + @Override + public CompletableFuture>> subscribe(@NonNull StorageChangeConsumer consumer, @NonNull Arg... args) { + if (args.length == 0) { + throw new IllegalArgumentException("Subscription can't be requested with no arguments."); + } + + val queryKeys = Arrays.stream(args) + .map(a -> { + ensureAllKeysWerePassed(a.getList()); + + return storageKeyProvider.get(a.getList()); + }) + .collect(Collectors.toList()); + + return rpc + .getState() + .subscribeStorage(queryKeys, (e, changeSet) -> { + if (e != null) { + consumer.accept(e, null, null, null); + } else { + changeSet.getChanges().forEach( + p -> { + try { + val value = p.getValue1() == null ? + null : + scaleReader.read(new ByteArrayInputStream(p.getValue1().getData())); + + val keys = storageKeyProvider.extractKeys(p.getValue0()); + consumer.accept(null, changeSet.getBlock(), value, keys); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + ); + } + }); + } + +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValue.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValue.java new file mode 100644 index 00000000..54ef80da --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValue.java @@ -0,0 +1,18 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public interface StorageValue { + CompletableFuture get(); + + CompletableFuture at(@NonNull BlockHash block); + + CompletableFuture>>> history(); + + QueryableKey query(); +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValueImpl.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValueImpl.java new file mode 100644 index 00000000..11224e22 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/StorageValueImpl.java @@ -0,0 +1,48 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.Rpc; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@RequiredArgsConstructor(staticName = "with") +public class StorageValueImpl implements StorageValue { + private final StorageNMap underlying; + + private StorageValueImpl(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + underlying = StorageNMapImpl.with(rpc, scaleReader, storageKeyProvider); + } + + public static StorageValue with(@NonNull Rpc rpc, + @NonNull ScaleReader scaleReader, + @NonNull StorageKeyProvider storageKeyProvider) { + return new StorageValueImpl<>(rpc, scaleReader, storageKeyProvider); + } + + @Override + public CompletableFuture get() { + return underlying.get(); + } + + @Override + public CompletableFuture at(@NonNull BlockHash block) { + return underlying.at(block); + } + + @Override + public CompletableFuture>>> history() { + return underlying.history(); + } + + @Override + public QueryableKey query() { + return underlying.query(); + } +} diff --git a/storage/src/main/java/com/strategyobject/substrateclient/storage/TwoX64Concat.java b/storage/src/main/java/com/strategyobject/substrateclient/storage/TwoX64Concat.java new file mode 100644 index 00000000..cf9c50f8 --- /dev/null +++ b/storage/src/main/java/com/strategyobject/substrateclient/storage/TwoX64Concat.java @@ -0,0 +1,87 @@ +package com.strategyobject.substrateclient.storage; + +import lombok.NonNull; +import lombok.val; +import lombok.var; +import net.openhft.hashing.LongHashFunction; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.function.Consumer; + +public class TwoX64Concat implements KeyHashingAlgorithm { + private static final int XX_HASH_SIZE = 8; + private static volatile TwoX64Concat instance; + + private static void xxhash64(ByteBuffer buf, byte[] value) { + val sourceOrder = buf.order(); + buf.order(ByteOrder.LITTLE_ENDIAN); + + buf.asLongBuffer() + .put(LongHashFunction.xx(0).hashBytes(value)); + + buf.position(buf.position() + XX_HASH_SIZE); + buf.order(sourceOrder); + } + + private static byte[] xxhash64(byte[] value) { + val hash = LongHashFunction.xx(0).hashBytes(value); + + return new byte[]{ + (byte) ((hash) & 0xff), + (byte) ((hash >> 8) & 0xff), + (byte) ((hash >> 16) & 0xff), + (byte) ((hash >> 24) & 0xff), + (byte) ((hash >> 32) & 0xff), + (byte) ((hash >> 40) & 0xff), + (byte) ((hash >> 48) & 0xff), + (byte) ((hash >> 56) & 0xff) + }; + } + + private static byte[] xxhash64_concat(byte[] value) { + val buf = ByteBuffer.allocate(XX_HASH_SIZE + value.length); + + xxhash64(buf, value); + buf.put(value); + + return buf.array(); + } + + public static TwoX64Concat getInstance() { + if (instance == null) { + synchronized (TwoX64Concat.class) { + if (instance == null) { + instance = new TwoX64Concat(); + } + } + } + return instance; + } + + @Override + public byte[] getHash(byte @NonNull [] encodedKey) { + return xxhash64_concat(encodedKey); + } + + @Override + public int hashSize() { + return XX_HASH_SIZE; + } + + public byte[] deriveKey(byte @NonNull [] suffix, @NonNull Consumer consumer) { + val hash = Arrays.copyOfRange(suffix, 0, XX_HASH_SIZE); + var key = new byte[0]; + var keyLength = 0; + + while (!Arrays.equals(hash, xxhash64(key))) { + keyLength++; + key = Arrays.copyOfRange(suffix, XX_HASH_SIZE, XX_HASH_SIZE + keyLength); + } + + consumer.accept(key); + + return Arrays.copyOfRange(suffix, XX_HASH_SIZE + keyLength, suffix.length); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/Blake2B128ConcatTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/Blake2B128ConcatTests.java new file mode 100644 index 00000000..49a7222d --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/Blake2B128ConcatTests.java @@ -0,0 +1,60 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.common.utils.HexConverter; +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayOutputStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class Blake2B128ConcatTests { + private static Stream getTestCasesForGetHash() { + return Stream.of( + Arguments.of( + decode(Integer.class, 5), + "0x969e061847da7e84337ea78dc577cd1d05000000" + ), + Arguments.of( + decode(String.class, "SomeString"), + "0x054439e9cc46decbb601602ece91182c28536f6d65537472696e67" + ), + Arguments.of( + decode(AccountId.class, + AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress())), + "0xde1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" + ) + ); + } + + @SneakyThrows + @SuppressWarnings("unchecked") + private static byte[] decode(Class type, T value) { + val buf = new ByteArrayOutputStream(); + ((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(type)).write(value, buf); + + return buf.toByteArray(); + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetHash") + public void getHash(byte[] encodedKey, String expectedInHex) { + val algorithm = Blake2B128Concat.getInstance(); + + val actual = algorithm.getHash(encodedKey); + val actualInHex = HexConverter.toHex(actual); + + assertEquals(expectedInHex, actualInHex); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/IdentityTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/IdentityTests.java new file mode 100644 index 00000000..3abd7aa5 --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/IdentityTests.java @@ -0,0 +1,52 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Random; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class IdentityTests { + private static Stream getTestCasesForGetHash() { + return Stream.of( + encode(Integer.class, -175), + encode(BlockHash.class, BlockHash.fromBytes(random(32))), + "TestString".getBytes(StandardCharsets.UTF_8), + random(new Random().nextInt(128) + 1)); + } + + @SneakyThrows + @SuppressWarnings("unchecked") + private static byte[] encode(Class type, T value) { + val buf = new ByteArrayOutputStream(); + ((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(type)).write(value, buf); + + return buf.toByteArray(); + } + + private static byte[] random(int length) { + val bytes = new byte[length]; + new Random().nextBytes(bytes); + + return bytes; + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetHash") + public void getHash(byte[] encodedKey) { + val algorithm = Identity.sizeOf(encodedKey.length); + + val actual = algorithm.getHash(encodedKey); + + assertEquals(encodedKey, actual); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/KeyHasherTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/KeyHasherTests.java new file mode 100644 index 00000000..8d706ddd --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/KeyHasherTests.java @@ -0,0 +1,167 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.common.utils.HexConverter; +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.readers.CompactIntegerReader; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import com.strategyobject.substrateclient.scale.writers.CompactIntegerWriter; +import com.strategyobject.substrateclient.types.Size; +import lombok.val; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class KeyHasherTests { + private static Stream getTestCasesForGetHash() { + return Stream.of( + Arguments.of( + 10, + ScaleReaderRegistry.getInstance().resolve(Integer.class), + ScaleWriterRegistry.getInstance().resolve(Integer.class), + TwoX64Concat.getInstance(), + "0xa6b274250e6753f00a000000" + ), + Arguments.of( + AccountId.fromBytes( + SS58Codec.decode( + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + .getAddress()), + ScaleReaderRegistry.getInstance().resolve(AccountId.class), + ScaleWriterRegistry.getInstance().resolve(AccountId.class), + Blake2B128Concat.getInstance(), + "0x4f9aea1afa791265fae359272badc1cf8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48" + ), + Arguments.of( + BlockHash.fromBytes(HexConverter.toBytes("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")), + ScaleReaderRegistry.getInstance().resolve(BlockHash.class), + ScaleWriterRegistry.getInstance().resolve(BlockHash.class), + Identity.sizeOf(Size.of32.getValue()), + "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ) + ); + } + + private static Stream getTestCasesForExtractKey() { + return Stream.of( + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0x5153cb1f00942ff401000000") + ), + ScaleReaderRegistry.getInstance().resolve(Integer.class), + ScaleWriterRegistry.getInstance().resolve(Integer.class), + TwoX64Concat.getInstance(), + 1, + 0 + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0x969e061847da7e84337ea78dc577cd1d05000000") + ), + ScaleReaderRegistry.getInstance().resolve(Integer.class), + ScaleWriterRegistry.getInstance().resolve(Integer.class), + Blake2B128Concat.getInstance(), + 5, + 0 + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0xa8") + ), + new CompactIntegerReader(), + new CompactIntegerWriter(), + Identity.sizeOf(1), + 42, + 0 + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0x") + ), + ScaleReaderRegistry.getInstance().resolve(Void.class), + ScaleWriterRegistry.getInstance().resolve(Void.class), + Identity.sizeOf(0), + null, + 0 + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0x4f9aea1afa791265fae359272badc1cf8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48" + + "a6b274250e6753f00a000000") + ), + ScaleReaderRegistry.getInstance().resolve(AccountId.class), + ScaleWriterRegistry.getInstance().resolve(AccountId.class), + Blake2B128Concat.getInstance(), + AccountId.fromBytes( + SS58Codec.decode( + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + .getAddress()), + HexConverter.toBytes("a6b274250e6753f00a000000").length + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0xabcdef98765432100123456789abcdefabcdef98765432100123456789abcdef")), + ScaleReaderRegistry.getInstance().resolve(BlockHash.class), + ScaleWriterRegistry.getInstance().resolve(BlockHash.class), + Identity.sizeOf(Size.of32.getValue()), + BlockHash.fromBytes(HexConverter.toBytes("0xabcdef98765432100123456789abcdefabcdef98765432100123456789abcdef")), + 0 + ), + Arguments.of( + new ByteArrayInputStream( + HexConverter.toBytes("0x518366b5b1bc7c99d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d")), + ScaleReaderRegistry.getInstance().resolve(BlockHash.class), + ScaleWriterRegistry.getInstance().resolve(BlockHash.class), + TwoX64Concat.getInstance(), + AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress()), + 0 + ) + ); + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetHash") + public void getHash(T key, + ScaleReader reader, + ScaleWriter writer, + KeyHashingAlgorithm algorithm, + String expectedInHex) throws IOException { + val hasher = KeyHasher.with(writer, reader, algorithm); + val actual = hasher.getHash(key); + + val expected = HexConverter.toBytes(expectedInHex); + assertArrayEquals(expected, actual); + } + + @ParameterizedTest + @MethodSource("getTestCasesForExtractKey") + public void extractKey(InputStream stream, + ScaleReader reader, + ScaleWriter writer, + KeyHashingAlgorithm algorithm, + T expected, + int expectedAvailable) throws IOException { + val hasher = KeyHasher.with(writer, reader, algorithm); + val actual = hasher.extractKey(stream); + + assertEquals(expected, actual); + + val available = stream.available(); + assertEquals(expectedAvailable, available); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImplTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImplTests.java new file mode 100644 index 00000000..d4ee338f --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageDoubleMapImplTests.java @@ -0,0 +1,61 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.RpcImpl; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import com.strategyobject.substrateclient.tests.containers.SubstrateVersion; +import com.strategyobject.substrateclient.tests.containers.TestSubstrateContainer; +import com.strategyobject.substrateclient.transport.ws.WsProvider; +import lombok.val; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertNull; + +@Testcontainers +public class StorageDoubleMapImplTests { + @Container + static final TestSubstrateContainer substrate = new TestSubstrateContainer(SubstrateVersion.V3_0_0); + private static final int CONNECTION_TIMEOUT = 1000; + + @Test + @SuppressWarnings("unchecked") + public void societyVotes() throws Exception { + val wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build(); + wsProvider.connect().get(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = StorageDoubleMapImpl.with( + rpc, + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(Void.class), + StorageKeyProvider.with("Society", "Votes") + .with(KeyHasher.with((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(AccountId.class), + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(AccountId.class), + TwoX64Concat.getInstance()), + KeyHasher.with((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(AccountId.class), + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(AccountId.class), + TwoX64Concat.getInstance()))); + val alice = AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress()); + val bob = AccountId.fromBytes( + SS58Codec.decode( + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + .getAddress()); + + val actual = storage.get(alice, bob).get(); + + assertNull(actual); + } + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageKeyProviderTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageKeyProviderTests.java new file mode 100644 index 00000000..af09bae5 --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageKeyProviderTests.java @@ -0,0 +1,253 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.common.utils.HexConverter; +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.rpc.types.StorageKey; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import lombok.NonNull; +import lombok.val; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StorageKeyProviderTests { + @SuppressWarnings("unchecked") + private static Stream getTestCasesForGetBySingleKey() { + return Stream.of( + Arguments.of("Balances", + "FreeBalance", + KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + Blake2B128Concat.getInstance() + ), + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"), + "0x" + + "c2261276cc9d1f8598ea4b6a74b15c2f" + + "6482b9ade7bc6657aaca787ba1add3b4" + + "de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d"), + + Arguments.of("System", + "BlockHash", + KeyHasher.with( + (ScaleWriter) ScaleWriterRegistry.getInstance().resolve(Integer.class), + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(Integer.class), + TwoX64Concat.getInstance() + ), + 2, + "0x" + + "26aa394eea5630e07c48ae0c9558cef7" + + "a44704b568d21667356a5a050c118746" + + "9eb2dcce60f37a2702000000") + ); + } + + @SuppressWarnings({"unchecked", "SameParameterValue"}) + private static ScaleReader resolveReader(Class clazz) { + return (ScaleReader) ScaleReaderRegistry.getInstance().resolve(clazz); + } + + @SuppressWarnings({"unchecked", "SameParameterValue"}) + private static ScaleWriter resolveWriter(Class clazz) { + return (ScaleWriter) ScaleWriterRegistry.getInstance().resolve(clazz); + } + + private static Stream getTestCasesForGetByDoubleKey() { + return Stream.of( + Arguments.of("Society", + "Votes", + KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + TwoX64Concat.getInstance() + ), + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"), + KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + TwoX64Concat.getInstance() + ), + newAccountId("5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"), + "0x" + + "426e15054d267946093858132eb537f1" + + "b4adc6a1ce4f7cc2e696ed0fd06bd01c" + + "518366b5b1bc7c99d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" + + "a647e755c30521d38eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48") + ); + } + + private static Stream getTestCasesForExtractKeys() { + val providers = Arrays.asList( + newFreeBalanceKeyProvider(), + newVotesKeyProvider() + ); + + val keys = Arrays.asList( + Collections.singletonList(newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY")), + Arrays.asList( + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"), + newAccountId("5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + ) + ); + + return IntStream + .range(0, providers.size()) + .mapToObj(i -> Arguments.of( + providers.get(i), + providers.get(i).get(keys.get(i).toArray()), + keys.get(i) + )); + } + + @NonNull + private static AccountId newAccountId(String encoded) { + return AccountId.fromBytes( + SS58Codec.decode( + encoded) + .getAddress()); + } + + private static Stream getTestCasesForExtractKeysUsingQueryKey() { + val providers = Arrays.asList( + newFreeBalanceKeyProvider(), + newVotesKeyProvider(), + newVotesKeyProvider() + ); + + val allKeys = Arrays.asList( + Collections.singletonList(newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY")), + Arrays.asList( + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"), + newAccountId("5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + ), + Arrays.asList( + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"), + newAccountId("5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty") + ) + ); + + val queryKeys = Arrays.asList( + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList( + newAccountId("5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + ) + ); + + return IntStream + .range(0, providers.size()) + .mapToObj(i -> Arguments.of( + providers.get(i), + providers.get(i).get(allKeys.get(i).toArray()), + providers.get(i).get(queryKeys.get(i).toArray()), + queryKeys.get(i).size(), + allKeys.get(i).stream().skip(queryKeys.get(i).size()).collect(Collectors.toList()) + )); + } + + private static StorageKeyProvider newVotesKeyProvider() { + return StorageKeyProvider.with("Society", "Votes") + .with(KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + TwoX64Concat.getInstance() + ), + KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + TwoX64Concat.getInstance() + ) + ); + } + + private static StorageKeyProvider newFreeBalanceKeyProvider() { + return StorageKeyProvider.with("Balances", "FreeBalance") + .with(KeyHasher.with( + resolveWriter(AccountId.class), + resolveReader(AccountId.class), + Blake2B128Concat.getInstance() + )); + } + + @ParameterizedTest + @CsvSource({ + "SomePallet,SomeStorage,0x832718a9c64cbad10dc0772e2c2d3d9c2e186e85ed8948269c15e1c78ccd4305", + "Sudo,Key,0x5c0d1176a568c1f92944340dbfed9e9c530ebca703c85910e7164cb7d1c9e47b"}) + public void getWithoutKey(String pallet, String storage, String expectedInHex) { + val provider = StorageKeyProvider.with(pallet, storage); + + val storageKey = provider.get(); + val actualInHex = HexConverter.toHex(storageKey.getData()); + + assertEquals(expectedInHex, actualInHex); + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetBySingleKey") + public void getBySingleKey(String pallet, + String storage, + KeyHasher keyHasher, + T key, + String expectedInHex) { + val provider = StorageKeyProvider.with(pallet, storage) + .with(keyHasher); + + val storageKey = provider.get(key); + val actualInHex = HexConverter.toHex(storageKey.getData()); + + assertEquals(expectedInHex, actualInHex); + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetByDoubleKey") + public void getByDoubleKey(String pallet, + String storage, + KeyHasher firstHasher, + F firstKey, + KeyHasher secondHasher, + S secondKey, + String expectedInHex) { + val provider = StorageKeyProvider.with(pallet, storage) + .with(firstHasher, secondHasher); + + val storageKey = provider.get(firstKey, secondKey); + val actualInHex = HexConverter.toHex(storageKey.getData()); + + assertEquals(expectedInHex, actualInHex); + } + + @ParameterizedTest + @MethodSource("getTestCasesForExtractKeys") + public void extractKeys(StorageKeyProvider provider, StorageKey fullKey, List expected) { + val actual = provider.extractKeys(fullKey); + + assertArrayEquals(expected.toArray(), actual.toArray()); + } + + @ParameterizedTest + @MethodSource("getTestCasesForExtractKeysUsingQueryKey") + public void extractKeysUsingQueryKey(StorageKeyProvider provider, + StorageKey fullKey, + StorageKey queryKey, + int countOfKeysInQuery, + List expected) { + val actual = provider.extractKeys(fullKey, queryKey, countOfKeysInQuery); + + assertArrayEquals(expected.toArray(), actual.toArray()); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageMapImplTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageMapImplTests.java new file mode 100644 index 00000000..b4b281ac --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageMapImplTests.java @@ -0,0 +1,50 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.rpc.RpcImpl; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import com.strategyobject.substrateclient.tests.containers.SubstrateVersion; +import com.strategyobject.substrateclient.tests.containers.TestSubstrateContainer; +import com.strategyobject.substrateclient.transport.ws.WsProvider; +import lombok.val; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.math.BigInteger; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +@Testcontainers +public class StorageMapImplTests { + @Container + static final TestSubstrateContainer substrate = new TestSubstrateContainer(SubstrateVersion.V3_0_0); + private static final int CONNECTION_TIMEOUT = 1000; + + @Test + @SuppressWarnings("unchecked") + public void systemBlockHash() throws Exception { + val wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build(); + wsProvider.connect().get(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = StorageMapImpl.with( + rpc, + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(BlockHash.class), + StorageKeyProvider.with("System", "BlockHash") + .with(KeyHasher.with((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(Integer.class), + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(Integer.class), + TwoX64Concat.getInstance()))); + + val actual = storage.get(0).get(); + + assertNotEquals(BigInteger.ZERO, new BigInteger(actual.getData())); + } + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageNMapImplTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageNMapImplTests.java new file mode 100644 index 00000000..d0d12482 --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageNMapImplTests.java @@ -0,0 +1,273 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.RpcImpl; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.rpc.types.BlockHash; +import com.strategyobject.substrateclient.scale.ScaleReader; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import com.strategyobject.substrateclient.tests.containers.SubstrateVersion; +import com.strategyobject.substrateclient.tests.containers.TestSubstrateContainer; +import com.strategyobject.substrateclient.transport.ws.WsProvider; +import com.strategyobject.substrateclient.types.tuples.Pair; +import lombok.NonNull; +import lombok.val; +import lombok.var; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; +import static org.testcontainers.shaded.org.hamcrest.number.OrderingComparison.greaterThan; + +@Testcontainers +public class StorageNMapImplTests { + private static final int CONNECTION_TIMEOUT = 1000; + private static final int WAIT_TIMEOUT = 10; + @Container + private final TestSubstrateContainer substrate = new TestSubstrateContainer(SubstrateVersion.V3_0_0); + + @SuppressWarnings("unchecked") + private static StorageNMapImpl newSystemBlockHashStorage(RpcImpl rpc) { + return StorageNMapImpl.with( + rpc, + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(BlockHash.class), + StorageKeyProvider.with("System", "BlockHash") + .with(KeyHasher.with((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(Integer.class), + (ScaleReader) ScaleReaderRegistry.getInstance().resolve(Integer.class), + TwoX64Concat.getInstance()))); + } + + @NonNull + private WsProvider getConnectedProvider() throws InterruptedException, ExecutionException, TimeoutException { + val wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build(); + wsProvider.connect().get(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + return wsProvider; + } + + @Test + public void keys() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = newSystemBlockHashStorage(rpc); + + val collection = storage.keys().get(); + + assertNotNull(collection); + assertEquals(1, collection.size()); + + val blockNumber = new AtomicReference(null); + collection.iterator().forEachRemaining(q -> q.consume((o -> blockNumber.set((Integer) o.get(0))))); + + assertEquals(0, blockNumber.get()); + + val blocks = collection.multi().execute().get(); + val list = new ArrayList<>(); + blocks.iterator().forEachRemaining(e -> e.consume((value, keys) -> list.add(value))); + + assertEquals(1, list.size()); + + val block = (BlockHash) list.stream().findFirst().orElseThrow(RuntimeException::new); + assertNotEquals(BigInteger.ZERO, new BigInteger(block.getData())); + } + } + + @Test + public void multiToDifferentStorages() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + val storageValue = StorageNMapImpl.with( + rpc, + ScaleReaderRegistry.getInstance().resolve(AccountId.class), + StorageKeyProvider.with("Sudo", "Key")); + val storageMap = newSystemBlockHashStorage(rpc); + + val getKey = storageValue.query(); + val getHash = storageMap.query(0); + + val multi = getKey.join(getHash); + val collection = multi.execute().get(); + + val expectedKey = AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress()); + val expectedBlock = storageMap.get(0).get(); + + val list = new ArrayList<>(2); + collection.iterator().forEachRemaining(e -> e.consume((value, keys) -> list.add(value))); + + assertEquals(expectedKey, list.get(0)); + assertEquals(expectedBlock, list.get(1)); + } + } + + @Test + public void entries() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = newSystemBlockHashStorage(rpc); + + val collection = storage.entries().get(); + + assertNotNull(collection); + + val blockNumber = new AtomicReference(null); + val blockHash = new AtomicReference(null); + collection.iterator().forEachRemaining(e -> e.consume((value, keys) -> { + blockHash.set(value); + blockNumber.set((Integer) keys.get(0)); + })); + + assertEquals(0, blockNumber.get()); + assertNotEquals(BigInteger.ZERO, new BigInteger(blockHash.get().getData())); + } + } + + @Test + public void multi() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = newSystemBlockHashStorage(rpc); + + val collection = storage.multi(Arg.of(0), Arg.of(1)).get(); + assertNotNull(collection); + + val list = new ArrayList>(); + collection.iterator().forEachRemaining(e -> e.consume((value, keys) -> list.add(Pair.of((Integer) keys.get(0), value)))); + + assertEquals(2, list.size()); + + assertEquals(0, list.get(0).getValue0()); + assertNotEquals(BigInteger.ZERO, new BigInteger(list.get(0).getValue1().getData())); + + assertEquals(1, list.get(1).getValue0()); + assertNull(list.get(1).getValue1()); + } + } + + @Test + public void keysPaged() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + waitForNewBlocks(rpc); + + val storage = newSystemBlockHashStorage(rpc); + + int pageSize = 2; + val pages = storage.keysPaged(pageSize); + + assertNotNull(pages); + + var pageCount = 0; + AtomicInteger total = new AtomicInteger(); + while (pages.moveNext().join()) { + pages.current().iterator().forEachRemaining(queryableKey -> total.getAndIncrement()); + pageCount++; + assertEquals(pageCount, pages.number()); + } + + assertTrue(pageCount > 1); + assertEquals(pageCount, Math.ceil((double) total.get() / pageSize)); + } + } + + @Test + public void entriesPaged() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + waitForNewBlocks(rpc); + + val storage = newSystemBlockHashStorage(rpc); + + int pageSize = 2; + val pages = storage.entriesPaged(pageSize); + + assertNotNull(pages); + + var pageCount = 0; + val pairs = new ArrayList>(); + while (pages.moveNext().join()) { + pages.current().iterator().forEachRemaining(e -> e.consume((value, keys) -> { + val key = (Integer) keys.get(0); + assertNotEquals(BigInteger.ZERO, new BigInteger(value.getData())); + + pairs.add(Pair.of(key, value)); + })); + pageCount++; + assertEquals(pageCount, pages.number()); + } + + assertEquals(pairs.size(), pairs.stream().map(Pair::getValue0).collect(Collectors.toSet()).size()); + + assertTrue(pageCount > 1); + assertEquals(pageCount, Math.ceil((double) pairs.size() / pageSize)); + } + } + + @Test + public void subscribe() throws Exception { + val wsProvider = getConnectedProvider(); + try (val rpc = new RpcImpl(wsProvider)) { + val blockNumber = 2; + val storage = newSystemBlockHashStorage(rpc); + val blockHash = new AtomicReference(); + val value = new AtomicReference(); + val argument = new AtomicInteger(); + storage.subscribe((exception, block, v, keys) -> { + if (exception == null) { + blockHash.set(block); + value.set(v); + argument.set((Integer) keys.get(0)); + } + }, Arg.of(blockNumber)); + + waitForNewBlocks(rpc); + + val expectedValue = rpc.getChain().getBlockHash(blockNumber).join(); + val history = storage.history(blockNumber).join(); + val changedAt = history.stream() + .findFirst() + .orElseThrow(RuntimeException::new) + .getValue0(); + + assertEquals(changedAt, blockHash.get()); + assertEquals(expectedValue, value.get()); + assertEquals(expectedValue, value.get()); + assertEquals(blockNumber, argument.get()); + } + } + + private void waitForNewBlocks(RpcImpl rpc) throws InterruptedException, ExecutionException, TimeoutException { + val blockCount = new AtomicInteger(0); + val unsubscribeFunc = rpc + .getChain() + .subscribeNewHeads((e, h) -> { + if (e == null) { + blockCount.incrementAndGet(); + } + }) + .get(WAIT_TIMEOUT, TimeUnit.SECONDS); + + await() + .atMost(WAIT_TIMEOUT * 3, TimeUnit.SECONDS) + .untilAtomic(blockCount, greaterThan(3)); + + unsubscribeFunc.get().join(); + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageValueImplTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageValueImplTests.java new file mode 100644 index 00000000..a1cbd4f2 --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/StorageValueImplTests.java @@ -0,0 +1,73 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.RpcImpl; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.scale.registries.ScaleReaderRegistry; +import com.strategyobject.substrateclient.tests.containers.SubstrateVersion; +import com.strategyobject.substrateclient.tests.containers.TestSubstrateContainer; +import com.strategyobject.substrateclient.transport.ws.WsProvider; +import lombok.val; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Testcontainers +public class StorageValueImplTests { + @Container + static final TestSubstrateContainer substrate = new TestSubstrateContainer(SubstrateVersion.V3_0_0); + private static final int CONNECTION_TIMEOUT = 1000; + + @Test + public void sudoKey() throws Exception { + val expected = AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress()); + + val wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build(); + wsProvider.connect().get(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + try (val rpc = new RpcImpl(wsProvider)) { + val storage = StorageValueImpl.with( + rpc, + ScaleReaderRegistry.getInstance().resolve(AccountId.class), + StorageKeyProvider.with("Sudo", "Key")); + + val actual = storage.get().get(); + + assertEquals(expected, actual); + } + } + + @Test + public void sudoKeyAtGenesis() throws Exception { + val expected = AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress()); + + val wsProvider = WsProvider.builder() + .setEndpoint(substrate.getWsAddress()) + .disableAutoConnect() + .build(); + wsProvider.connect().get(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + try (val rpc = new RpcImpl(wsProvider)) { + val blockHash = rpc.getChain().getBlockHash(0).get(); + val storage = StorageValueImpl.with( + rpc, + ScaleReaderRegistry.getInstance().resolve(AccountId.class), + StorageKeyProvider.with("Sudo", "Key")); + + val actual = storage.at(blockHash).get(); + + assertEquals(expected, actual); + } + } +} diff --git a/storage/src/test/java/com/strategyobject/substrateclient/storage/TwoX64ConcatTests.java b/storage/src/test/java/com/strategyobject/substrateclient/storage/TwoX64ConcatTests.java new file mode 100644 index 00000000..4e6b62fa --- /dev/null +++ b/storage/src/test/java/com/strategyobject/substrateclient/storage/TwoX64ConcatTests.java @@ -0,0 +1,73 @@ +package com.strategyobject.substrateclient.storage; + +import com.strategyobject.substrateclient.common.utils.HexConverter; +import com.strategyobject.substrateclient.crypto.ss58.SS58Codec; +import com.strategyobject.substrateclient.rpc.types.AccountId; +import com.strategyobject.substrateclient.scale.ScaleWriter; +import com.strategyobject.substrateclient.scale.registries.ScaleWriterRegistry; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TwoX64ConcatTests { + private static Stream getTestCasesForGetHash() { + return Stream.of( + Arguments.of( + decode(Integer.class, 10), + "0xa6b274250e6753f00a000000" + ), + Arguments.of( + "TestString".getBytes(StandardCharsets.UTF_8), + "0x3cd20663ed09cfc954657374537472696e67" + ), + Arguments.of( + "qwerty".getBytes(StandardCharsets.UTF_8), + "0x7e918fd4671efe8a717765727479" + ), + Arguments.of( + "abcd".getBytes(StandardCharsets.UTF_8), + "0xcc925dd2b02703de61626364" + ), + Arguments.of( + "Hello, World!".getBytes(StandardCharsets.UTF_8), + "0x7fe40f08f8ac9ac448656c6c6f2c20576f726c6421" + ), + Arguments.of( + decode(AccountId.class, + AccountId.fromBytes( + SS58Codec.decode( + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") + .getAddress())), + "0x518366b5b1bc7c99d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" + ) + ); + } + + @SneakyThrows + @SuppressWarnings("unchecked") + private static byte[] decode(Class type, T value) { + val buf = new ByteArrayOutputStream(); + ((ScaleWriter) ScaleWriterRegistry.getInstance().resolve(type)).write(value, buf); + + return buf.toByteArray(); + } + + @ParameterizedTest + @MethodSource("getTestCasesForGetHash") + public void getHash(byte[] encodedKey, String expectedInHex) { + val algorithm = new TwoX64Concat(); + + val actual = algorithm.getHash(encodedKey); + val actualInHex = HexConverter.toHex(actual); + + assertEquals(expectedInHex, actualInHex); + } +}