Generate storage key prefix methods (#381)

examples/README.md

@@ -12,6 +12,7 @@ polkadart:
 Feel free to change to whatever network you want. After that you can generate the classes by running:
 `dart run polkadart_cli:generate -v`
 
-Finally to run this example you can do:
-`dart run bin/demo.dart`
+Finally to run an example you can do:
+* `dart run bin/extrinsic_demo.dart`
+* `dart run bin/get_account_infos_demo.dart`
 

examples/analysis_options.yaml

@@ -13,18 +13,8 @@
 
 include: package:lints/recommended.yaml
 
-# Uncomment the following section to specify additional rules.
-
-# linter:
-#   rules:
-#     - camel_case_types
-
-# analyzer:
-#   exclude:
-#     - path/to/excluded/files/**
-
-# For more information about the core and recommended set of lints, see
-# https://dart.dev/go/core-lints
-
-# For additional information about configuring this file, see
-# https://dart.dev/guides/language/analysis-options
+linter:
+  rules:
+    # Generated files don't satisfy this.
+    avoid_renaming_method_parameters: false
+    no_leading_underscores_for_local_identifiers: false

examples/bin/demo.dart → examples/bin/extrinsic_demo.dart

@@ -1,10 +1,11 @@
 import 'package:convert/convert.dart';
-import '../lib/generated/polkadot/polkadot.dart';
-import '../lib/generated/polkadot/types/sp_runtime/multiaddress/multi_address.dart';
 import 'package:polkadart/polkadart.dart'
     show AuthorApi, Extrinsic, Provider, SigningPayload, StateApi;
 import 'package:polkadart_keyring/polkadart_keyring.dart';
 
+import 'package:polkadart_example/generated/polkadot/polkadot.dart';
+import 'package:polkadart_example/generated/polkadot/types/sp_runtime/multiaddress/multi_address.dart';
+
 Future<void> main(List<String> arguments) async {
   final provider = Provider.fromUri(Uri.parse('wss://rpc.polkadot.io'));
   final api = Polkadot(provider);

examples/bin/get_account_infos_demo.dart

@@ -0,0 +1,28 @@
+import 'package:convert/convert.dart';
+import 'package:polkadart/scale_codec.dart';
+import 'package:polkadart_example/generated/polkadot/types/sp_core/crypto/account_id32.dart';
+import 'package:polkadart/polkadart.dart' show Provider;
+
+import 'package:polkadart_example/generated/polkadot/polkadot.dart';
+
+Future<void> main(List<String> arguments) async {
+  final provider = Provider.fromUri(Uri.parse('wss://rpc.polkadot.io'));
+  final polkadot = Polkadot(provider);
+
+  final accountMapPrefix = polkadot.query.system.accountMapPrefix();
+  final keys = await polkadot.rpc.state.getKeysPaged(key: accountMapPrefix, count: 10);
+
+  print("First 10 account storage keys: ${keys.map((key) => '0x${hex.encode(key)}')}");
+
+  // Decoding of the keys has to be done manually for now, see how substrate storage keys are defined:
+  // https://www.shawntabrizi.com/blog/substrate/transparent-keys-in-substrate/
+  final accountIds = keys.map((key) => const AccountId32Codec().decode(ByteInput(key.sublist(32))));
+  print("First 10 account pubKeys: ${accountIds.map((account) => '0x${hex.encode(account)}')}");
+
+  // Get `AccountInfo`s of those keys
+  final accountInfos = await Future.wait(accountIds.map((account) => polkadot.query.system.account(account)));
+
+  for (final accountInfo in accountInfos) {
+    print('AccountInfo: ${accountInfo.toJson()}');
+  }
+}

examples/lib/README.md

@@ -0,0 +1 @@
+Directory for the generated files.

examples/pubspec.yaml

@@ -17,6 +17,9 @@ dependencies:
   substrate_bip39: ^0.2.0
   substrate_metadata: ^1.1.0
 
+  # Generated files depend on quiver
+  quiver: ^3.2.1
+
 dev_dependencies:
   lints: ^2.0.0
   test: ^1.21.0

examples/pubspec_overrides.yaml

@@ -0,0 +1,16 @@
+# melos_managed_dependency_overrides: polkadart,polkadart_cli,polkadart_keyring,polkadart_scale_codec,ss58,substrate_bip39,substrate_metadata
+dependency_overrides:
+  polkadart:
+    path: ../packages/polkadart
+  polkadart_cli:
+    path: ../packages/polkadart_cli
+  polkadart_keyring:
+    path: ../packages/polkadart_keyring
+  polkadart_scale_codec:
+    path: ../packages/polkadart_scale_codec
+  ss58:
+    path: ../packages/ss58
+  substrate_bip39:
+    path: ../packages/substrate_bip39
+  substrate_metadata:
+    path: ../packages/substrate_metadata

melos.yaml

@@ -7,6 +7,7 @@ command:
 
 packages:
   - packages/**
+  - examples
 
 scripts:
   test:

packages/polkadart/lib/substrate/storage.dart

@@ -115,15 +115,25 @@ class StorageMap<K, V> {
 
   Uint8List hashedKeyFor(K key) {
     final Uint8List hash = Uint8List(32 + hasher.size(key));
+    _hashPrefixTo(output: hash);
+    hasher.hashTo(
+        key: key, output: hash.buffer.asUint8List(hash.offsetInBytes + 32));
+    return hash;
+  }
+
+  Uint8List mapPrefix() {
+    final Uint8List hash = Uint8List(32);
+    _hashPrefixTo(output: hash);
+    return hash;
+  }
+
+  void _hashPrefixTo({required Uint8List output}) {
     Hasher.twoxx128.hashTo(
         data: Uint8List.fromList(utf8.encode(prefix)),
-        output: hash.buffer.asUint8List(hash.offsetInBytes, 16));
+        output: output.buffer.asUint8List(output.offsetInBytes, 16));
     Hasher.twoxx128.hashTo(
         data: Uint8List.fromList(utf8.encode(storage)),
-        output: hash.buffer.asUint8List(hash.offsetInBytes + 16, 16));
-    hasher.hashTo(
-        key: key, output: hash.buffer.asUint8List(hash.offsetInBytes + 32));
-    return hash;
+        output: output.buffer.asUint8List(output.offsetInBytes + 16, 16));
   }
 
   V decodeValue(Uint8List buffer) {
@@ -147,23 +157,32 @@ class StorageDoubleMap<K1, K2, V> {
   });
 
   Uint8List hashedKeyFor(K1 key1, K2 key2) {
-    final Uint8List hash =
-        Uint8List(32 + hasher1.size(key1) + hasher2.size(key2));
-    Hasher.twoxx128.hashTo(
-      data: Uint8List.fromList(utf8.encode(prefix)),
-      output: hash.buffer.asUint8List(hash.offsetInBytes, 16),
-    );
-    Hasher.twoxx128.hashTo(
-      data: Uint8List.fromList(utf8.encode(storage)),
-      output: hash.buffer.asUint8List(hash.offsetInBytes + 16, 16),
-    );
-    int cursor = hash.offsetInBytes + 32;
-    hasher1.hashTo(key: key1, output: hash.buffer.asUint8List(cursor));
-    cursor += hasher1.size(key1);
+    final Uint8List hash = Uint8List(32 + hasher1.size(key1) + hasher2.size(key2));
+    _hashPrefixTo(key1, output: hash);
+
+    final cursor = hash.offsetInBytes + 32 + hasher1.size(key1);
     hasher2.hashTo(key: key2, output: hash.buffer.asUint8List(cursor));
     return hash;
   }
 
+  Uint8List mapPrefix(K1 key1) {
+    final Uint8List hash = Uint8List(32 + hasher1.size(key1));
+    _hashPrefixTo(key1, output: hash);
+    return hash;
+  }
+
+  void _hashPrefixTo(K1 key1, {required Uint8List output}) {
+    Hasher.twoxx128.hashTo(
+        data: Uint8List.fromList(utf8.encode(prefix)),
+        output: output.buffer.asUint8List(output.offsetInBytes, 16));
+    Hasher.twoxx128.hashTo(
+        data: Uint8List.fromList(utf8.encode(storage)),
+        output: output.buffer.asUint8List(output.offsetInBytes + 16, 16));
+
+    final int cursor = output.offsetInBytes + 32;
+    hasher1.hashTo(key: key1, output: output.buffer.asUint8List(cursor));
+  }
+
   V decodeValue(Uint8List buffer) {
     return valueCodec.decode(ByteInput(buffer));
   }

packages/polkadart_cli/lib/src/generator/pallet.dart

@@ -448,13 +448,9 @@ Class createPalletQueries(
                       ..name = 'key${storage.hashers.indexOf(hasher) + 1}')))
                 ..body = Block((b) => b
                   // final hashedKey = _storageName.hashedKeyFor(key1);
-                  ..statements.add(declareFinal('hashedKey')
-                      .assign(refer('_$storageName')
-                          .property(storage.hashers.isEmpty
-                              ? 'hashedKey'
-                              : 'hashedKeyFor')
-                          .call(storage.hashers.map((hasher) => refer(
-                              'key${storage.hashers.indexOf(hasher) + 1}'))))
+                  ..statements
+                      .add(declareFinal('hashedKey')
+                      .assignHashedKey(storageName, storage)
                       .statement)
                   // final bytes = await api.queryStorage([hashedKey]);
                   ..statements.add(declareFinal('bytes')
@@ -481,21 +477,38 @@ Class createPalletQueries(
         ..methods.addAll(generator.storages.map((storage) => Method((builder) {
           final storageName = ReCase(storage.name).camelCase;
           builder
-            ..name = sanitize(storageKeyMethodName(storage), recase: false)
+            ..name = sanitize(storage.keyMethodName(), recase: false)
             ..docs.addAll(sanitizeDocs(['Returns the storage key for `$storageName`.']))
             ..returns = refs.uint8List
             ..requiredParameters
                 .addAll(storage.hashers.map((hasher) => Parameter((b) => b
               ..type = hasher.codec.primitive(dirname)
               ..name = 'key${storage.hashers.indexOf(hasher) + 1}')))
             ..body = Block((b) => b
-              ..statements.add(declareFinal('hashedKey')
-                  .assign(refer('_$storageName')
-                  .property(storage.hashers.isEmpty
-                  ? 'hashedKey'
-                  : 'hashedKeyFor')
-                  .call(storage.hashers.map((hasher) => refer(
-                  'key${storage.hashers.indexOf(hasher) + 1}'))))
+              ..statements
+                  .add(declareFinal('hashedKey')
+                  .assignHashedKey(storageName, storage)
+                  .statement)
+              ..statements
+                  .add(Code('  return hashedKey;')));
+        })))
+        ..methods.addAll(generator.storages
+            // We don't support maps with depth > 2 yet.
+            .where((storage) => storage.hashers.isNotEmpty && storage.hashers.length < 3)
+            .map((storage) => Method((builder) {
+          final storageName = ReCase(storage.name).camelCase;
+          builder
+            ..name = sanitize(storage.mapPrefixMethodName(), recase: false)
+            ..docs.addAll(sanitizeDocs(['Returns the storage map key prefix for `$storageName`.']))
+            ..returns = refs.uint8List
+            ..requiredParameters
+                .addAll(storage.hashers.getRange(0, storage.hashers.length - 1).map((hasher) => Parameter((b) => b
+              ..type = hasher.codec.primitive(dirname)
+              ..name = 'key${storage.hashers.indexOf(hasher) + 1}')))
+            ..body = Block((b) => b
+              ..statements
+                  .add(declareFinal('hashedKey')
+                  .assignMapPrefix(storageName, storage)
                   .statement)
               ..statements
                   .add(Code('  return hashedKey;')));
@@ -583,7 +596,50 @@ Class createPalletConstants(
               .code)));
     });
 
-/// Name of the generated method returning the key of a storage.
-String storageKeyMethodName(Storage storage) {
-  return '${storage.name}Key';
+extension MethodNameExtension on Storage {
+
+  /// Name of the generated method returning the key of a storage.
+  String keyMethodName() {
+    return '${name}Key';
+  }
+
+  /// Name of the generated method returning the key prefix of a storage.
+  String mapPrefixMethodName() {
+    return '${name}MapPrefix';
+  }
 }
+
+extension AssignHashedKeyExtension on Expression {
+  Expression assignHashedKey(String storageName, Storage storage) {
+    return assign(refer('_$storageName')
+        .property(storage.hashers.isEmpty
+        ? 'hashedKey'
+        : 'hashedKeyFor')
+        .call(storage.hashers.map((hasher) => refer(
+        'key${storage.hashers.indexOf(hasher) + 1}'))));
+  }
+
+  Expression assignMapPrefix(String storageName, Storage storage) {
+    if (storage.hashers.isEmpty) {
+      throw Exception(
+        'Bad code generation path; can\'t create keyPrefix method without hashers.',
+      );
+    }
+
+    if (storage.hashers.length > 2) {
+      throw Exception(
+        'Bad code generation path; keyPrefix method for maps with depth > 2 are not supported yet.',
+      );
+    }
+
+    return assign(refer('_$storageName')
+        .property('mapPrefix')
+        .call(storage.hashers
+        // Checked above that hasher is not empty.
+        .getRange(0, storage.hashers.length - 1)
+        .map((hasher) => refer(
+        'key${storage.hashers.indexOf(hasher) + 1}'))));
+  }
+}
+
+