diff --git a/assets/img/4-Substrate/dev-4-1-comms.svg b/assets/img/4-Substrate/dev-4-1-comms.svg index e6538fddc..555b35631 100644 --- a/assets/img/4-Substrate/dev-4-1-comms.svg +++ b/assets/img/4-Substrate/dev-4-1-comms.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-contracts.svg b/assets/img/4-Substrate/dev-4-1-contracts.svg new file mode 100644 index 000000000..9c23702e4 --- /dev/null +++ b/assets/img/4-Substrate/dev-4-1-contracts.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-forkless-1.svg b/assets/img/4-Substrate/dev-4-1-forkless-1.svg index d30448648..4fe7a4da6 100644 --- a/assets/img/4-Substrate/dev-4-1-forkless-1.svg +++ b/assets/img/4-Substrate/dev-4-1-forkless-1.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-forkless-2.svg b/assets/img/4-Substrate/dev-4-1-forkless-2.svg index 40b56f8e4..f796fcd76 100644 --- a/assets/img/4-Substrate/dev-4-1-forkless-2.svg +++ b/assets/img/4-Substrate/dev-4-1-forkless-2.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-ink.jpeg b/assets/img/4-Substrate/dev-4-1-ink.jpeg new file mode 100644 index 000000000..cdd24efce Binary files /dev/null and b/assets/img/4-Substrate/dev-4-1-ink.jpeg differ diff --git a/assets/img/4-Substrate/dev-4-1-state-opaqueu.svg b/assets/img/4-Substrate/dev-4-1-state-opaqueu.svg index dad6bf5c3..8129e0aca 100644 --- a/assets/img/4-Substrate/dev-4-1-state-opaqueu.svg +++ b/assets/img/4-Substrate/dev-4-1-state-opaqueu.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-state.svg b/assets/img/4-Substrate/dev-4-1-state.svg index c51327b34..3e9e6cad0 100644 --- a/assets/img/4-Substrate/dev-4-1-state.svg +++ b/assets/img/4-Substrate/dev-4-1-state.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-substrate.svg b/assets/img/4-Substrate/dev-4-1-substrate.svg index 510482f28..0598f4e22 100644 --- a/assets/img/4-Substrate/dev-4-1-substrate.svg +++ b/assets/img/4-Substrate/dev-4-1-substrate.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-1-wasm-langs.svg b/assets/img/4-Substrate/dev-4-1-wasm-langs.svg index 24110d71e..339d2456f 100644 --- a/assets/img/4-Substrate/dev-4-1-wasm-langs.svg +++ b/assets/img/4-Substrate/dev-4-1-wasm-langs.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-2-external.svg b/assets/img/4-Substrate/dev-4-2-external.svg index 2f6907d2a..8d7319148 100644 --- a/assets/img/4-Substrate/dev-4-2-external.svg +++ b/assets/img/4-Substrate/dev-4-2-external.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-block-opaqueu.svg b/assets/img/4-Substrate/dev-4-3-block-opaqueu.svg new file mode 100644 index 000000000..3a496b46e --- /dev/null +++ b/assets/img/4-Substrate/dev-4-3-block-opaqueu.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-child.svg b/assets/img/4-Substrate/dev-4-3-child.svg index b899fb127..313582f52 100644 --- a/assets/img/4-Substrate/dev-4-3-child.svg +++ b/assets/img/4-Substrate/dev-4-3-child.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-full-comm.svg b/assets/img/4-Substrate/dev-4-3-full-comm.svg index 6ab4b4b68..b4469631a 100644 --- a/assets/img/4-Substrate/dev-4-3-full-comm.svg +++ b/assets/img/4-Substrate/dev-4-3-full-comm.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-full.svg b/assets/img/4-Substrate/dev-4-3-full.svg index d492cc1ed..8a217a788 100644 --- a/assets/img/4-Substrate/dev-4-3-full.svg +++ b/assets/img/4-Substrate/dev-4-3-full.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-import.svg b/assets/img/4-Substrate/dev-4-3-import.svg index e94442918..44f979646 100644 --- a/assets/img/4-Substrate/dev-4-3-import.svg +++ b/assets/img/4-Substrate/dev-4-3-import.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-native-1.svg b/assets/img/4-Substrate/dev-4-3-native-1.svg new file mode 100644 index 000000000..205771d9d --- /dev/null +++ b/assets/img/4-Substrate/dev-4-3-native-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-native.svg b/assets/img/4-Substrate/dev-4-3-native.svg index 5a7f3c24e..f7da8ea17 100644 --- a/assets/img/4-Substrate/dev-4-3-native.svg +++ b/assets/img/4-Substrate/dev-4-3-native.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-pruning-1.svg b/assets/img/4-Substrate/dev-4-3-pruning-1.svg index 313582f52..493178fb1 100644 --- a/assets/img/4-Substrate/dev-4-3-pruning-1.svg +++ b/assets/img/4-Substrate/dev-4-3-pruning-1.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-pruning-2.svg b/assets/img/4-Substrate/dev-4-3-pruning-2.svg index 9a8a6dc36..b511fef8f 100644 --- a/assets/img/4-Substrate/dev-4-3-pruning-2.svg +++ b/assets/img/4-Substrate/dev-4-3-pruning-2.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-pruning-3.svg b/assets/img/4-Substrate/dev-4-3-pruning-3.svg index 7c2cc48ab..54e2b0105 100644 --- a/assets/img/4-Substrate/dev-4-3-pruning-3.svg +++ b/assets/img/4-Substrate/dev-4-3-pruning-3.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-pruning-4.svg b/assets/img/4-Substrate/dev-4-3-pruning-4.svg index 880a8c35a..cd1c26658 100644 --- a/assets/img/4-Substrate/dev-4-3-pruning-4.svg +++ b/assets/img/4-Substrate/dev-4-3-pruning-4.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-4-3-upgrade.svg b/assets/img/4-Substrate/dev-4-3-upgrade.svg index 766203827..18e34e3c7 100644 --- a/assets/img/4-Substrate/dev-4-3-upgrade.svg +++ b/assets/img/4-Substrate/dev-4-3-upgrade.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-kv-backend.svg b/assets/img/4-Substrate/dev-kv-backend.svg index 30464c27f..6d41b2c76 100644 --- a/assets/img/4-Substrate/dev-kv-backend.svg +++ b/assets/img/4-Substrate/dev-kv-backend.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-1.svg b/assets/img/4-Substrate/dev-overlay-1.svg index f3c6fa208..28bd6910c 100644 --- a/assets/img/4-Substrate/dev-overlay-1.svg +++ b/assets/img/4-Substrate/dev-overlay-1.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-2.svg b/assets/img/4-Substrate/dev-overlay-2.svg index e7493dbbb..c77232090 100644 --- a/assets/img/4-Substrate/dev-overlay-2.svg +++ b/assets/img/4-Substrate/dev-overlay-2.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-3.svg b/assets/img/4-Substrate/dev-overlay-3.svg index bceafaaf7..467ae42f4 100644 --- a/assets/img/4-Substrate/dev-overlay-3.svg +++ b/assets/img/4-Substrate/dev-overlay-3.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-4.svg b/assets/img/4-Substrate/dev-overlay-4.svg index 3dbd668d8..a5b9928bc 100644 --- a/assets/img/4-Substrate/dev-overlay-4.svg +++ b/assets/img/4-Substrate/dev-overlay-4.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-5.svg b/assets/img/4-Substrate/dev-overlay-5.svg index a735bf91c..ea08770b7 100644 --- a/assets/img/4-Substrate/dev-overlay-5.svg +++ b/assets/img/4-Substrate/dev-overlay-5.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-nested-1.svg b/assets/img/4-Substrate/dev-overlay-nested-1.svg index 723636693..778c02b3b 100644 --- a/assets/img/4-Substrate/dev-overlay-nested-1.svg +++ b/assets/img/4-Substrate/dev-overlay-nested-1.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-nested.svg b/assets/img/4-Substrate/dev-overlay-nested.svg index 0188faf6a..df6d1bb48 100644 --- a/assets/img/4-Substrate/dev-overlay-nested.svg +++ b/assets/img/4-Substrate/dev-overlay-nested.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay-root.svg b/assets/img/4-Substrate/dev-overlay-root.svg index 56e16cf9c..0572d6b47 100644 --- a/assets/img/4-Substrate/dev-overlay-root.svg +++ b/assets/img/4-Substrate/dev-overlay-root.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-overlay.svg b/assets/img/4-Substrate/dev-overlay.svg index 94372ab26..7fbb39bbe 100644 --- a/assets/img/4-Substrate/dev-overlay.svg +++ b/assets/img/4-Substrate/dev-overlay.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-trie-backend-unbalanced.svg b/assets/img/4-Substrate/dev-trie-backend-unbalanced.svg index 40d799797..e18aca16c 100644 --- a/assets/img/4-Substrate/dev-trie-backend-unbalanced.svg +++ b/assets/img/4-Substrate/dev-trie-backend-unbalanced.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-trie-backend-walk-0.svg b/assets/img/4-Substrate/dev-trie-backend-walk-0.svg index da318fe63..b489c7cc0 100644 --- a/assets/img/4-Substrate/dev-trie-backend-walk-0.svg +++ b/assets/img/4-Substrate/dev-trie-backend-walk-0.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-trie-backend-walk-1.svg b/assets/img/4-Substrate/dev-trie-backend-walk-1.svg index cee061a01..5d82b14e6 100644 --- a/assets/img/4-Substrate/dev-trie-backend-walk-1.svg +++ b/assets/img/4-Substrate/dev-trie-backend-walk-1.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-trie-backend-walk-2.svg b/assets/img/4-Substrate/dev-trie-backend-walk-2.svg index 8cf3d38c5..ef74d29e1 100644 --- a/assets/img/4-Substrate/dev-trie-backend-walk-2.svg +++ b/assets/img/4-Substrate/dev-trie-backend-walk-2.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/4-Substrate/dev-trie-backend-walk-full.svg b/assets/img/4-Substrate/dev-trie-backend-walk-full.svg index 5e7149b07..fe64be71b 100644 --- a/assets/img/4-Substrate/dev-trie-backend-walk-full.svg +++ b/assets/img/4-Substrate/dev-trie-backend-walk-full.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/syllabus/4-Substrate/4.1-Intro-to-Substrate_Slides.md b/syllabus/4-Substrate/4.1-Intro-to-Substrate_Slides.md index eb3ced548..c47122296 100644 --- a/syllabus/4-Substrate/4.1-Intro-to-Substrate_Slides.md +++ b/syllabus/4-Substrate/4.1-Intro-to-Substrate_Slides.md @@ -18,7 +18,7 @@ Substrate is a **Rust framework** for **building blockchains** in a modular and - โ›“๏ธ Future is multi-chain. - + ---v @@ -61,8 +61,7 @@ Outcomes of this: - Rust as a language - Upgradeability through a WASM meta-protocol. -- Generic. - APIs >> Opinions. +- Generic (build _APIs_, not _fixed implementations_). ---v @@ -157,13 +156,6 @@ Outcomes of this: ---v -### Negative Consequences of _WASM_ Runtime ๐Ÿฅฒ - -- ๐Ÿ˜ฉ Constrained resources -- ๐Ÿค” Client diversification != state-transition diversification - ----v - ### ๐Ÿค– Deterministic Execution - The need for determinism in a blockchain runtime is _absolute_. @@ -196,13 +188,20 @@ Outcomes of this: ### ๐Ÿ˜Ž Forkless Upgrade: - + + +---v + +### Negative Consequences of _WASM_ Runtime ๐Ÿฅฒ + +- ๐Ÿ˜ฉ Constrained resources +- ๐Ÿค” Client diversification != state-transition diversification ---v ### What is WASM Anyways? - + ---v @@ -228,29 +227,6 @@ Outcomes of this: ---v -### SMOLDOT - - - - - -A marvel of universe ๐Ÿคฏ. - -- (light) Substrate\* node compiled to WASM, by the browser. -- Itself executing another WASM blob, the aforementioned runtime. - - - - - - - - - - - ----v - ### How to Write a WASM Runtime? - Any language that can compile to WASM and exposes a fixed set of functions, to be used by the client. @@ -269,12 +245,14 @@ Everything else you need in a blockchain, except the consensus-critical, determi - Compiled to native. - Less need for determinism. - Has access to anything a normal native binary does (memory, disk, syscalls etc.) +- Does all the other shared things that most blockchains want + - Database, Networking, Mempool, Consensus.. ---v ### The Client - + ---v @@ -331,6 +309,29 @@ Because the runtime can change independently! +---v + +### SMOLDOT: Compile the Client to WASM + + + + + +A marvel of universe ๐Ÿคฏ. + +- (light) Substrate\* client compiled to WASM, by the browser. +- Itself executing another WASM blob, the aforementioned runtime. + + + + + + + + + + + --- ## Communication Paths @@ -436,24 +437,93 @@ https://www.cleanpng.com/png-game-boy-advance-deviantart-video-game-consoles-218 ## Substrate and Polkadot - + + +--- + +## Substrate and Smart Contracts + + + +NOTE: + +I was asked this yesterday as well. My latest answer is: if you don't need any of the customizations +that a blockchain client/runtime gives to you, and the performance of a shared platform is okay for +you, then go with a smart contract. If you need more, you need a "runtime" (some kind of chian, +parachain or solo) + +An example of customization is that a runtime has access to `on_initialize` etc. + +Also, a contract usually depends on a token for gas, while a runtime can be in principle token-less +fee-less. + +---v + +### Substrate and Smart Contracts + + + +---v + +### Substrate and Smart Contracts + +- So a SMOLDOT instance, syncing a substrate based chain which has pallet-contracts is ...๐Ÿค” + +---v + +### Substrate and Smart Contracts + + + + + + + + + +- a WASM blob (smoldot) +- that executed a WASM blob (runtime) +- that executed a WASM blob (contract) + + + --- ## Technical Freedom vs Ease - + --- ## Rest of This Module! ๐Ÿ˜ˆ + + + +##### Lecture + +- Day 0: + - **Introduction** + - Folder structure. +- Day 1: + - **WASM Meta-Protocol** + - SCALE, JSON-RPC +- Day 2: + - **Storage** + - Substrate CLI, TX-Pool + + + + +##### Activity + - Day 0: - - **Introduction To Substrate** - - Activities.. + - Compiling Rust to WASM - Day 1: - - **WASM Meta Protocol** - - Activities.. + - FRAME-less Activity - Day 2: - - **Substrate Storage** - - More activities.. + - FRAME-less Activity + + + diff --git a/syllabus/4-Substrate/4.2-Substrate-Folder-Structure_Slides.md b/syllabus/4-Substrate/4.2-Substrate-Folder-Structure_Slides.md index 20551c4a6..e6bb59ed3 100644 --- a/syllabus/4-Substrate/4.2-Substrate-Folder-Structure_Slides.md +++ b/syllabus/4-Substrate/4.2-Substrate-Folder-Structure_Slides.md @@ -27,8 +27,6 @@ Substrate is roughly composed of 3 parts: Primitives is the glue between the other two. -> Extra Activity: Go search for some crates based on these prefixes in https://paritytech.github.io/substrate/. - ---v ### Substrate Internally: `./client` @@ -63,11 +61,22 @@ Primitives is the glue between the other two. ---v +### Workshop + +- Go search for some crates based on these prefixes in https://paritytech.github.io/substrate/. +- Look into the codebase as well. + +---v + ### Substrate Internally +- So what's the point in all of this? ๐Ÿคจ + - When looking for the code related to a given topic, this information should help you find it. +

+
- Networking? only on `sc-*` - Database/Storage? Probably in `sc-*` and `sp-*` diff --git a/syllabus/4-Substrate/4.3-WASM-Meta-Protocol-Slides.md b/syllabus/4-Substrate/4.3-WASM-Meta-Protocol-Slides.md index b4b1686a1..8eb97fef9 100644 --- a/syllabus/4-Substrate/4.3-WASM-Meta-Protocol-Slides.md +++ b/syllabus/4-Substrate/4.3-WASM-Meta-Protocol-Slides.md @@ -159,6 +159,34 @@ Now, let's look at another example task that has functions in both client and ru ### Example #2: Block Import +- We have firmly established the fact that the client sees the state is opaque bytes. + +Let's think about it. Why is that? + +- So the runtime can change its storage layout/definition in a forkless manner! + + + +---v + +### Example #2: Block Import + +- Does the same situation apply to the extrinsic? + +- Of course! we want the runtime to be able to change its extrinsic format as well! + + + +---v + +### Example #2: Block Import + + + +---v + +### Example #2: Block Import + ```rust [1-100|1-2|4-6|8-9|1-100] // fetch the block from the outer world. It is opaque. let opaque_block: Vec = networking::import_queue::next_block(); @@ -176,49 +204,24 @@ runtime.execute_block(opaque_block); ### Example #2: Block Import - ๐Ÿ’ก The client needs a runtime API to ask the runtime to execute the block. -- ๐Ÿง  Notice that the api receives an `OpaqueBlock` but works on `RuntimeBlock`. ```rust -/// Some known type in the runtime. -type RuntimeBlock = ...; - -/// More host functions trait RuntimeApis { - fn execute_block(opaque_block: Vec) -> Result<_, _> { - let block: RuntimeBlock = opaque_block.decode(); - block.execute_and_stuff(); - ... - } + fn execute_block(opaque_block: Vec) -> Result<_, _> { .. } } ``` ----v - -### Example #2: Block Import - -- Question: Why is the block also opaque in the client? + -```rust -// Client's rough view on the block type. -struct ClientBlock { - header: H, - extrinsics: Vec>, -} +--- -trait Header { - type Number; - type Hash; - fn number() -> Self::Number; - fn parent_hash() -> Self::Hash; - ... -} -``` +# Detour - +- Let's talk about Blocks and Extrinsics for a second. ----v +--- -### Definition: Extrinsic +### Detour: Extrinsic > An Extrinsic is data that come from outside of the runtime. @@ -232,6 +235,58 @@ Yes, transactions are **a type of extrinsic**, but not all extrinsics are transa ---v +### Detour: Block, Header, Extrinsic + +- Both the client and runtime have a concrete type about what each of these are. +- But they need a common understanding of them. + +Which Rust abstraction is perfect for this? + +---v + +### Detour: Block, Header, Extrinsic + +- The traits defining what each are in `sp-runtime/traits` + - [Extrinsic](https://paritytech.github.io/substrate/master/sp_runtime/traits/trait.Extrinsic.html), [Block](https://paritytech.github.io/substrate/master/sp_runtime/traits/trait.Block.html), [Header](https://paritytech.github.io/substrate/master/sp_runtime/traits/trait.Header.html) +- One, somewhat opinionated set of types that implement these can be found in `sp-runtime/generic`. + +NOTE: + +This will come into play more in your frame-less activity + +---v + +### Detour: Client Block + +- Now we can imagine what the client's view on the block will look like. + +```rust +// Client's rough view on the block type. +struct ClientBlock { + header: H, + extrinsics: Vec>, +} + +impl sp_runtime::traits::Block for ClientBlock { .. } + +``` + +NOTE: + +sort of advance activity: how can `Vec` implement `traits::Extrinsic`? it kinda does, but also +kinda doesn't: + +```rust +impl traits::Extrinsic for OpaqueExtrinsic { + type Call = (); + type SignaturePayload = (); +} +``` + +TODO: this can certainly be improved in substrate + +---v + ### Example #2: Block Import - And what would be a `RuntimeBlock`? @@ -240,24 +295,40 @@ Yes, transactions are **a type of extrinsic**, but not all extrinsics are transa struct Header { .. } struct Extrinsic = { .. } +impl sp_runtime::traits::Block for RuntimeBlock { .. } +impl sp_runtime::traits::Header for Header { .. } + struct RuntimeBlock { header: Header, extrinsics: Vec, } ``` -Notes: +--- + +# Detour Ends Here. -Notice that the header type itself can still be generic, but it needs to fulfill some requirements of a header trait, such as having parent hash. +--- + +### Example #2: Block Import + +- With the information gained from that, let's expand this snipped: -Most of the block is only decodable in the runtime. Why? Because the block encodes -extrinsics/transaction. Transactions, are part of the application logic of the blockchain and can -change, when a runtime changes. So, the same rules that apply to "storage keys" technically apply to -transactions as well. They can change at the same abstraction layer, every time that the runtime -changes. Therefore, neither are known to the client. +```rust +trait RuntimeApis { + fn execute_block(opaque_block: Vec) -> Result<_, _> { .. } +} +``` -In reality, the block is partially decoded in the client (up to being split into `header` and -`extrinsics`). +- to + +```rust +trait RuntimeApis { + fn execute_block(opaque_block: ClientBlock) -> Result<_, _> { + // The implementor will convert this internally to `RuntimeBlock` + } +} +``` ---v @@ -345,8 +416,7 @@ let result = client.api.function_name(input_data, block_hash); ### Example #2: Block Import -- But this is even now, still not an accurate depiction ๐Ÿ‘Ž. -- Recall that anyone that imports the block should check its state root! +- I can add one more small touch to this to make it more accurate.. ๐ŸคŒ ---v @@ -537,8 +607,8 @@ No, beginning of next block. Because `execute_block` is one runtime api call. th - if the code changes, all the following can also change: -1. What state key is kian's balance. -2. What block/extrinsic format is valid. + - What state key is kian's balance. + - What block/extrinsic format is valid. - How on earth is an application (i.e. a wallet) is supposed to survive? @@ -554,8 +624,8 @@ trait RuntimeApis { } ``` -- Combined with the fact that every runtime API is tied to the runtime code of a given block. -- Two different wasm blobs in block `N` and `N+1` return different values in this function. +- Two different wasm blobs in block `N` and `N+1` return different metadata. +- Client(s) (including but not limited to Substrate's) should communicate with the metadata to get the missing information. Notes: @@ -568,13 +638,13 @@ probably had the same class of issues. This is the same matter, on a a different --- -## PHEW. That Was a Close One ๐Ÿ˜ฎโ€๐Ÿ’จ +## Oblivious Client ๐Ÿ™ˆ๐Ÿ™‰ - The underlying reason why the client is "**kept in the dark**" is so that it wouldn't need to care about the runtime upgrading from one block to the other.
-### THEREFORE: +#### THEREFORE: - Client does not know **storage layout**. It can change! - Client does not know the internal **extrinsic/block format**. It can change! @@ -584,7 +654,7 @@ Ideally, a client should only use the metadata to find the answer to these unkno Notes: -This is why forkless upgrades are possible in substrate. +This is why forkless upgrades are possible in substrate. All of the component that the client is oblivious to --- @@ -603,7 +673,7 @@ This is why forkless upgrades are possible in substrate. - look for `impl_runtime_apis! {...}` and `decl_runtime_apis! {...}` macro calls. - Try and find the corresponding the client code calling a given api as well. - Look for `#[runtime_interface]` macro. -- You have 15 minutes! +- You have 30 minutes! ---v @@ -621,6 +691,42 @@ This is why forkless upgrades are possible in substrate. - A blockchain validators implements `BlockBuilder` and `TxQueue` as well. - A lot of other runtime APIs _could_ be optional depending on the context. +---v + +### Example Update on `BlockBuilder` APIs + +- From the previous slides: + +```rust +trait RuntimeApis { + fn initialize_block(..) { ... } + // note the opaque type. + fn apply_extrinsic(ext: Vec) -> Result<_, _> { ... } + fn finalize_block(..) { ... } +} +``` + +- But now you have discovered: +- The client builds a raw header, passes it to the runtime in `on_initialize()` and expects to have + it returned to it `on_finalize()` + +---v + +### Example Update on `BlockBuilder` APIs + +- From the previous slides: + +```rust +trait RuntimeApis { + // pass in the raw header, this does not have any of the roots. + fn initialize_block(raw_header: Block::Header) { ... } + // note the opaque type. + fn apply_extrinsic(ext: Vec) -> Result<_, _> { ... } + // the final header is returned. + fn finalize_block() -> Block::Header { ... } +} +``` + --- # Part 2: Advance Topics @@ -658,6 +764,7 @@ let outcome: Vec = api.execute_block(block, block_hash).unwrap(); ### Defining a Runtime API: Takeaways +- All runtime APIs are generic over a `` by default. - All runtime APIs are executed on top of a **specific block**. This is the implicit _at_ parameter. - Going over the API, **everything is SCALE encoded both ways**: - Return type is `Result, _>` under the hood. Client needs to know how to decode the inner `Vec`. @@ -851,15 +958,18 @@ indefinitely. In other words, a DOS vector. > A panic in `initialize_block` and `finalize_block` have even more catastrophic effects, which will > be discussed further in the FRAME section. -TODO: workshop: make a panicing runtime, and DoS it out. +workshop idea: make a panicing runtime, and DoS it out. +workshop idea for FRAME: find all instances where the runtime actually correctly panics (wrong timestamp, disabled validator) ---v ### Considerations: Panic -- Panic in a user-callable code path is typically abusable ๐Ÿ˜ . +- Panic in a user-callable code path is _typically_ abusable ๐Ÿ˜ . - Panic on "automatic" part of your blockchain like `initialize_block` are deadly ๐Ÿ’€. +- Once you get to FRAME, you can find some examples where panics are expected. + --- ### Consideration: Adding Host Function @@ -1143,20 +1253,6 @@ SomeExternalities.execute_with(|| { ---v -### Activity: Frame-less Runtime: Part 1 - -- Add some variant to your simple extrinsic such that it writes a u32 to a single value onchain. -- Track extrinsic root and state root. -- Submit a bunch of extrinsics to set a u32 value, make sure a second node can also sync your chain. -- Which runtime APIs are called in the authoring node and which on the syncing node? - ----v - -### Activity: Frame-less Runtime: Part 2 +### Activity: Frame-less Runtime: -- Make your chain upgradable. This sounds scary, but is in reality super simple to do! -- Now change the format of BasicExtrinsic (e.g. u32 -> u128 + bump spec version), upgrade your runtime. -- idea: upgrade your runtime, but forget to bump the spec version. Now try having a node running - native, and one running wasm, and enjoy seeing the burn in fires. -- Now try and submit a new transaction... can you? This will fail, until you you make the extrinsic type opaque -- Finally, make your runtime have an inherent that sets the timestamp. +See: https://github.com/Polkadot-Blockchain-Academy/frameless-node-template diff --git a/syllabus/4-Substrate/4.4-Merkle-db_slides.md b/syllabus/4-Substrate/4.4-Merkle-db_slides.md index f61f24219..3a30761e6 100644 --- a/syllabus/4-Substrate/4.4-Merkle-db_slides.md +++ b/syllabus/4-Substrate/4.4-Merkle-db_slides.md @@ -46,6 +46,7 @@ By convention, an externality has a "**backend**" that is in charge of dealing w
+- "_Storage keys_" directly map to _database keys_. - O(1) Read and write. - Hash all the data once to get a root. diff --git a/syllabus/4-Substrate/4.5.Advacnce-Misc-Substrate.md b/syllabus/4-Substrate/4.5.Advacnce-Misc-Substrate.md deleted file mode 100644 index b19a14ebd..000000000 --- a/syllabus/4-Substrate/4.5.Advacnce-Misc-Substrate.md +++ /dev/null @@ -1,13 +0,0 @@ -# Advance/Misc Topics in Substrate - -## Extrinsic Types - -## Transaction Pool - -## Warp Sync - -## `feature = std` / `no_std` - -## Recovery: Panic `on_initialize` - -## Interesting Postmortems to read diff --git a/syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Deep-Dive-Slides.md b/syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Deep-Dive-Slides.md new file mode 100644 index 000000000..7af0b8b03 --- /dev/null +++ b/syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Deep-Dive-Slides.md @@ -0,0 +1,288 @@ +--- +title: FRAME Deep Dive +description: FRAME, Pallets, System Pallet, Executive, Runtime Amalgamator. +duration: 1 hour +instructors: ["Kian Paimani"] +--- + +# FRAME Deep Dive + +---v + +## Agenda + +Recall the following figure: + + + +Notes: + +Without frame, there is the runtime and there is the client, and an API that sits in between. + +---v + +## Agenda + +By the end of this lecture, you will fully understand this figure. + + + +--- + +## Expanding A Pallet + +- Grab a simple pallet code, and expand it. + +- `Pallet` implements the transactions as public functions. +- `Pallet` implements `Hooks`, and some equivalents like `OnInitialize`. +- `enum Call` that has in itself is just an encoding of the transaction's data + +- and implements `UnfilteredDispatchable` (which just forward the call back to `Pallet`) + +---v + +### Expanding A Pallet + +Make sure you understand why these 3 are the same! + +```rust +let origin = ..; + +// function call +Pallet::::set_value(origin, 10); + +// dispatch +Call::::set_value(10).dispatch_bypass_filter(origin); + +// fully qualified syntax. + as UnfilteredDispatch>::dispatch_bypass_filter(Call::::set_value(10), origin); +``` + +--- + +## `construct_runtime!` and Runtime Amalgamator. + +- Now, let's look at a minimal runtime amalgamator. +- This is where you glue the runtime together, and expose the things you care about as runtime apis. + +---v + +### `construct_runtime!` and Runtime Amalgamator. + +```rust +#![cfg_attr(not(feature = "std"), no_std)] + +#[sp_version::runtime_version] +pub const VERSION: RuntimeVersion = RuntimeVersion { .. }; + +pub mod opaque { .. } + +parameter_types! { .. } +impl frame_system::Config for Runtime { .. } + +parameter_types! { .. } +impl pallet_xyz::Config for Runtime { .. } + +parameter_types! { .. } +impl pallet_pqr::Config for Runtime { .. } + +construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = opaque::Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: frame_system, + PalletXyz: pallet_xyx, + PalletPqr: pallet_pqr, + } +); + +// This is what in your frameless-runtime was `BasicExtrinsic`. +type UncheckedExtrinsic = generic::UncheckedExtrinsic<_, _, _, _>; +type Header = .. +type Block = generic::Block; +type Executive = frame_executive::Executive; + +// this is the juicy part! all implementations seem to come from Executive! +impl_runtime_apis! { + impl sp_api::Core for Runtime { + fn version() -> RuntimeVersion { + VERSION + } + + fn execute_block(block: Block) { + Executive::execute_block(block); + } + + fn initialize_block(header: &::Header) { + Executive::initialize_block(header) + } + } + + ... +} +``` + +---v + +### `construct_runtime!` and Runtime Amalgamator. + +Let's expand a runtime, or alternatively, look at the rust-docs which also contain a lot of the +information. + +---v + +### `construct_runtime!` and Runtime Amalgamator. + +- struct `Runtime` +- implements the `Config` trait of all pallets. +- implements all of the runtime APIs as functions. +- `type System`, `type SimplePallet`. +- `AllPalletsWithSystem` etc. + - and recall that all pallets implement things like `Hooks`, `OnInitialize`, and all of these + traits are tuple-able. +- enum Call +- enum `Event`, `GenesisConfig`, etc. but we don't have them here. + +--- + +## 4. Executive + +- This part is somewhat optional to know in advance, but I want you to re-visit it in a week and then understand it all. + +- I present to you, Executive struct: + +```rust +pub struct Executive< + System, + Block, + Context, + UnsignedValidator, + AllPalletsWithSystem, + OnRuntimeUpgrade = (), +>(..); +``` + +---v + +#### Expanding The Generic Types. + +```rust +impl< + // System config, we know this now. + System: frame_system::Config, + // The block type. + Block: sp_runtime::traits::Block
, + // Something that has all the hooks. We don't know anything else about pallets here. + AllPalletsWithSystem: OnRuntimeUpgrade + + OnInitialize + + OnIdle + + OnFinalize + + OffchainWorker, + COnRuntimeUpgrade: OnRuntimeUpgrade, + > Executive +where + // This is the juicy party, and we have to learn more sp_runtime traits to follow. + Block::Extrinsic: Checkable, + ::Checked: Applyable + <::Checked as Applyable>::Call: Dispatchable<_>, +{...} +``` + +---v + +#### `Block::Extrinsic: Checkable` + +- Who implements `Checkable`? +- That's right, the `generic::UncheckedExtrinsic` that we indeed used as `Block::Extrinsic` in the + top level runtime. Recall: + +```rust +type UncheckedExtrinsic = generic::UncheckedExtrinsic<_, _, _, _>; +type Header = .. +type Block = generic::Block; +type Executive = frame_executive::Executive<_, Block, ...>; +``` + +---v + +#### What Does `Checkable<_>` Do? + +- Signature verification! + +```rust +impl Checkable<_> for UncheckedExtrinsic<_, _, _, _> { + // this is the output type. + type Checked = CheckedExtrinsic; + + fn check(self, lookup: &Lookup) -> Result { + .. + } +} +``` + +---v + +#### `::Checked: Applyable` + +- `UncheckedExtrinsic::Checked` is `CheckedExtrinsic`. +- And it surely does implement `Applyable`. + +---v + +#### What Does `Applyable<_>` Do? + +- TLDR: `Ok(self.call.dispatch(maybe_who.into()))` + +---v + +#### Lastly: `<::Checked as Applyable>::Call: Dispatchable` + +- And guess who implemented `Dispatchable`, which we already looked at! +- The `enum Call` that we had in our expanded file! + +---v + +### Circling Back.. + +So, to recap: + +```rust +struct Runtime; + +impl frame_system::Config for Runtime {} +impl simple_pallet::Config for Runtime {} + +enum Call { + System(frame_system::Call), + SimplePallet(simple_pallet::Call), +} + +impl Dispatchable for Call { + fn dispatch(self, origin: _) -> Result<_, _> { + match self { + Call::System(system_call) => system_call.dispatch(), + Call::SimplePallet(simple_pallet_call) => system_pallet_call.dispatch(), + } + } +} + +struct UncheckedExtrinsic { + function: Call, + signature: Option<_>, +} + +type Executive = Executive<_, UncheckedExtrinsic, ...>; + +// +let unchecked = UncheckedExtrinsic::new(); +let checked = unchecked.check(); +let _ = checked.apply(); +``` + +---v + +### Workshop + +- Walk over execute, namely `execute_block`, and see how much of it makes sense. diff --git a/syllabus/5-FRAME/4-Exotic_Stuff/frame-tips-and-tricks_slides.md b/syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Tipc-Tricks-Slides.md similarity index 69% rename from syllabus/5-FRAME/4-Exotic_Stuff/frame-tips-and-tricks_slides.md rename to syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Tipc-Tricks-Slides.md index 02e720ad2..936bc751d 100644 --- a/syllabus/5-FRAME/4-Exotic_Stuff/frame-tips-and-tricks_slides.md +++ b/syllabus/5-FRAME/4-Exotic_Stuff/FRAME-Tipc-Tricks-Slides.md @@ -14,11 +14,14 @@ These are relevant for coding in FRAME and Substrate. --- -## Recap: `trait Block`, `Header`, `Extrinsic` +# Part 1 Substrate Stuff -Generic definitions of what each of these _should be_.. +--- + +## Recap: Blocks, Headers, Extrinsics -.. One implementation of which can be found in `generic` folder. +- The traits defining what each are in `sp-runtime/traits` +- One, somewhat opinionated set of types that implement these can be found in `sp-runtime/generic`. ---v @@ -45,11 +48,31 @@ type BalanceOf = < ---v -## Detour: Traits, Generics, Associated Types +## Speaking of Traits.. -Notes: +- Anything that can be expressed with associated types can also be expressed with generics. +- Associated types usually lead to less boilerplate. + +```rust +trait Engine {} +trait Brand {} + +trait Car { + // brand is possibly the same among all, so make it associate! + type Brand: Brand; +} +struct Car; +// Car and Car are not the same type! +// Car and Car could not have different brands. +// fn foo>() { .. } ``` + +Notes: + +In cambridge, I did this this. But since students should now know traits really well, I will drop it. + +```rust trait Engine { fn start() {} } @@ -65,14 +88,14 @@ trait Car { type Brand: Brand; } -struct MyCar; -struct MyBrand; -impl Brand for MyBrand { - fn name() -> &'static str { - "Kian" +struct KianCarCo; +impl Brand for KianCarCo { + fn name() -> &'static str { + "KianCarCo!" } } +struct MyCar; impl Car for MyCar { type Brand = MyBrand; } @@ -101,281 +124,9 @@ fn main() { --- -## `trait Get` - -A very basic, yet very substrate-idiomatic way to pass _values_ through _types_. - -```rust -pub trait Get { - fn get() -> T; -} - -// very basic blanket implementation, which you should be very versed in reading. -impl Get for () { - fn get() -> T { - T::default() - } -} -``` - ----v - -### `trait Get` - -```rust -parameter_types! { - pub const Foo: u32 = 10; -} - -// expands to: -pub struct Foo; -impl Get for Foo { - fn get() -> u32 { - 10; - } -} -``` - -> Helps convey **values** using **types**. - ---- - -## `bounded` - -- `BoundedVec`, `BoundedSlice`, `BoundedBTreeMap`, `BoundedSlice` - -```rust -#[cfg_attr(feature = "std", derive(Serialize), serde(transparent))] -#[derive(Encode)] -pub struct BoundedVec>( - pub(super) Vec, - PhantomData, -); -``` - -> `PhantomData`? - ----v - -### `bounded` - -Food for your thought. - -```rust -#[cfg_attr(feature = "std", derive(Serialize))] -#[derive(Encode)] -pub struct BoundedVec( - pub(super) Vec, - u32, -); -``` - ---- - -## `trait Convert` - -```rust -pub trait Convert { - fn convert(a: A) -> B; -} - -pub struct Identity; -// blanket implementation! -impl Convert for Identity { - fn convert(a: T) -> T { - a - } -} -``` - -Notes: -this one's much simpler, but good excuse to teach them blanket implementations. - ----v - -### Example of `Get` and `Convert` - -```rust -/// Some configuration for my module. -trait InputConfig { - /// Something that gives you a `usize`. - type MaximumSize: Get; - /// Something that is capable of converting `u64` to `u32`. - type ImpossibleConvertor: Convertor; -} - -struct Pallet { - fn foo() { - let outcome: u32 = T::ImpossibleConvertor::convert(u64::max_value()); - } -} - -struct Runtime; -impl InputConfig for Runtime { - type MaximumSize = (); // remember what this means? - type ImpossibleConvertor = ... -} -``` - ---- - -## Implementing Traits For Tuples - -```rust -struct Module1; -struct Module2; -struct Module3; - -trait OnInitialize { - fn on_initialize(); -} - -impl OnInitialize for Module1 { fn on_initialize() {} } -impl OnInitialize for Module2 { fn on_initialize() {} } -impl OnInitialize for Module3 { fn on_initialize() {} } -``` - -How can I easily invoke `OnInitialize` on all 3 of `Module1, Module2, Module3`? Explore in Rust -Playground! - -Notes: - -take this to rust playground. - -add: - -trait OnInitializeDyn { -fn on_initialize(&self); -} - -impl OnInitializeDyn for Module1 { fn on_initialize(&self) {} } -impl OnInitializeDyn for Module2 { fn on_initialize(&self) {} } -impl OnInitializeDyn for Module3 { fn on_initialize(&self) {} } - -fn main() { -// let x = vec![Module1, Module1, Module1]; -// let x: Vec> = vec![Box::new(Module1), Box::new(Module2)]; -let x: Vec> = vec![Box::new(Module1), Box::new(Module2)]; -x.for_each(|i| i.on_initialize()) -x.for_each(OnInitialize::on_initialize) -} - ----v - -### Implementing Traits For Tuples - -> Dynamic dispatch could help us achieve what we want, but it adds runtime overhead. - -1. `on_initialize`, in its ideal form, does not have `&self`, it is defined on the **type**, not a **value**. -2. **Tuples** are the natural way to group **types** together (analogous to have a **vector** is the - natural way to group **values** together..) - -```rust -// fully-qualified syntax - turbo-fish. -<(Module1, Module2, Module3) as OnInitialize>::on_initialize(); -``` - ----v - -### Implementing Traits For Tuples - -```rust -struct Module1; -struct Module2; -struct Module3; - -trait OnInitialize { - fn on_initialize(); -} - -impl OnInitialize for Module1 { fn on_initialize() {} } -impl OnInitialize for Module2 { fn on_initialize() {} } -impl OnInitialize for Module3 { fn on_initialize() {} } - -impl OnInitialize for (T1, T2) { - fn on_initialize() { - T1::on_initialize(); - T2::on_initialize(); - } -} - -impl OnInitialize for (T1, T2, T3) { - fn on_initialize() { - T1::on_initialize(); - T2::on_initialize(); - T3::on_initialize(); - } -} -``` - ----v - -### Implementing Traits For Tuples - -Only problem: A lot of boilerplate. Macros! - -Historically, we made this work with `macro_rules!` - -e.g. https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=4822dfa5bc2acc528a0a1487789eb064 - -Notes: - -```rust -macro_rules! impl_for_tuples { - ( $( $elem:ident ),+ ) => { - impl<$( $elem: OnInitialize, )*> OnInitialize for ($( $elem, )*) { - fn on_initialize() { - $( $elem::on_initialize(); )* - } - } - } -} - -impl_for_tuples!(A, B, C, D); -impl_for_tuples!(A, B, C, D, E); -impl_for_tuples!(A, B, C, D, E, F); -``` - ----v - -### Implementing Traits For Tuples - -But then Basti saved us: - -```rust -// basic -#[impl_for_tuples(30)] -pub trait OnTimestampSet { - fn on_timestamp_set(moment: Moment); -} - -// slightly more advance -#[impl_for_tuples(30)] -impl OnRuntimeUpgrade for Tuple { - fn on_runtime_upgrade() -> crate::weights::Weight { - let mut weight = 0; - for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); - weight - } -} -``` - ----v - -### Implementing Traits for Tuples: Further Reading - -- https://stackoverflow.com/questions/64332037/how-can-i-store-a-type-in-an-array -- https://doc.rust-lang.org/book/ch17-02-trait-objects.html#trait-objects-perform-dynamic-dispatch -- https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name -- https://turbo.fish/ -- https://techblog.tonsser.com/posts/what-is-rusts-turbofish -- https://docs.rs/impl-trait-for-tuples/latest/impl_trait_for_tuples/ - ---- - ## The `std` Paradigm -https://docs.rust-embedded.org/embedonomicon/smallest-no-std.html +- [source](https://docs.rust-embedded.org/embedonomicon/smallest-no-std.html) > #![no_std] is a crate level attribute that indicates that the crate will link to the `core` crate > instead of the `std` crate.. std crate is Rust's standard library. It contains functionality @@ -396,8 +147,8 @@ All crates in substrate that eventually compile to WASM are compiled in a dual m #![cfg_attr(not(feature = "std"), no_std)] ``` -> The name "`std`" is just an idiom in the rust ecosystem. -> https://rust-lang.github.io/api-guidelines/naming.html#feature-names-are-free-of-placeholder-words-c-feature +- The name "`std`" is just an idiom in the rust ecosystem. +- `no_std` DOES NOT MEAN WASM. ---v @@ -464,9 +215,6 @@ error: duplicate lang item in crate sp_io (which frame_support depends on): oom. A subset of the standard types in rust that also exist in rust `core` are re-exported from `sp_std`. -- https://doc.rust-lang.org/core/index.html -- https://doc.rust-lang.org/std/index.html - ```rust sp_std::prelude::*; ``` @@ -512,19 +260,22 @@ fn foo() { ### The `std` Paradigm: Further Reading: - https://paritytech.github.io/substrate/master/sp_std/index.html +- https://doc.rust-lang.org/core/index.html +- https://doc.rust-lang.org/std/index.html +- https://rust-lang.github.io/api-guidelines/naming.html#feature-names-are-free-of-placeholder-words-c-feature --- ## Logging And Prints In The Runtime. -First, why the fuss? +- First, why the fuss? -Size of the wasm blob matters.. +- Size of the wasm blob matters.. -Any logging increases the size of the WASM blob. **String literals** are stored somewhere in your -program! +- Any logging increases the size of the WASM blob. **String literals** are stored somewhere in your + program! @@ -532,19 +283,17 @@ program! ### Logging And Prints In The Runtime. -> wasm2wat polkadot_runtime.wasm > dump | rg stripped +- `wasm2wat polkadot_runtime.wasm > dump | rg stripped` -Should get you the `.rodata` (read-only data) line of the wasm blob, which contains all the logging -noise. +- Should get you the `.rodata` (read-only data) line of the wasm blob, which contains all the logging + noise. -> This contains string literals form errors, logs, metadata, etc. +- This contains string literals form errors, logs, metadata, etc. ---v ### Logging And Prints In The Runtime. -- `Debug` vs. `RuntimeDebug`. - ```rust #[derive(RuntimeDebug)] pub struct WithDebug { @@ -575,233 +324,481 @@ impl ::core::fmt::Debug for WithDebug { Once types implement `Debug` or `RuntimeDebug`, they can be printed. Various ways: -1. If you only want something in tests, native builds etc +- If you only want something in tests, native builds etc + +```rust +sp_std::if_std! { + println!("hello world!"); + dbg!(foo); +} +``` + +- Or you can use the common frame-support logging (which is just the `log` crate re-exported): + +```rust +frame_support::log::info!(target: "target", "hello world!"); +frame_support::log::debug!(target: "target", "hello world! ({})", 10u32); +``` + +---v + +### Logging And Prints In The Runtime. + +- Log statements are only evaluated if the corresponding level and target is met. + +```rust +/// only executed if `RUST_LOG=KIAN=trace` +frame_support::log::trace!(target: "KIAN", "({:?})", (0..100000).into_iter().collect()); +``` + +- `disable-logging` compilation flag blocks all sp-io calls to do any logging. This is used in + official polkadot releases. + +Notes: + +`log` in rust does not do anything -- it only tracks what needs to be logged. Then you need a logger +to actually export them. In rust this is often `env_logger` or `sp_tracing` in substrate tests. + +In the runtime, the log messages are sent via the host functions to the client to be printed. + +If the interface is built with `disable-logging`, it omits all log messages. + +--- + +## Arithmetic Helpers, and the `f32`, `f64` Story. + +- Floating point numbers have different standards, and (**_slightly_**) different implementations on + different architectures and vendors. + +- If my balance is `10.000000000000001` DOT on one validator and `10.000000000000000` DOT on another validator, game over for your consensus ๐Ÿ˜ฎโ€๐Ÿ’จ. + +---v + +### PerThing. + +```python +> .2 + .2 + .2 == .6 +> false +``` + +``` +> a = 10 +> b = 0.1 +> c = 0.2 +> a*(b+c) == a*b + a*c +> false +``` + +- Google "weird float behavior" fro more entertainment around this. + +---v + +### PerThing. + +- We store ratios and such in the runtime with "Fixed-Point" arithmetic types. + +```rust +struct Percent(u8); + +impl Percent { + fn new(x: u8) { + Self(x.min(100)); + } +} + +impl Mul for Percent { + ... +} + +``` + +---v + +### PerThing. + +```rust +use sp_arithmetic::Perbill; + +let p = Perbill::from_part_parts(1_000_000_000u32 / 4); +let p = Perbill::from_percent(25); +let p = Perbill::from_rational(1, 4); + +> p * 100u32; +> 25u32; +``` + +- Some precision concerns exist, but that's a story for another day. + +---v + +### Fixed Point Numbers + +`Per-thing` is great for representing `[0, 1]` range. + +What if we need more? + +``` +100 ~ 1 +200 ~ 2 +300 ~ 3 +350 ~ 3.5 +``` + +---v + +### Fixed Point Numbers + +```rust +use sp_arithmetic::FixedU64; + +let x = FixedU64::from_rational(5, 2); +let y = 10u32; +let z = x * y; +> 25 +``` + +---v + +### Larger Types + +- [`U256`](https://paritytech.github.io/substrate/master/sp_core/struct.U256.html), `U512`: battle-tested since the ethereum days. +- substrate-fixed: community project. Supercharged `PerThing` and `Fixed`. +- [`big_uint.rs`](https://paritytech.github.io/substrate/master/sp_arithmetic/biguint/index.html) (unaudited) + +```rust + +pub struct BigUint { + /// digits (limbs) of this number (sorted as msb -> lsb). + pub(crate) digits: Vec, +} +``` + +---v + +### Arithmetic Types + +- Everything said here can be found in `sp-arithmetic` and `sp-core`, and a lot of it is re-exported from `sp-runtime` +- Because they are used a LOT. + +--- + +### Fallibility: Math Operations + +Things like **addition**, **multiplication**, **division** could all easily fail. + +- Panic + + - `u32::MAX * u32::MAX / 2` (in debug builds) + - `100 / 0` + +- Overflow + - `u32::MAX * u32::MAX / 2` (in release builds) + +---v + +### Fallibility + +- `Checked` -- prevention โœ‹๐Ÿป + + ``` + if let Some(outcome) = a.checked_mul(b) { ... } else { ... } + ``` + +- `Saturating` -- silent recovery ๐Ÿคซ + +``` +let certain_output = a.saturating_mul(b); +``` + +NOTE: + +Why would you ever want to saturate? only in cases where you know if the number is overflowing, +other aspects of the system is so fundamentally screwed that there is no point in doing any kind of +recovery. + +There's also `wrapping_op` and `carrying_op` etc on all rust primitives, but not quite +relevant. + +---v + +### Fallibility: Conversion + +- Luckily, rust is already pretty strict for the primitive types. +- `TryInto` / `TryFrom` / `From` / `Into` + +```rust +/// T is u32 or larger. +struct Foo> + +/// T is u32 or smaller +struct Foo> + +/// It can maybe be converted to u32 +struct Foo> +``` + +---v + +### Fallibility: Conversion + +- Substrate also provides a trait for infallible saturated conversion as well. ```rust -sp_std::if_std! { - println!("hello world!") +trait SaturatedConversion { + fn saturated_into(self) -> T } -``` -1. Or you can use the common frame-support logging (which is just the `log` crate re-exported): - -```rust -frame_support::log::info!(target: "target", "hello world!"); -frame_support::log::debug!(target: "target", "hello world! ({})", 10u32); +assert_eq!(u128::MAX.saturating_into::(), u32::MAX); ``` ----v - -### Logging And Prints In The Runtime. +--- -1. Log statements are only evaluated if the corresponding level and target is met. +# Part 2: FRAME Stuff -``` -/// only executed if `RUST_LOG=target=trace` -frame_support::log::trace!(target: "target", "({:?})", (0..100000).into_iter().collect()); -``` +--- -2. `disable-logging` compilation flag blocks all sp-io calls to do any logging. This is used in - official polkadot releases. +## `trait Get` -Notes: +A very basic, yet very substrate-idiomatic way to pass _values_ through _types_. -`log` in rust does not do anything -- it only tracks what needs to be logged. Then you need a logger -to actually export them. In rust this is often `env_logger` or `sp_tracing` in substrate tests. +```rust +pub trait Get { + fn get() -> T; +} -In the runtime, the log messages are sent via the host functions to the client to be printed. +// very basic blanket implementation, which you should be very versed in reading. +impl Get for () { + fn get() -> T { + T::default() + } +} +``` -If the interface is built with `disable-logging`, it omits all log messages. +---v ---- +### `trait Get` -## Arithmetic Helpers, and the `f32`, `f64` Story. +```rust +parameter_types! { + pub const Foo: u32 = 10; +} -Floating point numbers have different standards, and (**_slightly_**) different implementations on -different architectures and vendors. +// expands to: +pub struct Foo; +impl Get for Foo { + fn get() -> u32 { + 10; + } +} +``` -> If my balance is `10.000000000000001` DOT on one validator and `10.000000000000000` DOT on another -> validator, game over for your consensus. +- Helps convey **values** using **types**. ----v +--- -### PerThing. +## `bounded` -```python -> .2 + .2 + .2 == .6 -> false -``` +- `BoundedVec`, `BoundedSlice`, `BoundedBTreeMap`, `BoundedSlice` -``` -> a = 10 -> b = 0.1 -> c = 0.2 -> a*(b+c) == a*b + a*c -> false +```rust +#[derive(Encode, Decode)] +pub struct BoundedVec>( + pub(super) Vec, + PhantomData, +); ``` -Google "weird float behavior" fro more entertainment around this. +- `PhantomData`? ---v -### PerThing. +### `bounded` -- We store ratios and such in the runtime with "Fixed-Point" arithmetic types. +- Food for your thought. ```rust -implement_per_thing!( - Percent, - 100u8, - u8, - "_Percent_" -); -implement_per_thing!( - Perbill, - 1_000_000_000u32, - u32, - "_Parts per Billion_", -); -implement_per_thing!( - Perquintill, - 1_000_000_000_000_000_000u64, - u64, - "_Parts per Quintillion_", +#[cfg_attr(feature = "std", derive(Serialize))] +#[derive(Encode)] +pub struct BoundedVec( + pub(super) Vec, + u32, ); ``` ----v +--- -### PerThing. +## `trait Convert` -``` -let p = Perbill::from_part_parts(1_000_000_000u32 / 4); -let p = Perbill::from_percent(25); -let p = Perbill::from_rational(1, 4); +```rust +pub trait Convert { + fn convert(a: A) -> B; +} -> p * 100u32; -> 25u32; +pub struct Identity; +// blanket implementation! +impl Convert for Identity { + fn convert(a: T) -> T { + a + } +} ``` -Some precision concerns exist, but that's a story for another day. +Notes: +this one's much simpler, but good excuse to teach them blanket implementations. ---v -### Fixed Point Numbers - -`Per-thing` is great for representing `[0, 1]` range. +### Example of `Get` and `Convert` -What if we need more? +```rust +/// Some configuration for my module. +trait Config { + /// Something that gives you a `u32`. + type MaximumSize: Get; + /// Something that is capable of converting `u64` to `u32`, which is pretty damn impossible. + type Convertor: Convertor; +} -``` -100 ~ 1 -200 ~ 2 -300 ~ 3 -350 ~ 3.5 +// in your top level runtime. +struct Runtime; +impl Config for Runtime { + type MaximumSize = (); // remember what this means? + type ImpossibleConvertor = ... +} ``` ---v -### Fixed Point Numbers +### Example of `Get` and `Convert` ```rust -implement_fixed!( - FixedU64, - u64, - 1_000_000_000, - "_Fixed Point 64 bits unsigned, range = [0.000000000, 18446744073.709551615]_", -); - -implement_fixed!( - FixedU128, - u128, - 1_000_000_000_000_000_000, - "_Fixed Point 128 bits unsigned, range = \ - [0.000000000000000000, 340282366920938463463.374607431768211455]_", -); +// in your pallet +impl Pallet { + fn foo() { + let outcome: u32 = T::Convertor::convert(u64::max_value()); + } +} ``` ----v - -### Larger Types +--- -- `U256`, `U512`: battle-tested since the ethereum days. -- substrate-fixed: community project. Supercharged `PerThing` and `Fixed`. -- `big_uint.rs` (unaudited) +## Implementing Traits For Tuples ```rust +struct Module1; +struct Module2; +struct Module3; -pub struct BigUint { - /// digits (limbs) of this number (sorted as msb -> lsb). - pub(crate) digits: Vec, +trait OnInitialize { + fn on_initialize(); } -``` ----v +impl OnInitialize for Module1 { fn on_initialize() {} } +impl OnInitialize for Module2 { fn on_initialize() {} } +impl OnInitialize for Module3 { fn on_initialize() {} } +``` -### Further Reading: +How can I easily invoke `OnInitialize` on all 3 of `Module1, Module2, Module3`? -- https://paritytech.github.io/substrate/master/sp_arithmetic/index.html -- https://paritytech.github.io/substrate/master/sp_core/uint/index.html +Notes: ----v +take this to rust playground. -### Fallibility +add: -**Conversions** are very much fallible operations. So are things like **addition**, -**multiplication**, **division** (all in `std::ops`, if keen on reading some rustdocs). +trait OnInitializeDyn { +fn on_initialize(&self); +} -- Panic +impl OnInitializeDyn for Module1 { fn on_initialize(&self) {} } +impl OnInitializeDyn for Module2 { fn on_initialize(&self) {} } +impl OnInitializeDyn for Module3 { fn on_initialize(&self) {} } -`u32::MAX * u32::MAX / 2` (in debug builds) +fn main() { +// let x = vec![Module1, Module1, Module1]; +// let x: Vec> = vec![Box::new(Module1), Box::new(Module2)]; +let x: Vec> = vec![Box::new(Module1), Box::new(Module2)]; +x.for_each(|i| i.on_initialize()) +x.for_each(OnInitialize::on_initialize) +} -`100 / 0` +---v -- Overflow +### Implementing Traits For Tuples -`u32::MAX * u32::MAX / 2` (in release builds) +> Dynamic dispatch could help us achieve what we want, but it adds runtime overhead. ----v +1. `on_initialize`, in its ideal form, does not have `&self`, it is defined on the **type**, not a **value**. +2. **Tuples** are the natural way to group **types** together (analogous to have a **vector** is the + natural way to group **values** together..) -### Fallibility +```rust +// fully-qualified syntax - turbo-fish. +<(Module1, Module2, Module3) as OnInitialize>::on_initialize(); +``` -1. `Checked` -- recover +---v -`if let Some(outcome) = a.checked_mul(b) { ... } else { ... }` +### Implementing Traits For Tuples -2. `Saturating` -- soft recovery +Only problem: A lot of boilerplate. Macros! -`let certain_output = a.saturating_mul(b);` +Historically, we made this work with `macro_rules!` -
+Notes: -There's also `wrapping_op` and `carrying_op` etc on all rust primitives, but not quite relevant. +```rust +macro_rules! impl_for_tuples { + ( $( $elem:ident ),+ ) => { + impl<$( $elem: OnInitialize, )*> OnInitialize for ($( $elem, )*) { + fn on_initialize() { + $( $elem::on_initialize(); )* + } + } + } +} -https://doc.rust-lang.org/std/primitive.u32.html +impl_for_tuples!(A, B, C, D); +impl_for_tuples!(A, B, C, D, E); +impl_for_tuples!(A, B, C, D, E, F); +``` ---v -### Fallibility - -Luckily, rust is already pretty strict for the primitive types. +### Implementing Traits For Tuples -- `TryInto` / `TryFrom` / `From` / `Into` +But then Basti saved us: ```rust -/// T is u32 or larger. -struct Foo> +// basic +#[impl_for_tuples(30)] +pub trait OnTimestampSet { + fn on_timestamp_set(moment: Moment); +} -/// T is u32 or smaller -struct Foo> +// slightly more advance +#[impl_for_tuples(30)] +impl OnRuntimeUpgrade for Tuple { + fn on_runtime_upgrade() -> crate::weights::Weight { + let mut weight = 0; + for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); + weight + } +} +``` -/// It can maybe be converted to u32 -struct Foo> +---v -/// It can be converted into u32 at the cost of loss of accuracy. -/// This is a substrate trait. -struct Foo> +### Implementing Traits for Tuples: Further Reading -assert_eq!(u128::MAX.saturating_into::(), u32::MAX); -``` +- useful links: + +* https://stackoverflow.com/questions/64332037/how-can-i-store-a-type-in-an-array +* https://doc.rust-lang.org/book/ch17-02-trait-objects.html#trait-objects-perform-dynamic-dispatch +* https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name +* https://turbo.fish/ +* https://techblog.tonsser.com/posts/what-is-rusts-turbofish +* https://docs.rs/impl-trait-for-tuples/latest/impl_trait_for_tuples/ --- @@ -812,6 +809,12 @@ assert_eq!(u128::MAX.saturating_into::(), u32::MAX); > used where **high availability**, **safety**, or **security** is needed. - As you know, you should (almost) never panic in your runtime code. + +---v + +### Defensive Programming + +- First reminder: don't panic, unless if you want to punish someone! - `.unwrap()`? no no - be careful with implicit unwraps in standard operations! - slice/vector indexing can panic if out of bound @@ -822,9 +825,9 @@ assert_eq!(u128::MAX.saturating_into::(), u32::MAX); ### Defensive Programming -When using operations that could panic, comment exactly above it why you are sure it won't panic. +- When using operations that could panic, comment exactly above it why you are sure it won't panic. -``` +```rust let pos = announcements .binary_search(&announcement) .ok() @@ -839,7 +842,7 @@ announcements.remove(pos); Or when using options or results that need to be unwrapped but are known to be `Ok(_)`, `Some()`: -``` +```rust let maybe_value: Option<_> = ... if maybe_value.is_none() { return "..." @@ -848,7 +851,8 @@ if maybe_value.is_none() { let value = maybe_value.expect("value checked to be 'Some'; qed"); ``` -> Q.E.D. or QED is an initialism of the Latin phrase quod erat demonstrandum, meaning "which was to be demonstrated". +- Q.E.D. or QED is an initialism of the Latin phrase "quod erat demonstrandum", meaning "**which was + to be demonstrated**". ---v @@ -877,9 +881,45 @@ pub fn try_insert(&mut self, index: usize, element: T) -> Result<(), ()> { ### Defensive Programming -The overall ethos of defensive programming is along the lines of: +- Speaking of documentation, [here's a very good guideline](https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html)! + +```` +/// Multiplies the given input by two. +/// +/// Some further information about what this does, and where it could be used. +/// +/// ``` +/// fn main() { +/// let x = multiply_by_2(10); +/// assert_eq!(10, 20); +/// } +/// ``` +/// +/// ## Panics +/// +/// Panics under such and such condition. +fn multiply_by_2(x: u32) -> u32 { .. } +```` + +---v + +### Defensive Programming ``` +/// This function works with module x and multiples the given input by two. If +/// we optimize the other variant of it, we would be able to achieve more +/// efficiency but I have to think about it. Probably can panic if the input +/// overflows u32. +fn multiply_by_2(x: u32) -> u32 { .. } +``` + +---v + +### Defensive Programming + +- The overall ethos of defensive programming is along the lines of: + +```rust // we have good reasons to believe this is `Some`. let y: Option<_> = ... @@ -893,13 +933,13 @@ let x = y.unwrap_or(reasonable_default); let x = y.ok_or(Error::DefensiveError)?; ``` +- But, for example, you are absolutely sure that `Error::DefensiveError` will never happen, can we enforce it better? + ---v ### Defensive Programming -Slightly better: - -[Defensive traits](https://paritytech.github.io/substrate/master/frame_support/traits/trait.Defensive.html): +- Yes: [Defensive traits](https://paritytech.github.io/substrate/master/frame_support/traits/trait.Defensive.html): ``` // either return a reasonable default.. diff --git a/syllabus/5-FRAME/4-Exotic_Stuff/Migrations_and_try_runtime_slides.md b/syllabus/5-FRAME/4-Exotic_Stuff/Migrations_and_try_runtime_slides.md index efdef7fd6..302b7ad88 100644 --- a/syllabus/5-FRAME/4-Exotic_Stuff/Migrations_and_try_runtime_slides.md +++ b/syllabus/5-FRAME/4-Exotic_Stuff/Migrations_and_try_runtime_slides.md @@ -35,11 +35,10 @@ https://www.crowdcast.io/e/substrate-seminar/41 ### When is a Migration Required? -In a typical runtime upgrade, you typically only replace `:code:`. This is _Runtime Upgrade_. +- In a typical runtime upgrade, you typically only replace `:code:`. This is _**Runtime Upgrade**_. +- If you change the _storage layout_, then this is also a _**Runtime Migration**_. -If you change the _storage layout_, then this is also a _Runtime Migration_. - -> What is a change here? Anything that changes **encoding**! +> Anything that changes **encoding** is a migration! ---v @@ -51,10 +50,10 @@ pub type FooValue = StorageValue<_, Foo>; ``` ```rust - // old - pub struct Foo(u32) - // new - pub struct Foo(u64) +// old +pub struct Foo(u32) +// new +pub struct Foo(u64) ``` A clear migration. @@ -71,15 +70,15 @@ pub type FooValue = StorageValue<_, Foo>; ``` ```rust - // old - pub struct Foo(u32) - // new - pub struct Foo(i32) - // or - pub struct Foo(u16, u16) +// old +pub struct Foo(u32) +// new +pub struct Foo(i32) +// or +pub struct Foo(u16, u16) ``` -Not so clear: The data still fits, but the interpretations is almost certainly different! +The data still _fits_, but the _interpretations_ is almost certainly different! @@ -93,14 +92,12 @@ pub type FooValue = StorageValue<_, Foo>; ``` ```rust - // old - pub struct Foo { a: u32, b: u32 } - // new - pub struct Foo { a: u32, b: u32, c: u32 } +// old +pub struct Foo { a: u32, b: u32 } +// new +pub struct Foo { a: u32, b: u32, c: u32 } ``` -Extending a struct is an interesting edge case... - This is still a migration, because `Foo`'s decoding changed. @@ -115,10 +112,10 @@ pub type FooValue = StorageValue<_, Foo>; ``` ```rust - // old - pub struct Foo { a: u32, b: u32 } - // new - pub struct Foo { a: u32, b: u32, c: PhantomData<_> } +// old +pub struct Foo { a: u32, b: u32 } +// new +pub struct Foo { a: u32, b: u32, c: PhantomData<_> } ``` If for whatever reason `c` has a type that its encoding is like `()`, then this would work. @@ -143,6 +140,8 @@ pub type FooValue = StorageValue<_, Foo>; Extending an enum is even more interesting, because if you add the variant to the end, no migration is needed. + + Assuming that no value is initialized with `C`, this is _not_ a migration. @@ -157,10 +156,10 @@ pub type FooValue = StorageValue<_, Foo>; ``` ```rust - // old - pub enum Foo { A(u32), B(u32) } - // new - pub enum Foo { A(u32), C(u128), B(u32) } +// old +pub enum Foo { A(u32), B(u32) } +// new +pub enum Foo { A(u32), C(u128), B(u32) } ``` You probably _never_ want to do this, but it is a migration. @@ -176,8 +175,6 @@ Enums are encoded as the variant enum, followed by the inner data: - The order matters! - Enums that implement `Encode` cannot have more than 255 variants. -> Remember: It is all about how a type `encode` and `decodes`. - ---v ### When is a Migration Required? @@ -193,8 +190,13 @@ pub type FooValue = StorageValue<_, u32>; pub type BarValue = StorageValue<_, u32>; ``` -So far everything is changing the _value_ format.
-The _key_ changing is also a migration! +- So far everything is changing the _value_ format.
+ +
+ +- The _key_ changing is also a migration! + +
@@ -204,14 +206,14 @@ The _key_ changing is also a migration! ```rust #[pallet::storage] -pub type FooValue = StorageValue<_, Foo>; +pub type FooValue = StorageValue<_, u32>; ``` ```rust // new #[pallet::storage_prefix = "FooValue"] #[pallet::storage] -pub type I_can_NOW_BE_renamEd = StorageValue<_, u32>; +pub type I_can_NOW_BE_renamEd_hahAA = StorageValue<_, u32>; ``` Handy macro if you must rename a storage type.
@@ -223,25 +225,21 @@ This does _not_ require a migration. ## Writing Runtime Migrations -Notes: - -Now that we know how to detect if a storage change is a **migration**, let's see how we write one. +- Now that we know how to detect if a storage change is a **migration**, let's see how we write one. ---v ### Writing Runtime Migrations -Once you upgrade a runtime, the code is expecting the data to be in a new format. - -Any `on_initialize` or transaction might fail decoding data, and potentially `panic!` +- Once you upgrade a runtime, the code is expecting the data to be in a new format. +- Any `on_initialize` or transaction might fail decoding data, and potentially `panic!` ---v ### Writing Runtime Migrations -We need a **_hook_** that is executed **ONCE** as a part of the new runtime... - -But before **ANY** other code (on_initialize, any transaction) with the new runtime is migrated. +- We need a **_hook_** that is executed **ONCE** as a part of the new runtime... +- But before **ANY** other code (on_initialize, any transaction) with the new runtime is migrated. > This is `OnRuntimeUpgrade`. @@ -251,11 +249,8 @@ But before **ANY** other code (on_initialize, any transaction) with the new runt ### Writing Runtime Migrations -`OnRuntimeUpgrade` is called before any block being `initialize`ed, every time the spec version changes. - -> Q: why don't we need to keep the old storage definitions in the code (similar to host functions)? -> -> A: Every block is executed with the wasm code of **that block**. +- Optional activity: Go into `executive` and `system`, and find out how `OnRuntimeUpgrade` is called + only when the code changes! ---