From a6c3ebff0b0a32f46e033449b60602249badd464 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 22 Jan 2024 19:00:47 +0100 Subject: [PATCH 01/57] merkle list --- src/examples/zkapps/token/merkle-list.ts | 68 ++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/examples/zkapps/token/merkle-list.ts diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts new file mode 100644 index 0000000000..7a1e696884 --- /dev/null +++ b/src/examples/zkapps/token/merkle-list.ts @@ -0,0 +1,68 @@ +import { Field, Poseidon, Provable, Struct, Unconstrained } from 'o1js'; + +export { MerkleList }; + +class Element extends Struct({ previousHash: Field, element: Field }) {} + +const emptyHash = Field(0); +const dummyElement = Field(0); + +class MerkleList { + hash: Field; + value: Unconstrained; + + private constructor(hash: Field, value: Element[]) { + this.hash = hash; + this.value = Unconstrained.from(value); + } + + isEmpty() { + return this.hash.equals(emptyHash); + } + + static create(): MerkleList { + return new MerkleList(emptyHash, []); + } + + push(element: Field) { + let previousHash = this.hash; + this.hash = Poseidon.hash([previousHash, element]); + Provable.asProver(() => { + this.value.set([...this.value.get(), { previousHash, element }]); + }); + } + + private popWitness() { + return Provable.witness(Element, () => { + let value = this.value.get(); + let head = value.at(-1) ?? { + previousHash: emptyHash, + element: dummyElement, + }; + this.value.set(value.slice(0, -1)); + return head; + }); + } + + pop(): Field { + let { previousHash, element } = this.popWitness(); + + let requiredHash = Poseidon.hash([previousHash, element]); + this.hash.assertEquals(requiredHash); + + this.hash = previousHash; + return element; + } + + popOrDummy(): Field { + let { previousHash, element } = this.popWitness(); + + let isEmpty = this.isEmpty(); + let correctHash = Poseidon.hash([previousHash, element]); + let requiredHash = Provable.if(isEmpty, emptyHash, correctHash); + this.hash.assertEquals(requiredHash); + + this.hash = Provable.if(isEmpty, emptyHash, previousHash); + return Provable.if(isEmpty, dummyElement, element); + } +} From f89c5a71280e32ee0a9459f020e1cb9d1e512e40 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 22 Jan 2024 21:48:50 +0100 Subject: [PATCH 02/57] export unconstrained properly --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 8fb4e0c9ef..8b8aa0672e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,6 @@ export type { FlexibleProvable, FlexibleProvablePure, InferProvable, - Unconstrained, } from './lib/circuit_value.js'; export { CircuitValue, @@ -31,6 +30,7 @@ export { provable, provablePure, Struct, + Unconstrained, } from './lib/circuit_value.js'; export { Provable } from './lib/provable.js'; export { Circuit, Keypair, public_, circuitMain } from './lib/circuit.js'; From dc58706d5a8584ea5dd6364e483d7b518d3d971a Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 22 Jan 2024 21:49:04 +0100 Subject: [PATCH 03/57] export hash with prefix --- src/lib/hash.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/lib/hash.ts b/src/lib/hash.ts index 6f4dbc9347..7f766c9412 100644 --- a/src/lib/hash.ts +++ b/src/lib/hash.ts @@ -66,6 +66,17 @@ const Poseidon = { return MlFieldArray.from(newState) as [Field, Field, Field]; }, + hashWithPrefix(prefix: string, input: Field[]) { + let init = Poseidon.update(Poseidon.initialState(), [ + prefixToField(prefix), + ]); + return Poseidon.update(init, input)[0]; + }, + + initialState(): [Field, Field, Field] { + return [Field(0), Field(0), Field(0)]; + }, + hashToGroup(input: Field[]) { if (isConstant(input)) { let result = PoseidonBigint.hashToGroup(toBigints(input)); @@ -118,10 +129,6 @@ const Poseidon = { return Poseidon.hash(packed); }, - initialState(): [Field, Field, Field] { - return [Field(0), Field(0), Field(0)]; - }, - Sponge, }; From 9719eb76357ce7c200e888d5d632302380198d14 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 22 Jan 2024 21:49:19 +0100 Subject: [PATCH 04/57] generic merkle tree --- src/examples/zkapps/token/merkle-list.ts | 52 +++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index 7a1e696884..3cefc1e941 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -4,65 +4,87 @@ export { MerkleList }; class Element extends Struct({ previousHash: Field, element: Field }) {} +type WithHash = { previousHash: Field; element: T }; +function WithHash(type: ProvableHashable): Provable> { + return Struct({ previousHash: Field, element: type }); +} + const emptyHash = Field(0); -const dummyElement = Field(0); -class MerkleList { +class MerkleList { hash: Field; - value: Unconstrained; + value: Unconstrained[]>; + provable: ProvableHashable; + nextHash: (hash: Field, t: T) => Field; - private constructor(hash: Field, value: Element[]) { + private constructor( + hash: Field, + value: WithHash[], + provable: ProvableHashable, + nextHash: (hash: Field, t: T) => Field + ) { this.hash = hash; this.value = Unconstrained.from(value); + this.provable = provable; + this.nextHash = nextHash; } isEmpty() { return this.hash.equals(emptyHash); } - static create(): MerkleList { - return new MerkleList(emptyHash, []); + static create( + provable: ProvableHashable, + nextHash: (hash: Field, t: T) => Field + ): MerkleList { + return new MerkleList(emptyHash, [], provable, nextHash); } - push(element: Field) { + push(element: T) { let previousHash = this.hash; - this.hash = Poseidon.hash([previousHash, element]); + this.hash = this.nextHash(previousHash, element); Provable.asProver(() => { this.value.set([...this.value.get(), { previousHash, element }]); }); } private popWitness() { - return Provable.witness(Element, () => { + return Provable.witness(WithHash(this.provable), () => { let value = this.value.get(); let head = value.at(-1) ?? { previousHash: emptyHash, - element: dummyElement, + element: this.provable.empty(), }; this.value.set(value.slice(0, -1)); return head; }); } - pop(): Field { + pop(): T { let { previousHash, element } = this.popWitness(); - let requiredHash = Poseidon.hash([previousHash, element]); + let requiredHash = this.nextHash(previousHash, element); this.hash.assertEquals(requiredHash); this.hash = previousHash; return element; } - popOrDummy(): Field { + popOrDummy(): T { let { previousHash, element } = this.popWitness(); let isEmpty = this.isEmpty(); - let correctHash = Poseidon.hash([previousHash, element]); + let correctHash = this.nextHash(previousHash, element); let requiredHash = Provable.if(isEmpty, emptyHash, correctHash); this.hash.assertEquals(requiredHash); this.hash = Provable.if(isEmpty, emptyHash, previousHash); - return Provable.if(isEmpty, dummyElement, element); + return Provable.if(isEmpty, this.provable, this.provable.empty(), element); } } + +type HashInput = { fields?: Field[]; packed?: [Field, number][] }; +type ProvableHashable = Provable & { + toInput: (x: T) => HashInput; + empty: () => T; +}; From 6540c420728c6866266df30a458b09eeef963517 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 22 Jan 2024 21:49:30 +0100 Subject: [PATCH 05/57] start call forest --- src/examples/zkapps/token/call-forest.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/examples/zkapps/token/call-forest.ts diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts new file mode 100644 index 0000000000..4384f51c54 --- /dev/null +++ b/src/examples/zkapps/token/call-forest.ts @@ -0,0 +1,17 @@ +import { AccountUpdate, Field, Hashed } from 'o1js'; +import { MerkleList } from './merkle-list.js'; + +export { CallForest }; + +class CallTree { + accountUpdate: Hashed; + calls: CallTree[]; +} + +// basically a MerkleList if MerkleList were generic +type CallForest = MerkleList[]; + +class PartialCallForest { + forest: Hashed; + pendingForests: MerkleList; +} From 8219ef0aa63367d2df4519852f93226b2a39d82e Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Tue, 23 Jan 2024 00:55:24 +0100 Subject: [PATCH 06/57] call forest logic --- src/examples/zkapps/token/call-forest.ts | 69 +++++++++++++++++++++--- src/examples/zkapps/token/merkle-list.ts | 24 +++++---- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 4384f51c54..0c431bb991 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -1,17 +1,72 @@ -import { AccountUpdate, Field, Hashed } from 'o1js'; -import { MerkleList } from './merkle-list.js'; +import { AccountUpdate, Field, Hashed, Poseidon, Unconstrained } from 'o1js'; +import { MerkleList, WithStackHash, emptyHash } from './merkle-list.js'; export { CallForest }; -class CallTree { +class HashedAccountUpdate extends Hashed.create(AccountUpdate, (a) => + a.hash() +) {} + +type CallTree = { accountUpdate: Hashed; - calls: CallTree[]; -} + calls: CallForest; +}; + +type CallForest = WithStackHash; + +const CallForest = { + fromAccountUpdates(updates: AccountUpdate[]): CallForest { + let forest = CallForest.empty(); + + for (let update of [...updates].reverse()) { + let accountUpdate = HashedAccountUpdate.hash(update); + let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); + CallForest.cons(forest, { accountUpdate, calls }); + } + return forest; + }, + + empty(): CallForest { + return { hash: emptyHash, stack: Unconstrained.from([]) }; + }, -// basically a MerkleList if MerkleList were generic -type CallForest = MerkleList[]; + cons(forest: CallForest, tree: CallTree) { + let node = { previousHash: forest.hash, element: tree }; + let nodeHash = CallForest.hashNode(tree); + + forest.stack.set([...forest.stack.get(), node]); + forest.hash = CallForest.hashCons(forest, nodeHash); + }, + + hashNode(tree: CallTree) { + return Poseidon.hashWithPrefix('MinaAcctUpdateNode**', [ + tree.accountUpdate.hash, + tree.calls.hash, + ]); + }, + hashCons(forest: CallForest, nodeHash: Field) { + return Poseidon.hashWithPrefix('MinaAcctUpdateCons**', [ + forest.hash, + nodeHash, + ]); + }, + + provable: WithStackHash(), +}; + +const CallForestHashed = Hashed.create( + CallForest.provable, + (forest) => forest.hash +); class PartialCallForest { forest: Hashed; pendingForests: MerkleList; + + constructor(forest: CallForest) { + this.forest = CallForestHashed.hash(forest); + this.pendingForests = MerkleList.create(CallForest.provable, (hash, t) => + Poseidon.hash([hash, t.hash]) + ); + } } diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index 3cefc1e941..1fd6b09c2c 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -1,19 +1,25 @@ -import { Field, Poseidon, Provable, Struct, Unconstrained } from 'o1js'; +import { Field, Provable, ProvableExtended, Struct, Unconstrained } from 'o1js'; -export { MerkleList }; - -class Element extends Struct({ previousHash: Field, element: Field }) {} +export { MerkleList, WithHash, WithStackHash, emptyHash }; type WithHash = { previousHash: Field; element: T }; function WithHash(type: ProvableHashable): Provable> { return Struct({ previousHash: Field, element: type }); } +type WithStackHash = { + hash: Field; + stack: Unconstrained[]>; +}; +function WithStackHash(): ProvableExtended> { + return Struct({ hash: Field, stack: Unconstrained.provable }); +} + const emptyHash = Field(0); class MerkleList { hash: Field; - value: Unconstrained[]>; + stack: Unconstrained[]>; provable: ProvableHashable; nextHash: (hash: Field, t: T) => Field; @@ -24,7 +30,7 @@ class MerkleList { nextHash: (hash: Field, t: T) => Field ) { this.hash = hash; - this.value = Unconstrained.from(value); + this.stack = Unconstrained.from(value); this.provable = provable; this.nextHash = nextHash; } @@ -44,18 +50,18 @@ class MerkleList { let previousHash = this.hash; this.hash = this.nextHash(previousHash, element); Provable.asProver(() => { - this.value.set([...this.value.get(), { previousHash, element }]); + this.stack.set([...this.stack.get(), { previousHash, element }]); }); } private popWitness() { return Provable.witness(WithHash(this.provable), () => { - let value = this.value.get(); + let value = this.stack.get(); let head = value.at(-1) ?? { previousHash: emptyHash, element: this.provable.empty(), }; - this.value.set(value.slice(0, -1)); + this.stack.set(value.slice(0, -1)); return head; }); } From c65c92c506375c13d065fbc7f40fd1ab31018f41 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Tue, 23 Jan 2024 01:37:19 +0100 Subject: [PATCH 07/57] properly generic merkle list --- src/examples/zkapps/token/merkle-list.ts | 88 ++++++++++++++++++------ 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index 1fd6b09c2c..b58599fb15 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -1,6 +1,14 @@ -import { Field, Provable, ProvableExtended, Struct, Unconstrained } from 'o1js'; - -export { MerkleList, WithHash, WithStackHash, emptyHash }; +import { + Field, + Provable, + ProvableExtended, + Struct, + Unconstrained, + assert, +} from 'o1js'; +import { provableFromClass } from 'src/bindings/lib/provable-snarky.js'; + +export { MerkleList, WithHash, WithStackHash, emptyHash, ProvableHashable }; type WithHash = { previousHash: Field; element: T }; function WithHash(type: ProvableHashable): Provable> { @@ -20,30 +28,18 @@ const emptyHash = Field(0); class MerkleList { hash: Field; stack: Unconstrained[]>; - provable: ProvableHashable; - nextHash: (hash: Field, t: T) => Field; - private constructor( - hash: Field, - value: WithHash[], - provable: ProvableHashable, - nextHash: (hash: Field, t: T) => Field - ) { + constructor(hash: Field, value: WithHash[]) { this.hash = hash; this.stack = Unconstrained.from(value); - this.provable = provable; - this.nextHash = nextHash; } isEmpty() { return this.hash.equals(emptyHash); } - static create( - provable: ProvableHashable, - nextHash: (hash: Field, t: T) => Field - ): MerkleList { - return new MerkleList(emptyHash, [], provable, nextHash); + static empty(): MerkleList { + return new this(emptyHash, []); } push(element: T) { @@ -55,11 +51,11 @@ class MerkleList { } private popWitness() { - return Provable.witness(WithHash(this.provable), () => { + return Provable.witness(WithHash(this.innerProvable), () => { let value = this.stack.get(); let head = value.at(-1) ?? { previousHash: emptyHash, - element: this.provable.empty(), + element: this.innerProvable.empty(), }; this.stack.set(value.slice(0, -1)); return head; @@ -85,7 +81,57 @@ class MerkleList { this.hash.assertEquals(requiredHash); this.hash = Provable.if(isEmpty, emptyHash, previousHash); - return Provable.if(isEmpty, this.provable, this.provable.empty(), element); + let provable = this.innerProvable; + return Provable.if(isEmpty, provable, provable.empty(), element); + } + + /** + * Create a Merkle list type + */ + static create( + type: ProvableHashable, + nextHash: (hash: Field, t: T) => Field + ): typeof MerkleList { + return class MerkleList_ extends MerkleList { + static _innerProvable = type; + + static _provable = provableFromClass(MerkleList_, { + hash: Field, + stack: Unconstrained.provable, + }) as ProvableHashable>; + + static _nextHash = nextHash; + }; + } + + // dynamic subclassing infra + static _nextHash: ((hash: Field, t: any) => Field) | undefined; + + static _provable: ProvableHashable> | undefined; + static _innerProvable: ProvableHashable | undefined; + + get Constructor() { + return this.constructor as typeof MerkleList; + } + + nextHash(hash: Field, t: T): Field { + assert( + this.Constructor._nextHash !== undefined, + 'MerkleList not initialized' + ); + return this.Constructor._nextHash(hash, t); + } + + static get provable(): ProvableHashable> { + assert(this._provable !== undefined, 'MerkleList not initialized'); + return this._provable; + } + get innerProvable(): ProvableHashable { + assert( + this.Constructor._innerProvable !== undefined, + 'MerkleList not initialized' + ); + return this.Constructor._innerProvable; } } From 85f917412e294fe174e19493e7832b6fd51e943f Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Tue, 23 Jan 2024 01:37:58 +0100 Subject: [PATCH 08/57] make callforest a merkle list --- src/examples/zkapps/token/call-forest.ts | 82 +++++++++++------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 0c431bb991..1646d3e476 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -1,5 +1,5 @@ -import { AccountUpdate, Field, Hashed, Poseidon, Unconstrained } from 'o1js'; -import { MerkleList, WithStackHash, emptyHash } from './merkle-list.js'; +import { AccountUpdate, Field, Hashed, Poseidon, Struct } from 'o1js'; +import { MerkleList, ProvableHashable, WithStackHash } from './merkle-list.js'; export { CallForest }; @@ -9,64 +9,58 @@ class HashedAccountUpdate extends Hashed.create(AccountUpdate, (a) => type CallTree = { accountUpdate: Hashed; - calls: CallForest; + calls: WithStackHash; }; +const CallTree: ProvableHashable = Struct({ + accountUpdate: HashedAccountUpdate.provable, + calls: WithStackHash(), +}); -type CallForest = WithStackHash; +class CallForest extends MerkleList.create(CallTree, function (hash, tree) { + return hashCons(hash, hashNode(tree)); +}) { + static empty(): CallForest { + return super.empty(); + } -const CallForest = { - fromAccountUpdates(updates: AccountUpdate[]): CallForest { + static fromAccountUpdates(updates: AccountUpdate[]): CallForest { let forest = CallForest.empty(); for (let update of [...updates].reverse()) { let accountUpdate = HashedAccountUpdate.hash(update); let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); - CallForest.cons(forest, { accountUpdate, calls }); + forest.push({ accountUpdate, calls }); } - return forest; - }, - - empty(): CallForest { - return { hash: emptyHash, stack: Unconstrained.from([]) }; - }, - - cons(forest: CallForest, tree: CallTree) { - let node = { previousHash: forest.hash, element: tree }; - let nodeHash = CallForest.hashNode(tree); - forest.stack.set([...forest.stack.get(), node]); - forest.hash = CallForest.hashCons(forest, nodeHash); - }, - - hashNode(tree: CallTree) { - return Poseidon.hashWithPrefix('MinaAcctUpdateNode**', [ - tree.accountUpdate.hash, - tree.calls.hash, - ]); - }, - hashCons(forest: CallForest, nodeHash: Field) { - return Poseidon.hashWithPrefix('MinaAcctUpdateCons**', [ - forest.hash, - nodeHash, - ]); - }, - - provable: WithStackHash(), -}; + return forest; + } +} -const CallForestHashed = Hashed.create( - CallForest.provable, - (forest) => forest.hash +const PendingForests = MerkleList.create(CallForest.provable, (hash, t) => + Poseidon.hash([hash, t.hash]) ); class PartialCallForest { - forest: Hashed; + forest: CallForest; pendingForests: MerkleList; constructor(forest: CallForest) { - this.forest = CallForestHashed.hash(forest); - this.pendingForests = MerkleList.create(CallForest.provable, (hash, t) => - Poseidon.hash([hash, t.hash]) - ); + this.forest = forest; + this.pendingForests = PendingForests.empty(); } } + +// how to hash a forest + +function hashNode(tree: CallTree) { + return Poseidon.hashWithPrefix('MinaAcctUpdateNode**', [ + tree.accountUpdate.hash, + tree.calls.hash, + ]); +} +function hashCons(forestHash: Field, nodeHash: Field) { + return Poseidon.hashWithPrefix('MinaAcctUpdateCons**', [ + forestHash, + nodeHash, + ]); +} From c4dc1f47f0d793dc8c20d928e53072c99bd24eed Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Tue, 23 Jan 2024 02:33:46 +0100 Subject: [PATCH 09/57] start writing pop account update --- src/examples/zkapps/token/call-forest.ts | 29 +++++++++++++++++++--- src/examples/zkapps/token/merkle-list.ts | 31 +++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 1646d3e476..fa476bf727 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -16,12 +16,13 @@ const CallTree: ProvableHashable = Struct({ calls: WithStackHash(), }); -class CallForest extends MerkleList.create(CallTree, function (hash, tree) { - return hashCons(hash, hashNode(tree)); -}) { +class CallForest extends MerkleList.create(CallTree, nextHash) { static empty(): CallForest { return super.empty(); } + static from(value: WithStackHash): CallForest { + return new this(value.hash, value.stack); + } static fromAccountUpdates(updates: AccountUpdate[]): CallForest { let forest = CallForest.empty(); @@ -34,6 +35,11 @@ class CallForest extends MerkleList.create(CallTree, function (hash, tree) { return forest; } + + pop() { + let { accountUpdate, calls } = super.pop(); + return { accountUpdate, calls: CallForest.from(calls) }; + } } const PendingForests = MerkleList.create(CallForest.provable, (hash, t) => @@ -48,10 +54,27 @@ class PartialCallForest { this.forest = forest; this.pendingForests = PendingForests.empty(); } + + popAccountUpdate() { + let { accountUpdate, calls: forest } = this.forest.pop(); + let restOfForest = this.forest; + + this.pendingForests.pushIf(restOfForest.isEmpty().not(), restOfForest); + this.forest = forest; + + // TODO add a notion of 'current token' to partial call forest, + // or as input to this method + // TODO replace forest with empty forest if account update can't access current token + let update = accountUpdate.unhash(); + } } // how to hash a forest +function nextHash(forestHash: Field, tree: CallTree) { + return hashCons(forestHash, hashNode(tree)); +} + function hashNode(tree: CallTree) { return Poseidon.hashWithPrefix('MinaAcctUpdateNode**', [ tree.accountUpdate.hash, diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index b58599fb15..16e5f0390f 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -1,4 +1,5 @@ import { + Bool, Field, Provable, ProvableExtended, @@ -15,23 +16,27 @@ function WithHash(type: ProvableHashable): Provable> { return Struct({ previousHash: Field, element: type }); } +const emptyHash = Field(0); + type WithStackHash = { hash: Field; stack: Unconstrained[]>; }; function WithStackHash(): ProvableExtended> { - return Struct({ hash: Field, stack: Unconstrained.provable }); + return class extends Struct({ hash: Field, stack: Unconstrained.provable }) { + static empty(): WithStackHash { + return { hash: emptyHash, stack: Unconstrained.from([]) }; + } + }; } -const emptyHash = Field(0); - class MerkleList { hash: Field; stack: Unconstrained[]>; - constructor(hash: Field, value: WithHash[]) { + constructor(hash: Field, value: Unconstrained[]>) { this.hash = hash; - this.stack = Unconstrained.from(value); + this.stack = value; } isEmpty() { @@ -39,7 +44,7 @@ class MerkleList { } static empty(): MerkleList { - return new this(emptyHash, []); + return new this(emptyHash, Unconstrained.from([])); } push(element: T) { @@ -50,6 +55,20 @@ class MerkleList { }); } + pushIf(condition: Bool, element: T) { + let previousHash = this.hash; + this.hash = Provable.if( + condition, + this.nextHash(previousHash, element), + previousHash + ); + Provable.asProver(() => { + if (condition.toBoolean()) { + this.stack.set([...this.stack.get(), { previousHash, element }]); + } + }); + } + private popWitness() { return Provable.witness(WithHash(this.innerProvable), () => { let value = this.stack.get(); From c21ca68a72bd0154a321febc4f8fbbff46a6e09c Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Tue, 23 Jan 2024 02:51:01 +0100 Subject: [PATCH 10/57] finish core pop account update logic --- src/examples/zkapps/token/call-forest.ts | 24 ++++++++++++++++++++++-- src/examples/zkapps/token/merkle-list.ts | 9 +++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index fa476bf727..274d13fe8a 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -1,4 +1,4 @@ -import { AccountUpdate, Field, Hashed, Poseidon, Struct } from 'o1js'; +import { AccountUpdate, Field, Hashed, Poseidon, Provable, Struct } from 'o1js'; import { MerkleList, ProvableHashable, WithStackHash } from './merkle-list.js'; export { CallForest }; @@ -60,12 +60,32 @@ class PartialCallForest { let restOfForest = this.forest; this.pendingForests.pushIf(restOfForest.isEmpty().not(), restOfForest); - this.forest = forest; // TODO add a notion of 'current token' to partial call forest, // or as input to this method // TODO replace forest with empty forest if account update can't access current token let update = accountUpdate.unhash(); + + let currentIsEmpty = forest.isEmpty(); + + let pendingForests = this.pendingForests.clone(); + let nextForest = this.pendingForests.pop(); + let newPendingForests = this.pendingForests; + + this.forest = Provable.if( + currentIsEmpty, + CallForest.provable, + nextForest, + forest + ); + this.pendingForests = Provable.if( + currentIsEmpty, + PendingForests.provable, + newPendingForests, + pendingForests + ); + + return update; } } diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index 16e5f0390f..b79402ee9a 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -81,7 +81,7 @@ class MerkleList { }); } - pop(): T { + popExn(): T { let { previousHash, element } = this.popWitness(); let requiredHash = this.nextHash(previousHash, element); @@ -91,7 +91,7 @@ class MerkleList { return element; } - popOrDummy(): T { + pop(): T { let { previousHash, element } = this.popWitness(); let isEmpty = this.isEmpty(); @@ -104,6 +104,11 @@ class MerkleList { return Provable.if(isEmpty, provable, provable.empty(), element); } + clone(): MerkleList { + let stack = Unconstrained.witness(() => [...this.stack.get()]); + return new this.Constructor(this.hash, stack); + } + /** * Create a Merkle list type */ From ae3ac66907eb5c5eaf1805eb71f92b9039db0d10 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 12:11:17 +0100 Subject: [PATCH 11/57] finish pop account update & refactor partial call forest --- src/examples/zkapps/token/call-forest.ts | 112 ++++++++++++++--------- src/examples/zkapps/token/merkle-list.ts | 73 +++++++++------ 2 files changed, 115 insertions(+), 70 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 274d13fe8a..504618408b 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -1,7 +1,15 @@ -import { AccountUpdate, Field, Hashed, Poseidon, Provable, Struct } from 'o1js'; +import { + AccountUpdate, + Field, + Hashed, + Poseidon, + Provable, + Struct, + TokenId, +} from 'o1js'; import { MerkleList, ProvableHashable, WithStackHash } from './merkle-list.js'; -export { CallForest }; +export { CallForest, PartialCallForest }; class HashedAccountUpdate extends Hashed.create(AccountUpdate, (a) => a.hash() @@ -16,15 +24,8 @@ const CallTree: ProvableHashable = Struct({ calls: WithStackHash(), }); -class CallForest extends MerkleList.create(CallTree, nextHash) { - static empty(): CallForest { - return super.empty(); - } - static from(value: WithStackHash): CallForest { - return new this(value.hash, value.stack); - } - - static fromAccountUpdates(updates: AccountUpdate[]): CallForest { +class CallForest extends MerkleList.create(CallTree, merkleListHash) { + static fromAccountUpdates(updates: AccountUpdate[]) { let forest = CallForest.empty(); for (let update of [...updates].reverse()) { @@ -35,63 +36,90 @@ class CallForest extends MerkleList.create(CallTree, nextHash) { return forest; } - - pop() { - let { accountUpdate, calls } = super.pop(); - return { accountUpdate, calls: CallForest.from(calls) }; - } } -const PendingForests = MerkleList.create(CallForest.provable, (hash, t) => - Poseidon.hash([hash, t.hash]) -); +class ForestMayUseToken extends Struct({ + forest: CallForest.provable, + mayUseToken: AccountUpdate.MayUseToken.type, +}) {} +const PendingForests = MerkleList.create(ForestMayUseToken); + +type MayUseToken = AccountUpdate['body']['mayUseToken']; +const MayUseToken = AccountUpdate.MayUseToken; class PartialCallForest { - forest: CallForest; - pendingForests: MerkleList; + current: ForestMayUseToken; + pending: MerkleList; - constructor(forest: CallForest) { - this.forest = forest; - this.pendingForests = PendingForests.empty(); + constructor(forest: CallForest, mayUseToken: MayUseToken) { + this.current = { forest, mayUseToken }; + this.pending = PendingForests.empty(); } - popAccountUpdate() { - let { accountUpdate, calls: forest } = this.forest.pop(); - let restOfForest = this.forest; + popAccountUpdate(selfToken: Field) { + // get next account update from the current forest (might be a dummy) + let { accountUpdate, calls } = this.current.forest.pop(); + let forest = new CallForest(calls); + let restOfForest = this.current.forest; - this.pendingForests.pushIf(restOfForest.isEmpty().not(), restOfForest); + this.pending.pushIf(restOfForest.notEmpty(), { + forest: restOfForest, + mayUseToken: this.current.mayUseToken, + }); - // TODO add a notion of 'current token' to partial call forest, - // or as input to this method - // TODO replace forest with empty forest if account update can't access current token + // check if this account update / it's children can use the token let update = accountUpdate.unhash(); + let canAccessThisToken = Provable.equal( + MayUseToken.type, + update.body.mayUseToken, + this.current.mayUseToken + ); + let isSelf = TokenId.derive(update.publicKey, update.tokenId).equals( + selfToken + ); + + let usesThisToken = update.tokenId + .equals(selfToken) + .and(canAccessThisToken); + + // if we don't have to check the children, replace forest with an empty one + let checkSubtree = canAccessThisToken.and(isSelf.not()); + forest = Provable.if( + checkSubtree, + CallForest.provable, + forest, + CallForest.empty() + ); + + // if the current forest is empty, switch to the next pending forest + let current = { forest, mayUseToken: MayUseToken.InheritFromParent }; let currentIsEmpty = forest.isEmpty(); - let pendingForests = this.pendingForests.clone(); - let nextForest = this.pendingForests.pop(); - let newPendingForests = this.pendingForests; + let pendingForests = this.pending.clone(); + let next = this.pending.pop(); + let nextPendingForests = this.pending; - this.forest = Provable.if( + this.current = Provable.if( currentIsEmpty, - CallForest.provable, - nextForest, - forest + ForestMayUseToken, + next, + current ); - this.pendingForests = Provable.if( + this.pending = Provable.if( currentIsEmpty, PendingForests.provable, - newPendingForests, + nextPendingForests, pendingForests ); - return update; + return { update, usesThisToken }; } } // how to hash a forest -function nextHash(forestHash: Field, tree: CallTree) { +function merkleListHash(forestHash: Field, tree: CallTree) { return hashCons(forestHash, hashNode(tree)); } diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index b79402ee9a..be02ccfa79 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -1,6 +1,7 @@ import { Bool, Field, + Poseidon, Provable, ProvableExtended, Struct, @@ -8,25 +9,15 @@ import { assert, } from 'o1js'; import { provableFromClass } from 'src/bindings/lib/provable-snarky.js'; +import { packToFields } from 'src/lib/hash.js'; export { MerkleList, WithHash, WithStackHash, emptyHash, ProvableHashable }; -type WithHash = { previousHash: Field; element: T }; -function WithHash(type: ProvableHashable): Provable> { - return Struct({ previousHash: Field, element: type }); -} - -const emptyHash = Field(0); - -type WithStackHash = { - hash: Field; - stack: Unconstrained[]>; -}; -function WithStackHash(): ProvableExtended> { - return class extends Struct({ hash: Field, stack: Unconstrained.provable }) { - static empty(): WithStackHash { - return { hash: emptyHash, stack: Unconstrained.from([]) }; - } +function merkleListHash(provable: ProvableHashable, prefix = '') { + return function nextHash(hash: Field, value: T) { + let input = provable.toInput(value); + let packed = packToFields(input); + return Poseidon.hashWithPrefix(prefix, [hash, ...packed]); }; } @@ -34,17 +25,16 @@ class MerkleList { hash: Field; stack: Unconstrained[]>; - constructor(hash: Field, value: Unconstrained[]>) { + constructor({ hash, stack }: WithStackHash) { this.hash = hash; - this.stack = value; + this.stack = stack; } isEmpty() { return this.hash.equals(emptyHash); } - - static empty(): MerkleList { - return new this(emptyHash, Unconstrained.from([])); + notEmpty() { + return this.hash.equals(emptyHash).not(); } push(element: T) { @@ -106,7 +96,7 @@ class MerkleList { clone(): MerkleList { let stack = Unconstrained.witness(() => [...this.stack.get()]); - return new this.Constructor(this.hash, stack); + return new this.Constructor({ hash: this.hash, stack }); } /** @@ -114,8 +104,11 @@ class MerkleList { */ static create( type: ProvableHashable, - nextHash: (hash: Field, t: T) => Field - ): typeof MerkleList { + nextHash: (hash: Field, value: T) => Field = merkleListHash(type) + ): typeof MerkleList & { + empty: () => MerkleList; + provable: ProvableHashable>; + } { return class MerkleList_ extends MerkleList { static _innerProvable = type; @@ -125,6 +118,15 @@ class MerkleList { }) as ProvableHashable>; static _nextHash = nextHash; + + static empty(): MerkleList { + return new this({ hash: emptyHash, stack: Unconstrained.from([]) }); + } + + static get provable(): ProvableHashable> { + assert(this._provable !== undefined, 'MerkleList not initialized'); + return this._provable; + } }; } @@ -146,10 +148,6 @@ class MerkleList { return this.Constructor._nextHash(hash, t); } - static get provable(): ProvableHashable> { - assert(this._provable !== undefined, 'MerkleList not initialized'); - return this._provable; - } get innerProvable(): ProvableHashable { assert( this.Constructor._innerProvable !== undefined, @@ -159,6 +157,25 @@ class MerkleList { } } +type WithHash = { previousHash: Field; element: T }; +function WithHash(type: ProvableHashable): Provable> { + return Struct({ previousHash: Field, element: type }); +} + +const emptyHash = Field(0); + +type WithStackHash = { + hash: Field; + stack: Unconstrained[]>; +}; +function WithStackHash(): ProvableExtended> { + return class extends Struct({ hash: Field, stack: Unconstrained.provable }) { + static empty(): WithStackHash { + return { hash: emptyHash, stack: Unconstrained.from([]) }; + } + }; +} + type HashInput = { fields?: Field[]; packed?: [Field, number][] }; type ProvableHashable = Provable & { toInput: (x: T) => HashInput; From c6b7f6cf3ac40578a07d65426d55c64704da0d54 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 12:25:59 +0100 Subject: [PATCH 12/57] improve variable naming --- src/examples/zkapps/token/call-forest.ts | 50 ++++++++++++------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 504618408b..34a845391d 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -38,33 +38,33 @@ class CallForest extends MerkleList.create(CallTree, merkleListHash) { } } -class ForestMayUseToken extends Struct({ +class Layer extends Struct({ forest: CallForest.provable, mayUseToken: AccountUpdate.MayUseToken.type, }) {} -const PendingForests = MerkleList.create(ForestMayUseToken); +const ParentLayers = MerkleList.create(Layer); type MayUseToken = AccountUpdate['body']['mayUseToken']; const MayUseToken = AccountUpdate.MayUseToken; class PartialCallForest { - current: ForestMayUseToken; - pending: MerkleList; + currentLayer: Layer; + nonEmptyParentLayers: MerkleList; constructor(forest: CallForest, mayUseToken: MayUseToken) { - this.current = { forest, mayUseToken }; - this.pending = PendingForests.empty(); + this.currentLayer = { forest, mayUseToken }; + this.nonEmptyParentLayers = ParentLayers.empty(); } popAccountUpdate(selfToken: Field) { // get next account update from the current forest (might be a dummy) - let { accountUpdate, calls } = this.current.forest.pop(); + let { accountUpdate, calls } = this.currentLayer.forest.pop(); let forest = new CallForest(calls); - let restOfForest = this.current.forest; + let restOfForest = this.currentLayer.forest; - this.pending.pushIf(restOfForest.notEmpty(), { + this.nonEmptyParentLayers.pushIf(restOfForest.notEmpty(), { forest: restOfForest, - mayUseToken: this.current.mayUseToken, + mayUseToken: this.currentLayer.mayUseToken, }); // check if this account update / it's children can use the token @@ -73,7 +73,7 @@ class PartialCallForest { let canAccessThisToken = Provable.equal( MayUseToken.type, update.body.mayUseToken, - this.current.mayUseToken + this.currentLayer.mayUseToken ); let isSelf = TokenId.derive(update.publicKey, update.tokenId).equals( selfToken @@ -92,25 +92,27 @@ class PartialCallForest { CallForest.empty() ); - // if the current forest is empty, switch to the next pending forest - let current = { forest, mayUseToken: MayUseToken.InheritFromParent }; + // if the current forest is empty, step up to the next non-empty parent layer + // invariant: the current layer will _never_ be empty _except_ at the point where we stepped + // through the entire forest and there are no remaining parent layers + let currentLayer = { forest, mayUseToken: MayUseToken.InheritFromParent }; let currentIsEmpty = forest.isEmpty(); - let pendingForests = this.pending.clone(); - let next = this.pending.pop(); - let nextPendingForests = this.pending; + let parentLayers = this.nonEmptyParentLayers.clone(); + let nextParentLayer = this.nonEmptyParentLayers.pop(); + let parentLayersIfSteppingUp = this.nonEmptyParentLayers; - this.current = Provable.if( + this.currentLayer = Provable.if( currentIsEmpty, - ForestMayUseToken, - next, - current + Layer, + nextParentLayer, + currentLayer ); - this.pending = Provable.if( + this.nonEmptyParentLayers = Provable.if( currentIsEmpty, - PendingForests.provable, - nextPendingForests, - pendingForests + ParentLayers.provable, + parentLayersIfSteppingUp, + parentLayers ); return { update, usesThisToken }; From 6a76e43e25e992133f17d7206b9f107039746f05 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 13:49:04 +0100 Subject: [PATCH 13/57] merkle array type --- src/examples/zkapps/token/merkle-list.ts | 165 ++++++++++++++++++++++- src/lib/circuit_value.ts | 7 + src/lib/provable.ts | 2 - 3 files changed, 171 insertions(+), 3 deletions(-) diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index be02ccfa79..3e0ecf7122 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -11,7 +11,14 @@ import { import { provableFromClass } from 'src/bindings/lib/provable-snarky.js'; import { packToFields } from 'src/lib/hash.js'; -export { MerkleList, WithHash, WithStackHash, emptyHash, ProvableHashable }; +export { + MerkleArray, + MerkleList, + WithHash, + WithStackHash, + emptyHash, + ProvableHashable, +}; function merkleListHash(provable: ProvableHashable, prefix = '') { return function nextHash(hash: Field, value: T) { @@ -181,3 +188,159 @@ type ProvableHashable = Provable & { toInput: (x: T) => HashInput; empty: () => T; }; + +// merkle array + +type MerkleArrayBase = { + readonly array: Unconstrained[]>; + readonly fullHash: Field; + + currentHash: Field; + currentIndex: Unconstrained; +}; + +/** + * MerkleArray is similar to a MerkleList, but it maintains the entire array througout a computation, + * instead of needlessly mutating itself / throwing away context while stepping through it. + * + * We maintain two commitments, both of which are equivalent to a Merkle list hash starting _from the end_ of the array: + * - One to the entire array, to prove that we start iterating at the beginning. + * - One to the array from the current index until the end, to efficiently step forward. + */ +class MerkleArray implements MerkleArrayBase { + // fixed parts + readonly array: Unconstrained[]>; + readonly fullHash: Field; + + // mutable parts + currentHash: Field; + currentIndex: Unconstrained; + + constructor(value: MerkleArrayBase) { + Object.assign(this, value); + } + + isEmpty() { + return this.fullHash.equals(emptyHash); + } + isAtEnd() { + return this.currentHash.equals(emptyHash); + } + assertAtStart() { + return this.currentHash.assertEquals(this.fullHash); + } + + next() { + // next corresponds to `pop()` in MerkleList + // it returns a dummy element if we're at the end of the array + let index = Unconstrained.witness(() => this.currentIndex.get() + 1); + + let { previousHash, element } = Provable.witness( + WithHash(this.innerProvable), + () => + this.array.get()[index.get()] ?? { + previousHash: this.fullHash, + element: this.innerProvable.empty(), + } + ); + + let isDummy = this.isAtEnd(); + let correctHash = this.nextHash(previousHash, element); + let requiredHash = Provable.if(isDummy, emptyHash, correctHash); + this.currentHash.assertEquals(requiredHash); + + this.currentIndex.setTo(index); + this.currentHash = Provable.if(isDummy, emptyHash, previousHash); + + return Provable.if( + isDummy, + this.innerProvable, + this.innerProvable.empty(), + element + ); + } + + clone(): MerkleArray { + let array = Unconstrained.witness(() => [...this.array.get()]); + let currentIndex = Unconstrained.witness(() => this.currentIndex.get()); + return new this.Constructor({ + array, + fullHash: this.fullHash, + currentHash: this.currentHash, + currentIndex, + }); + } + + /** + * Create a Merkle list type + */ + static create( + type: ProvableHashable, + nextHash: (hash: Field, value: T) => Field = merkleListHash(type) + ): typeof MerkleArray & { + from: (array: T[]) => MerkleArray; + provable: ProvableHashable>; + } { + return class MerkleArray_ extends MerkleArray { + static _innerProvable = type; + + static _provable = provableFromClass(MerkleArray_, { + array: Unconstrained.provable, + fullHash: Field, + currentHash: Field, + currentIndex: Unconstrained.provable, + }) as ProvableHashable>; + + static _nextHash = nextHash; + + static from(array: T[]): MerkleArray { + let n = array.length; + let arrayWithHashes = Array>(n); + let currentHash = emptyHash; + + for (let i = n - 1; i >= 0; i--) { + arrayWithHashes[i] = { previousHash: currentHash, element: array[i] }; + currentHash = nextHash(currentHash, array[i]); + } + + return new this({ + array: Unconstrained.from(arrayWithHashes), + fullHash: currentHash, + currentHash: currentHash, + currentIndex: Unconstrained.from(0), + }); + } + + static get provable(): ProvableHashable> { + assert(this._provable !== undefined, 'MerkleArray not initialized'); + return this._provable; + } + }; + } + + // dynamic subclassing infra + static _nextHash: ((hash: Field, t: any) => Field) | undefined; + + static _provable: ProvableHashable> | undefined; + static _innerProvable: ProvableHashable | undefined; + + get Constructor() { + return this.constructor as typeof MerkleArray; + } + + nextHash(hash: Field, t: T): Field { + assert( + this.Constructor._nextHash !== undefined, + 'MerkleArray not initialized' + ); + return this.Constructor._nextHash(hash, t); + } + + get innerProvable(): ProvableHashable { + assert( + this.Constructor._innerProvable !== undefined, + 'MerkleArray not initialized' + ); + return this.Constructor._innerProvable; + } +} diff --git a/src/lib/circuit_value.ts b/src/lib/circuit_value.ts index 6720c343cb..adccc9a249 100644 --- a/src/lib/circuit_value.ts +++ b/src/lib/circuit_value.ts @@ -538,6 +538,13 @@ and Provable.asProver() blocks, which execute outside the proof. this.option = { isSome: true, value }; } + /** + * Set the unconstrained value to the same as another `Unconstrained`. + */ + setTo(value: Unconstrained) { + this.option = value.option; + } + /** * Create an `Unconstrained` with the given `value`. */ diff --git a/src/lib/provable.ts b/src/lib/provable.ts index 8094bcf89e..961f723451 100644 --- a/src/lib/provable.ts +++ b/src/lib/provable.ts @@ -341,8 +341,6 @@ function ifImplicit(condition: Bool, x: T, y: T): T { `If x, y are Structs or other custom types, you can use the following:\n` + `Provable.if(bool, MyType, x, y)` ); - // TODO remove second condition once we have consolidated field class back into one - // if (type !== y.constructor) { if (type !== y.constructor) { throw Error( 'Provable.if: Mismatched argument types. Try using an explicit type argument:\n' + From e096d3c156418372ab587c14db44e2ae6adfefea Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 14:28:00 +0100 Subject: [PATCH 14/57] make call forest a merkle array, iteration --- src/examples/zkapps/token/call-forest.ts | 72 ++++++++++--------- src/examples/zkapps/token/merkle-list.ts | 89 ++++++++++++++++++++---- 2 files changed, 109 insertions(+), 52 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 34a845391d..938cb5e1b1 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -7,7 +7,12 @@ import { Struct, TokenId, } from 'o1js'; -import { MerkleList, ProvableHashable, WithStackHash } from './merkle-list.js'; +import { + MerkleArray, + MerkleArrayBase, + MerkleList, + ProvableHashable, +} from './merkle-list.js'; export { CallForest, PartialCallForest }; @@ -17,24 +22,22 @@ class HashedAccountUpdate extends Hashed.create(AccountUpdate, (a) => type CallTree = { accountUpdate: Hashed; - calls: WithStackHash; + calls: MerkleArrayBase; }; const CallTree: ProvableHashable = Struct({ accountUpdate: HashedAccountUpdate.provable, - calls: WithStackHash(), + calls: MerkleArrayBase(), }); -class CallForest extends MerkleList.create(CallTree, merkleListHash) { - static fromAccountUpdates(updates: AccountUpdate[]) { - let forest = CallForest.empty(); - - for (let update of [...updates].reverse()) { +class CallForest extends MerkleArray.create(CallTree, merkleListHash) { + static fromAccountUpdates(updates: AccountUpdate[]): CallForest { + let nodes = updates.map((update) => { let accountUpdate = HashedAccountUpdate.hash(update); let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); - forest.push({ accountUpdate, calls }); - } + return { accountUpdate, calls }; + }); - return forest; + return CallForest.from(nodes); } } @@ -49,21 +52,21 @@ const MayUseToken = AccountUpdate.MayUseToken; class PartialCallForest { currentLayer: Layer; - nonEmptyParentLayers: MerkleList; + unfinishedParentLayers: MerkleList; constructor(forest: CallForest, mayUseToken: MayUseToken) { this.currentLayer = { forest, mayUseToken }; - this.nonEmptyParentLayers = ParentLayers.empty(); + this.unfinishedParentLayers = ParentLayers.empty(); } - popAccountUpdate(selfToken: Field) { + nextAccountUpdate(selfToken: Field) { // get next account update from the current forest (might be a dummy) - let { accountUpdate, calls } = this.currentLayer.forest.pop(); - let forest = new CallForest(calls); - let restOfForest = this.currentLayer.forest; + let { accountUpdate, calls } = this.currentLayer.forest.next(); + let forest = CallForest.startIterating(calls); + let parentLayer = this.currentLayer.forest; - this.nonEmptyParentLayers.pushIf(restOfForest.notEmpty(), { - forest: restOfForest, + this.unfinishedParentLayers.pushIf(parentLayer.isAtEnd(), { + forest: parentLayer, mayUseToken: this.currentLayer.mayUseToken, }); @@ -83,33 +86,28 @@ class PartialCallForest { .equals(selfToken) .and(canAccessThisToken); - // if we don't have to check the children, replace forest with an empty one - let checkSubtree = canAccessThisToken.and(isSelf.not()); - forest = Provable.if( - checkSubtree, - CallForest.provable, - forest, - CallForest.empty() - ); + // if we don't have to check the children, ignore the forest by jumping to its end + let skipSubtree = canAccessThisToken.not().or(isSelf); + forest.jumpToEndIf(skipSubtree); - // if the current forest is empty, step up to the next non-empty parent layer - // invariant: the current layer will _never_ be empty _except_ at the point where we stepped - // through the entire forest and there are no remaining parent layers + // if we're at the end of the current layer, step up to the next unfinished parent layer + // invariant: the new current layer will _never_ be finished _except_ at the point where we stepped + // through the entire forest and there are no remaining parent layers to finish let currentLayer = { forest, mayUseToken: MayUseToken.InheritFromParent }; - let currentIsEmpty = forest.isEmpty(); + let currentIsFinished = forest.isAtEnd(); - let parentLayers = this.nonEmptyParentLayers.clone(); - let nextParentLayer = this.nonEmptyParentLayers.pop(); - let parentLayersIfSteppingUp = this.nonEmptyParentLayers; + let parentLayers = this.unfinishedParentLayers.clone(); + let nextParentLayer = this.unfinishedParentLayers.pop(); + let parentLayersIfSteppingUp = this.unfinishedParentLayers; this.currentLayer = Provable.if( - currentIsEmpty, + currentIsFinished, Layer, nextParentLayer, currentLayer ); - this.nonEmptyParentLayers = Provable.if( - currentIsEmpty, + this.unfinishedParentLayers = Provable.if( + currentIsFinished, ParentLayers.provable, parentLayersIfSteppingUp, parentLayers diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/examples/zkapps/token/merkle-list.ts index 3e0ecf7122..a82c4f0e56 100644 --- a/src/examples/zkapps/token/merkle-list.ts +++ b/src/examples/zkapps/token/merkle-list.ts @@ -3,7 +3,6 @@ import { Field, Poseidon, Provable, - ProvableExtended, Struct, Unconstrained, assert, @@ -13,6 +12,8 @@ import { packToFields } from 'src/lib/hash.js'; export { MerkleArray, + MerkleArrayIterator, + MerkleArrayBase, MerkleList, WithHash, WithStackHash, @@ -175,7 +176,7 @@ type WithStackHash = { hash: Field; stack: Unconstrained[]>; }; -function WithStackHash(): ProvableExtended> { +function WithStackHash(): ProvableHashable> { return class extends Struct({ hash: Field, stack: Unconstrained.provable }) { static empty(): WithStackHash { return { hash: emptyHash, stack: Unconstrained.from([]) }; @@ -193,12 +194,43 @@ type ProvableHashable = Provable & { type MerkleArrayBase = { readonly array: Unconstrained[]>; - readonly fullHash: Field; + readonly hash: Field; +}; + +function MerkleArrayBase(): ProvableHashable> { + return class extends Struct({ array: Unconstrained.provable, hash: Field }) { + static empty(): MerkleArrayBase { + return { array: Unconstrained.from([]), hash: emptyHash }; + } + }; +} + +type MerkleArrayIterator = { + readonly array: Unconstrained[]>; + readonly hash: Field; currentHash: Field; currentIndex: Unconstrained; }; +function MerkleArrayIterator(): ProvableHashable> { + return class extends Struct({ + array: Unconstrained.provable, + hash: Field, + currentHash: Field, + currentIndex: Unconstrained.provable, + }) { + static empty(): MerkleArrayIterator { + return { + array: Unconstrained.from([]), + hash: emptyHash, + currentHash: emptyHash, + currentIndex: Unconstrained.from(0), + }; + } + }; +} + /** * MerkleArray is similar to a MerkleList, but it maintains the entire array througout a computation, * instead of needlessly mutating itself / throwing away context while stepping through it. @@ -207,27 +239,47 @@ type MerkleArrayBase = { * - One to the entire array, to prove that we start iterating at the beginning. * - One to the array from the current index until the end, to efficiently step forward. */ -class MerkleArray implements MerkleArrayBase { +class MerkleArray implements MerkleArrayIterator { // fixed parts readonly array: Unconstrained[]>; - readonly fullHash: Field; + readonly hash: Field; // mutable parts currentHash: Field; currentIndex: Unconstrained; - constructor(value: MerkleArrayBase) { + constructor(value: MerkleArrayIterator) { Object.assign(this, value); } - isEmpty() { - return this.fullHash.equals(emptyHash); + static startIterating({ array, hash }: MerkleArrayBase) { + return new this({ + array, + hash, + currentHash: hash, + currentIndex: Unconstrained.from(0), + }); + } + assertAtStart() { + return this.currentHash.assertEquals(this.hash); } + isAtEnd() { return this.currentHash.equals(emptyHash); } - assertAtStart() { - return this.currentHash.assertEquals(this.fullHash); + jumpToEnd() { + this.currentIndex.setTo( + Unconstrained.witness(() => this.array.get().length) + ); + this.currentHash = emptyHash; + } + jumpToEndIf(condition: Bool) { + Provable.asProver(() => { + if (condition.toBoolean()) { + this.currentIndex.set(this.array.get().length); + } + }); + this.currentHash = Provable.if(condition, emptyHash, this.currentHash); } next() { @@ -239,7 +291,7 @@ class MerkleArray implements MerkleArrayBase { WithHash(this.innerProvable), () => this.array.get()[index.get()] ?? { - previousHash: this.fullHash, + previousHash: this.hash, element: this.innerProvable.empty(), } ); @@ -265,7 +317,7 @@ class MerkleArray implements MerkleArrayBase { let currentIndex = Unconstrained.witness(() => this.currentIndex.get()); return new this.Constructor({ array, - fullHash: this.fullHash, + hash: this.hash, currentHash: this.currentHash, currentIndex, }); @@ -279,6 +331,7 @@ class MerkleArray implements MerkleArrayBase { nextHash: (hash: Field, value: T) => Field = merkleListHash(type) ): typeof MerkleArray & { from: (array: T[]) => MerkleArray; + empty: () => MerkleArray; provable: ProvableHashable>; } { return class MerkleArray_ extends MerkleArray { @@ -286,10 +339,12 @@ class MerkleArray implements MerkleArrayBase { static _provable = provableFromClass(MerkleArray_, { array: Unconstrained.provable, - fullHash: Field, + hash: Field, currentHash: Field, currentIndex: Unconstrained.provable, - }) as ProvableHashable>; + }) satisfies ProvableHashable> as ProvableHashable< + MerkleArray + >; static _nextHash = nextHash; @@ -305,12 +360,16 @@ class MerkleArray implements MerkleArrayBase { return new this({ array: Unconstrained.from(arrayWithHashes), - fullHash: currentHash, + hash: currentHash, currentHash: currentHash, currentIndex: Unconstrained.from(0), }); } + static empty(): MerkleArray { + return this.from([]); + } + static get provable(): ProvableHashable> { assert(this._provable !== undefined, 'MerkleArray not initialized'); return this._provable; From fa64041ea49bd9ea9db930e933c9ac5bc891c214 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 16:11:06 +0100 Subject: [PATCH 15/57] tweaks, doccomments --- src/examples/zkapps/token/call-forest.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 938cb5e1b1..84e6320a22 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -59,14 +59,24 @@ class PartialCallForest { this.unfinishedParentLayers = ParentLayers.empty(); } + /** + * Make a single step through a tree of account updates. + * + * This function will visit each account update in the tree exactly once when called repeatedly, + * and the internal state of `PartialCallForest` represents the work still to be done. + * + * Makes a best effort to avoid visiting account updates that are not using the token and in particular, to avoid returning dummy updates + * -- but both can't be ruled out, so we're returning { update, usesThisToken } and let the caller handle the irrelevant case. + */ nextAccountUpdate(selfToken: Field) { // get next account update from the current forest (might be a dummy) + // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); let forest = CallForest.startIterating(calls); - let parentLayer = this.currentLayer.forest; + let parentForest = this.currentLayer.forest; - this.unfinishedParentLayers.pushIf(parentLayer.isAtEnd(), { - forest: parentLayer, + this.unfinishedParentLayers.pushIf(parentForest.isAtEnd().not(), { + forest: parentForest, mayUseToken: this.currentLayer.mayUseToken, }); From 7576435e2a1edbdf07642c0ad0dff2ad6e03d382 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 16:16:07 +0100 Subject: [PATCH 16/57] improve comment --- src/examples/zkapps/token/call-forest.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/examples/zkapps/token/call-forest.ts index 84e6320a22..090ed664dd 100644 --- a/src/examples/zkapps/token/call-forest.ts +++ b/src/examples/zkapps/token/call-forest.ts @@ -60,13 +60,18 @@ class PartialCallForest { } /** - * Make a single step through a tree of account updates. + * Make a single step along a tree of account updates. * - * This function will visit each account update in the tree exactly once when called repeatedly, - * and the internal state of `PartialCallForest` represents the work still to be done. + * This function is guaranteed to visit each account update in the tree that uses the token + * exactly once, when called repeatedly. * - * Makes a best effort to avoid visiting account updates that are not using the token and in particular, to avoid returning dummy updates - * -- but both can't be ruled out, so we're returning { update, usesThisToken } and let the caller handle the irrelevant case. + * The internal state of `PartialCallForest` represents the work still to be done, and + * can be passed from one proof to the next. + * + * The method makes a best effort to avoid visiting account updates that are not using the token, + * and in particular, to avoid returning dummy updates. + * However, neither can be ruled out. We're returning { update, usesThisToken: Bool } and let the + * caller handle the irrelevant case where `usesThisToken` is false. */ nextAccountUpdate(selfToken: Field) { // get next account update from the current forest (might be a dummy) From 482007d806c3842994e17fb2ade00cc0f3994299 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 16:31:09 +0100 Subject: [PATCH 17/57] move files for now --- src/{examples/zkapps => lib/mina}/token/call-forest.ts | 0 src/{examples/zkapps => lib/mina}/token/merkle-list.ts | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/{examples/zkapps => lib/mina}/token/call-forest.ts (100%) rename src/{examples/zkapps => lib/mina}/token/merkle-list.ts (100%) diff --git a/src/examples/zkapps/token/call-forest.ts b/src/lib/mina/token/call-forest.ts similarity index 100% rename from src/examples/zkapps/token/call-forest.ts rename to src/lib/mina/token/call-forest.ts diff --git a/src/examples/zkapps/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts similarity index 100% rename from src/examples/zkapps/token/merkle-list.ts rename to src/lib/mina/token/merkle-list.ts From f679c156697a37c63e5493e090f567aaeefb43d9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 17:14:55 +0100 Subject: [PATCH 18/57] start creating test --- src/lib/mina/token/call-forest.ts | 17 +++++++-- src/lib/mina/token/call-forest.unit-test.ts | 40 +++++++++++++++++++++ src/lib/mina/token/merkle-list.ts | 15 ++++++-- src/lib/testing/random.ts | 3 +- 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 src/lib/mina/token/call-forest.unit-test.ts diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 090ed664dd..d5e00897ae 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -1,3 +1,4 @@ +import { prefixes } from '../../../provable/poseidon-bigint.js'; import { AccountUpdate, Field, @@ -6,18 +7,20 @@ import { Provable, Struct, TokenId, -} from 'o1js'; +} from '../../../index.js'; import { MerkleArray, MerkleArrayBase, MerkleList, ProvableHashable, + genericHash, } from './merkle-list.js'; export { CallForest, PartialCallForest }; -class HashedAccountUpdate extends Hashed.create(AccountUpdate, (a) => - a.hash() +class HashedAccountUpdate extends Hashed.create( + AccountUpdate, + hashAccountUpdate ) {} type CallTree = { @@ -59,6 +62,10 @@ class PartialCallForest { this.unfinishedParentLayers = ParentLayers.empty(); } + static create(forest: CallForest) { + return new PartialCallForest(forest, MayUseToken.ParentsOwnToken); + } + /** * Make a single step along a tree of account updates. * @@ -150,3 +157,7 @@ function hashCons(forestHash: Field, nodeHash: Field) { nodeHash, ]); } + +function hashAccountUpdate(update: AccountUpdate) { + return genericHash(AccountUpdate, prefixes.body, update); +} diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts new file mode 100644 index 0000000000..73521b60a9 --- /dev/null +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -0,0 +1,40 @@ +import { Random, test } from '../../testing/property.js'; +import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; +import { CallForest, PartialCallForest } from './call-forest.js'; +import { AccountUpdate, ZkappCommand } from '../../account_update.js'; +import { TypesBigint } from '../../../bindings/mina-transaction/types.js'; +import { Pickles } from '../../../snarky.js'; +import { Field } from '../../../lib/core.js'; + +// rng for account updates + +let [, data, hashMl] = Pickles.dummyVerificationKey(); +let verificationKey = { data, hash: Field(hashMl) }; + +const accountUpdates = Random.map( + RandomTransaction.zkappCommand, + (txBigint) => { + // bigint to json, then to provable + let txJson = TypesBigint.ZkappCommand.toJSON(txBigint); + + let accountUpdates = txJson.accountUpdates.map((aJson) => { + let a = AccountUpdate.fromJSON(aJson); + + // fix verification key + if (a.body.update.verificationKey.isSome) { + a.body.update.verificationKey.value = verificationKey; + } + return a; + }); + + return accountUpdates; + } +); + +// tests begin here + +test.custom({ timeBudget: 10000 })(accountUpdates, (updates) => { + console.log({ length: updates.length }); + + CallForest.fromAccountUpdates(updates); +}); diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index a82c4f0e56..edd1577079 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -7,8 +7,8 @@ import { Unconstrained, assert, } from 'o1js'; -import { provableFromClass } from 'src/bindings/lib/provable-snarky.js'; -import { packToFields } from 'src/lib/hash.js'; +import { provableFromClass } from '../../../bindings/lib/provable-snarky.js'; +import { packToFields } from '../../hash.js'; export { MerkleArray, @@ -19,8 +19,19 @@ export { WithStackHash, emptyHash, ProvableHashable, + genericHash, }; +function genericHash( + provable: ProvableHashable, + prefix: string, + value: T +) { + let input = provable.toInput(value); + let packed = packToFields(input); + return Poseidon.hashWithPrefix(prefix, packed); +} + function merkleListHash(provable: ProvableHashable, prefix = '') { return function nextHash(hash: Field, value: T) { let input = provable.toInput(value); diff --git a/src/lib/testing/random.ts b/src/lib/testing/random.ts index 358b0ac206..12b82802ff 100644 --- a/src/lib/testing/random.ts +++ b/src/lib/testing/random.ts @@ -162,7 +162,8 @@ const accountUpdate = mapWithInvalid( a.body.authorizationKind.isProved = Bool(false); } if (!a.body.authorizationKind.isProved) { - a.body.authorizationKind.verificationKeyHash = Field(0); + a.body.authorizationKind.verificationKeyHash = + VerificationKeyHash.empty(); } // ensure mayUseToken is valid let { inheritFromParent, parentsOwnToken } = a.body.mayUseToken; From 342705f91f19667eebec631d3b306438ad9b70e7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 17:39:58 +0100 Subject: [PATCH 19/57] compute stack hash that is equivalent between provable & mins-aigner, but not yet the new implementation --- src/lib/mina/token/call-forest.unit-test.ts | 73 +++++++++++++++++---- src/mina-signer/src/sign-zkapp-command.ts | 1 + 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 73521b60a9..9387ff8ba6 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -1,40 +1,85 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; import { CallForest, PartialCallForest } from './call-forest.js'; -import { AccountUpdate, ZkappCommand } from '../../account_update.js'; +import { + AccountUpdate, + CallForest as ProvableCallForest, +} from '../../account_update.js'; import { TypesBigint } from '../../../bindings/mina-transaction/types.js'; import { Pickles } from '../../../snarky.js'; -import { Field } from '../../../lib/core.js'; +import { + accountUpdatesToCallForest, + callForestHash, + CallForest as SimpleCallForest, +} from '../../../mina-signer/src/sign-zkapp-command.js'; // rng for account updates let [, data, hashMl] = Pickles.dummyVerificationKey(); -let verificationKey = { data, hash: Field(hashMl) }; +let verificationKey = { data, hash: hashMl[1] }; -const accountUpdates = Random.map( +const callForest: Random = Random.map( RandomTransaction.zkappCommand, (txBigint) => { - // bigint to json, then to provable - let txJson = TypesBigint.ZkappCommand.toJSON(txBigint); - - let accountUpdates = txJson.accountUpdates.map((aJson) => { - let a = AccountUpdate.fromJSON(aJson); - + let flatUpdates = txBigint.accountUpdates.map((a) => { // fix verification key if (a.body.update.verificationKey.isSome) { a.body.update.verificationKey.value = verificationKey; } return a; }); - - return accountUpdates; + return accountUpdatesToCallForest(flatUpdates); } ); // tests begin here -test.custom({ timeBudget: 10000 })(accountUpdates, (updates) => { +test.custom({ timeBudget: 10000 })(callForest, (forestBigint) => { + // reference: bigint callforest hash from mina-signer + let stackHash = callForestHash(forestBigint); + + let updates = callForestToNestedArray( + mapCallForest(forestBigint, accountUpdateFromBigint) + ); + console.log({ length: updates.length }); - CallForest.fromAccountUpdates(updates); + let dummyParent = AccountUpdate.dummy(); + dummyParent.children.accountUpdates = updates; + + let hash = ProvableCallForest.hashChildren(dummyParent); + hash.assertEquals(stackHash); + + let forest = CallForest.fromAccountUpdates(updates); }); + +// call forest helpers + +type AbstractSimpleCallForest = { + accountUpdate: A; + children: AbstractSimpleCallForest; +}[]; + +function mapCallForest( + forest: AbstractSimpleCallForest, + mapOne: (a: A) => B +): AbstractSimpleCallForest { + return forest.map(({ accountUpdate, children }) => ({ + accountUpdate: mapOne(accountUpdate), + children: mapCallForest(children, mapOne), + })); +} + +function accountUpdateFromBigint(a: TypesBigint.AccountUpdate): AccountUpdate { + // bigint to json, then to provable + return AccountUpdate.fromJSON(TypesBigint.AccountUpdate.toJSON(a)); +} + +function callForestToNestedArray( + forest: AbstractSimpleCallForest +): AccountUpdate[] { + return forest.map(({ accountUpdate, children }) => { + accountUpdate.children.accountUpdates = callForestToNestedArray(children); + return accountUpdate; + }); +} diff --git a/src/mina-signer/src/sign-zkapp-command.ts b/src/mina-signer/src/sign-zkapp-command.ts index 7644c2ee32..4daea29443 100644 --- a/src/mina-signer/src/sign-zkapp-command.ts +++ b/src/mina-signer/src/sign-zkapp-command.ts @@ -33,6 +33,7 @@ export { createFeePayer, accountUpdateFromFeePayer, isCallDepthValid, + CallForest, }; function signZkappCommand( From 895dd71cd9694d2a4e7a80201dae71caccc01716 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 17:48:00 +0100 Subject: [PATCH 20/57] test on wider/deeper trees --- src/lib/mina/token/call-forest.unit-test.ts | 13 ++++++++++--- src/mina-signer/src/random-transaction.ts | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 9387ff8ba6..2d06732feb 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -18,16 +18,23 @@ import { let [, data, hashMl] = Pickles.dummyVerificationKey(); let verificationKey = { data, hash: hashMl[1] }; +let accountUpdates = Random.array( + RandomTransaction.accountUpdateWithCallDepth, + Random.int(0, 50), + { reset: true } +); + const callForest: Random = Random.map( - RandomTransaction.zkappCommand, - (txBigint) => { - let flatUpdates = txBigint.accountUpdates.map((a) => { + accountUpdates, + (accountUpdates) => { + let flatUpdates = accountUpdates.map((a) => { // fix verification key if (a.body.update.verificationKey.isSome) { a.body.update.verificationKey.value = verificationKey; } return a; }); + console.log({ totalLength: flatUpdates.length }); return accountUpdatesToCallForest(flatUpdates); } ); diff --git a/src/mina-signer/src/random-transaction.ts b/src/mina-signer/src/random-transaction.ts index 8b58405ae3..f75c7ae93b 100644 --- a/src/mina-signer/src/random-transaction.ts +++ b/src/mina-signer/src/random-transaction.ts @@ -142,4 +142,5 @@ const RandomTransaction = { zkappCommandAndFeePayerKey, zkappCommandJson, networkId: Random.oneOf('testnet', 'mainnet'), + accountUpdateWithCallDepth: accountUpdate, }; From 7372293b81fc4638e5bbb0b4bdf6ee937908002a Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 23 Jan 2024 21:38:10 +0100 Subject: [PATCH 21/57] debugging --- src/lib/mina/token/call-forest.ts | 2 ++ src/lib/mina/token/call-forest.unit-test.ts | 40 +++++++++++++-------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index d5e00897ae..6354199c37 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -18,6 +18,8 @@ import { export { CallForest, PartialCallForest }; +export { HashedAccountUpdate }; + class HashedAccountUpdate extends Hashed.create( AccountUpdate, hashAccountUpdate diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 2d06732feb..538f101344 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -1,6 +1,6 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; -import { CallForest, PartialCallForest } from './call-forest.js'; +import { CallForest, HashedAccountUpdate } from './call-forest.js'; import { AccountUpdate, CallForest as ProvableCallForest, @@ -41,24 +41,36 @@ const callForest: Random = Random.map( // tests begin here -test.custom({ timeBudget: 10000 })(callForest, (forestBigint) => { - // reference: bigint callforest hash from mina-signer - let stackHash = callForestHash(forestBigint); +test.custom({ timeBudget: 10000, logFailures: false })( + callForest, + (forestBigint) => { + // reference: bigint callforest hash from mina-signer + let stackHash = callForestHash(forestBigint); - let updates = callForestToNestedArray( - mapCallForest(forestBigint, accountUpdateFromBigint) - ); + let updates = callForestToNestedArray( + mapCallForest(forestBigint, accountUpdateFromBigint) + ); - console.log({ length: updates.length }); + console.log({ length: updates.length }); - let dummyParent = AccountUpdate.dummy(); - dummyParent.children.accountUpdates = updates; + let dummyParent = AccountUpdate.dummy(); + dummyParent.children.accountUpdates = updates; - let hash = ProvableCallForest.hashChildren(dummyParent); - hash.assertEquals(stackHash); + let hash = ProvableCallForest.hashChildren(dummyParent); + hash.assertEquals(stackHash); - let forest = CallForest.fromAccountUpdates(updates); -}); + let nodes = updates.map((update) => { + let accountUpdate = HashedAccountUpdate.hash(update); + let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); + return { accountUpdate, calls }; + }); + + console.log({ nodes: nodes.map((n) => n.calls.hash.toBigInt()) }); + + let forest = CallForest.fromAccountUpdates(updates); + forest.hash.assertEquals(stackHash); + } +); // call forest helpers From 12d7449e97eb5a1647835a09610c62e1c8d210a6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 10:12:13 +0100 Subject: [PATCH 22/57] fix test by fixing order in hashCons --- src/lib/mina/token/call-forest.ts | 6 ++--- src/lib/mina/token/call-forest.unit-test.ts | 27 ++++++--------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 6354199c37..9a4a540388 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -148,15 +148,15 @@ function merkleListHash(forestHash: Field, tree: CallTree) { } function hashNode(tree: CallTree) { - return Poseidon.hashWithPrefix('MinaAcctUpdateNode**', [ + return Poseidon.hashWithPrefix(prefixes.accountUpdateNode, [ tree.accountUpdate.hash, tree.calls.hash, ]); } function hashCons(forestHash: Field, nodeHash: Field) { - return Poseidon.hashWithPrefix('MinaAcctUpdateCons**', [ - forestHash, + return Poseidon.hashWithPrefix(prefixes.accountUpdateCons, [ nodeHash, + forestHash, ]); } diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 538f101344..782a4e7d9e 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -39,40 +39,27 @@ const callForest: Random = Random.map( } ); -// tests begin here +// TESTS + +// correctly hashes a call forest test.custom({ timeBudget: 10000, logFailures: false })( callForest, (forestBigint) => { // reference: bigint callforest hash from mina-signer - let stackHash = callForestHash(forestBigint); + let expectedHash = callForestHash(forestBigint); + // convert to o1js-style list of nested `AccountUpdate`s let updates = callForestToNestedArray( mapCallForest(forestBigint, accountUpdateFromBigint) ); - console.log({ length: updates.length }); - - let dummyParent = AccountUpdate.dummy(); - dummyParent.children.accountUpdates = updates; - - let hash = ProvableCallForest.hashChildren(dummyParent); - hash.assertEquals(stackHash); - - let nodes = updates.map((update) => { - let accountUpdate = HashedAccountUpdate.hash(update); - let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); - return { accountUpdate, calls }; - }); - - console.log({ nodes: nodes.map((n) => n.calls.hash.toBigInt()) }); - let forest = CallForest.fromAccountUpdates(updates); - forest.hash.assertEquals(stackHash); + forest.hash.assertEquals(expectedHash); } ); -// call forest helpers +// HELPERS type AbstractSimpleCallForest = { accountUpdate: A; From 4c4ce8e3bc4fdcceb35a68553ffb7daf38d607c2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 12:08:01 +0100 Subject: [PATCH 23/57] fix hash.empty() and merkleArray.next() --- src/lib/mina/token/merkle-list.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index edd1577079..b5cca46e5a 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -268,7 +268,7 @@ class MerkleArray implements MerkleArrayIterator { array, hash, currentHash: hash, - currentIndex: Unconstrained.from(0), + currentIndex: Unconstrained.from(-1), }); } assertAtStart() { @@ -302,7 +302,7 @@ class MerkleArray implements MerkleArrayIterator { WithHash(this.innerProvable), () => this.array.get()[index.get()] ?? { - previousHash: this.hash, + previousHash: emptyHash, element: this.innerProvable.empty(), } ); @@ -372,8 +372,8 @@ class MerkleArray implements MerkleArrayIterator { return new this({ array: Unconstrained.from(arrayWithHashes), hash: currentHash, - currentHash: currentHash, - currentIndex: Unconstrained.from(0), + currentHash, + currentIndex: Unconstrained.from(-1), }); } From 737fc24cdc29e29d86a16838962c05319c8a63f7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 12:08:15 +0100 Subject: [PATCH 24/57] make some code generic for reuse --- src/mina-signer/src/sign-zkapp-command.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/mina-signer/src/sign-zkapp-command.ts b/src/mina-signer/src/sign-zkapp-command.ts index 4daea29443..004a3f0940 100644 --- a/src/mina-signer/src/sign-zkapp-command.ts +++ b/src/mina-signer/src/sign-zkapp-command.ts @@ -123,16 +123,22 @@ function transactionCommitments(zkappCommand: ZkappCommand) { return { commitment, fullCommitment }; } -type CallTree = { accountUpdate: AccountUpdate; children: CallForest }; -type CallForest = CallTree[]; +type CallTree = { + accountUpdate: AccountUpdate; + children: CallForest; +}; +type CallForest = CallTree[]; /** * Turn flat list into a hierarchical structure (forest) by letting the callDepth * determine parent-child relationships */ -function accountUpdatesToCallForest(updates: AccountUpdate[], callDepth = 0) { +function accountUpdatesToCallForest( + updates: A[], + callDepth = 0 +) { let remainingUpdates = callDepth > 0 ? updates : [...updates]; - let forest: CallForest = []; + let forest: CallForest = []; while (remainingUpdates.length > 0) { let accountUpdate = remainingUpdates[0]; if (accountUpdate.body.callDepth < callDepth) return forest; @@ -150,7 +156,7 @@ function accountUpdateHash(update: AccountUpdate) { return hashWithPrefix(prefixes.body, fields); } -function callForestHash(forest: CallForest): Field { +function callForestHash(forest: CallForest): Field { let stackHash = 0n; for (let callTree of [...forest].reverse()) { let calls = callForestHash(callTree.children); From fb8b3fa1e0a18c3840028555f0a0edbb816a77bc Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 12:42:32 +0100 Subject: [PATCH 25/57] refactor, start test that traverses the tree --- src/lib/mina/token/call-forest.ts | 22 ++-- src/lib/mina/token/call-forest.unit-test.ts | 111 +++++++++++++------- 2 files changed, 85 insertions(+), 48 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 9a4a540388..a60a93ea1b 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -16,7 +16,7 @@ import { genericHash, } from './merkle-list.js'; -export { CallForest, PartialCallForest }; +export { CallForest, PartialCallForest, hashAccountUpdate }; export { HashedAccountUpdate }; @@ -58,14 +58,20 @@ const MayUseToken = AccountUpdate.MayUseToken; class PartialCallForest { currentLayer: Layer; unfinishedParentLayers: MerkleList; + selfToken: Field; - constructor(forest: CallForest, mayUseToken: MayUseToken) { + constructor(forest: CallForest, mayUseToken: MayUseToken, selfToken: Field) { this.currentLayer = { forest, mayUseToken }; this.unfinishedParentLayers = ParentLayers.empty(); + this.selfToken = selfToken; } - static create(forest: CallForest) { - return new PartialCallForest(forest, MayUseToken.ParentsOwnToken); + static create(forest: CallForest, selfToken: Field) { + return new PartialCallForest( + forest, + MayUseToken.ParentsOwnToken, + selfToken + ); } /** @@ -82,7 +88,7 @@ class PartialCallForest { * However, neither can be ruled out. We're returning { update, usesThisToken: Bool } and let the * caller handle the irrelevant case where `usesThisToken` is false. */ - nextAccountUpdate(selfToken: Field) { + next() { // get next account update from the current forest (might be a dummy) // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); @@ -103,11 +109,11 @@ class PartialCallForest { this.currentLayer.mayUseToken ); let isSelf = TokenId.derive(update.publicKey, update.tokenId).equals( - selfToken + this.selfToken ); let usesThisToken = update.tokenId - .equals(selfToken) + .equals(this.selfToken) .and(canAccessThisToken); // if we don't have to check the children, ignore the forest by jumping to its end @@ -137,7 +143,7 @@ class PartialCallForest { parentLayers ); - return { update, usesThisToken }; + return { accountUpdate: update, usesThisToken }; } } diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 782a4e7d9e..0d3d792bf9 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -1,9 +1,15 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; -import { CallForest, HashedAccountUpdate } from './call-forest.js'; +import { + CallForest, + HashedAccountUpdate, + PartialCallForest, + hashAccountUpdate, +} from './call-forest.js'; import { AccountUpdate, CallForest as ProvableCallForest, + TokenId, } from '../../account_update.js'; import { TypesBigint } from '../../../bindings/mina-transaction/types.js'; import { Pickles } from '../../../snarky.js'; @@ -12,46 +18,47 @@ import { callForestHash, CallForest as SimpleCallForest, } from '../../../mina-signer/src/sign-zkapp-command.js'; +import assert from 'assert'; +import { Field } from '../../field.js'; -// rng for account updates +// RANDOM NUMBER GENERATORS for account updates let [, data, hashMl] = Pickles.dummyVerificationKey(); -let verificationKey = { data, hash: hashMl[1] }; +let dummyVerificationKey = { data, hash: hashMl[1] }; -let accountUpdates = Random.array( +const accountUpdateBigint = Random.map( RandomTransaction.accountUpdateWithCallDepth, - Random.int(0, 50), - { reset: true } -); - -const callForest: Random = Random.map( - accountUpdates, - (accountUpdates) => { - let flatUpdates = accountUpdates.map((a) => { - // fix verification key - if (a.body.update.verificationKey.isSome) { - a.body.update.verificationKey.value = verificationKey; - } - return a; - }); - console.log({ totalLength: flatUpdates.length }); - return accountUpdatesToCallForest(flatUpdates); + (a) => { + // fix verification key + if (a.body.update.verificationKey.isSome) + a.body.update.verificationKey.value = dummyVerificationKey; + return a; } ); +const accountUpdate = Random.map(accountUpdateBigint, accountUpdateFromBigint); + +const arrayOf = (x: Random) => + // `reset: true` to start callDepth at 0 for each array + Random.array(x, Random.int(20, 50), { reset: true }); + +const flatAccountUpdatesBigint = arrayOf(accountUpdateBigint); +const flatAccountUpdates = arrayOf(accountUpdate); // TESTS // correctly hashes a call forest -test.custom({ timeBudget: 10000, logFailures: false })( - callForest, - (forestBigint) => { +test.custom({ timeBudget: 1000, logFailures: false })( + flatAccountUpdatesBigint, + (flatUpdatesBigint) => { // reference: bigint callforest hash from mina-signer + let forestBigint = accountUpdatesToCallForest(flatUpdatesBigint); let expectedHash = callForestHash(forestBigint); // convert to o1js-style list of nested `AccountUpdate`s + let flatUpdates = flatUpdatesBigint.map(accountUpdateFromBigint); let updates = callForestToNestedArray( - mapCallForest(forestBigint, accountUpdateFromBigint) + accountUpdatesToCallForest(flatUpdates) ); let forest = CallForest.fromAccountUpdates(updates); @@ -59,22 +66,46 @@ test.custom({ timeBudget: 10000, logFailures: false })( } ); -// HELPERS +// traverses a call forest in correct depth-first order -type AbstractSimpleCallForest = { - accountUpdate: A; - children: AbstractSimpleCallForest; -}[]; - -function mapCallForest( - forest: AbstractSimpleCallForest, - mapOne: (a: A) => B -): AbstractSimpleCallForest { - return forest.map(({ accountUpdate, children }) => ({ - accountUpdate: mapOne(accountUpdate), - children: mapCallForest(children, mapOne), - })); -} +test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( + flatAccountUpdates, + (flatUpdates) => { + // convert to o1js-style list of nested `AccountUpdate`s + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + + let forest = CallForest.fromAccountUpdates(updates); + let tokenId = Field.random(); + let partialForest = PartialCallForest.create(forest, tokenId); + + let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + + let n = flatUpdates.length; + for (let i = 0; i < n; i++) { + assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); + + let expected = flatUpdates[i]; + let actual = partialForest.next().accountUpdate; + + console.log( + 'expected: ', + expected.body.callDepth, + expected.body.publicKey.toBase58(), + hashAccountUpdate(expected).toBigInt() + ); + console.log( + 'actual: ', + actual.body.callDepth, + actual.body.publicKey.toBase58(), + hashAccountUpdate(actual).toBigInt() + ); + } + } +); + +// HELPERS function accountUpdateFromBigint(a: TypesBigint.AccountUpdate): AccountUpdate { // bigint to json, then to provable @@ -82,7 +113,7 @@ function accountUpdateFromBigint(a: TypesBigint.AccountUpdate): AccountUpdate { } function callForestToNestedArray( - forest: AbstractSimpleCallForest + forest: SimpleCallForest ): AccountUpdate[] { return forest.map(({ accountUpdate, children }) => { accountUpdate.children.accountUpdates = callForestToNestedArray(children); From e1adaaf223ff0c2b6b6b78a63470a84cf0d09abe Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 13:07:08 +0100 Subject: [PATCH 26/57] confirm that callforest.next() works --- src/lib/mina/token/call-forest.unit-test.ts | 63 ++++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 0d3d792bf9..291d84660d 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -39,7 +39,7 @@ const accountUpdate = Random.map(accountUpdateBigint, accountUpdateFromBigint); const arrayOf = (x: Random) => // `reset: true` to start callDepth at 0 for each array - Random.array(x, Random.int(20, 50), { reset: true }); + Random.array(x, 10, { reset: true }); const flatAccountUpdatesBigint = arrayOf(accountUpdateBigint); const flatAccountUpdates = arrayOf(accountUpdate); @@ -66,6 +66,31 @@ test.custom({ timeBudget: 1000, logFailures: false })( } ); +// traverses the top level of a call forest in correct order +// i.e., CallForest.next() works + +test.custom({ timeBudget: 10000, logFailures: false })( + flatAccountUpdates, + (flatUpdates) => { + // convert to o1js-style list of nested `AccountUpdate`s + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + let forest = CallForest.fromAccountUpdates(updates); + + let n = updates.length; + for (let i = 0; i < n; i++) { + let expected = updates[i]; + let actual = forest.next().accountUpdate.value.get(); + assertEqual(actual, expected); + } + + // doing next again should return a dummy + let actual = forest.next().accountUpdate.value.get(); + assertEqual(actual, AccountUpdate.dummy()); + } +); + // traverses a call forest in correct depth-first order test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( @@ -76,6 +101,10 @@ test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( accountUpdatesToCallForest(flatUpdates) ); + let dummyParent = AccountUpdate.dummy(); + dummyParent.children.accountUpdates = updates; + console.log(dummyParent.toPrettyLayout()); + let forest = CallForest.fromAccountUpdates(updates); let tokenId = Field.random(); let partialForest = PartialCallForest.create(forest, tokenId); @@ -87,20 +116,31 @@ test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); let expected = flatUpdates[i]; + let expectedHash = hashAccountUpdate(expected).toBigInt(); let actual = partialForest.next().accountUpdate; + let actualHash = hashAccountUpdate(actual).toBigInt(); + + let isCorrect = String(expectedHash === actualHash).padStart(5, ' '); + console.log( + 'actual: ', + actual.body.callDepth, + isCorrect, + actual.body.publicKey.toBase58(), + actualHash + ); + } + console.log(); + + for (let i = 0; i < n; i++) { + let expected = flatUpdates[i]; console.log( 'expected: ', expected.body.callDepth, + ' true', expected.body.publicKey.toBase58(), hashAccountUpdate(expected).toBigInt() ); - console.log( - 'actual: ', - actual.body.callDepth, - actual.body.publicKey.toBase58(), - hashAccountUpdate(actual).toBigInt() - ); } } ); @@ -120,3 +160,12 @@ function callForestToNestedArray( return accountUpdate; }); } + +function assertEqual(actual: AccountUpdate, expected: AccountUpdate) { + let actualHash = hashAccountUpdate(actual).toBigInt(); + let expectedHash = hashAccountUpdate(expected).toBigInt(); + + assert.deepStrictEqual(actual.body, expected.body); + assert.deepStrictEqual(actual.authorization, expected.authorization); + assert.deepStrictEqual(actualHash, expectedHash); +} From 32e606b8d3aed14a24c21d444af504e26545dc13 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 14:04:46 +0100 Subject: [PATCH 27/57] it actually works when not skipping subtrees --- src/lib/mina/token/call-forest.ts | 8 ++-- src/lib/mina/token/call-forest.unit-test.ts | 52 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index a60a93ea1b..607a1c365a 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -1,7 +1,6 @@ import { prefixes } from '../../../provable/poseidon-bigint.js'; import { AccountUpdate, - Field, Hashed, Poseidon, Provable, @@ -15,6 +14,7 @@ import { ProvableHashable, genericHash, } from './merkle-list.js'; +import { Field, Bool } from '../../core.js'; export { CallForest, PartialCallForest, hashAccountUpdate }; @@ -88,7 +88,7 @@ class PartialCallForest { * However, neither can be ruled out. We're returning { update, usesThisToken: Bool } and let the * caller handle the irrelevant case where `usesThisToken` is false. */ - next() { + next({ skipSubtrees = true } = {}) { // get next account update from the current forest (might be a dummy) // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); @@ -117,7 +117,9 @@ class PartialCallForest { .and(canAccessThisToken); // if we don't have to check the children, ignore the forest by jumping to its end - let skipSubtree = canAccessThisToken.not().or(isSelf); + let skipSubtree = skipSubtrees + ? canAccessThisToken.not().or(isSelf) + : new Bool(false); forest.jumpToEndIf(skipSubtree); // if we're at the end of the current layer, step up to the next unfinished parent layer diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 291d84660d..ae5f79abb6 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -39,7 +39,7 @@ const accountUpdate = Random.map(accountUpdateBigint, accountUpdateFromBigint); const arrayOf = (x: Random) => // `reset: true` to start callDepth at 0 for each array - Random.array(x, 10, { reset: true }); + Random.array(x, Random.int(10, 40), { reset: true }); const flatAccountUpdatesBigint = arrayOf(accountUpdateBigint); const flatAccountUpdates = arrayOf(accountUpdate); @@ -69,7 +69,7 @@ test.custom({ timeBudget: 1000, logFailures: false })( // traverses the top level of a call forest in correct order // i.e., CallForest.next() works -test.custom({ timeBudget: 10000, logFailures: false })( +test.custom({ timeBudget: 1000, logFailures: false })( flatAccountUpdates, (flatUpdates) => { // convert to o1js-style list of nested `AccountUpdate`s @@ -85,13 +85,46 @@ test.custom({ timeBudget: 10000, logFailures: false })( assertEqual(actual, expected); } - // doing next again should return a dummy + // doing next() again should return a dummy let actual = forest.next().accountUpdate.value.get(); assertEqual(actual, AccountUpdate.dummy()); } ); -// traverses a call forest in correct depth-first order +// traverses a call forest in correct depth-first order, +// assuming we don't skip any subtrees + +test.custom({ timeBudget: 10000, logFailures: false })( + flatAccountUpdates, + (flatUpdates) => { + // convert to o1js-style list of nested `AccountUpdate`s + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + + let forest = CallForest.fromAccountUpdates(updates); + let tokenId = Field.random(); + let partialForest = PartialCallForest.create(forest, tokenId); + + let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + + let n = flatUpdates.length; + for (let i = 0; i < n; i++) { + assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); + + let expected = flatUpdates[i]; + let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; + assertEqual(actual, expected); + } + + // doing next() again should return a dummy + let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; + assertEqual(actual, AccountUpdate.dummy()); + } +); + +// TODO +// traverses a call forest in correct depth-first order, when skipping subtrees test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( flatAccountUpdates, @@ -117,7 +150,7 @@ test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( let expected = flatUpdates[i]; let expectedHash = hashAccountUpdate(expected).toBigInt(); - let actual = partialForest.next().accountUpdate; + let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; let actualHash = hashAccountUpdate(actual).toBigInt(); let isCorrect = String(expectedHash === actualHash).padStart(5, ' '); @@ -166,6 +199,13 @@ function assertEqual(actual: AccountUpdate, expected: AccountUpdate) { let expectedHash = hashAccountUpdate(expected).toBigInt(); assert.deepStrictEqual(actual.body, expected.body); - assert.deepStrictEqual(actual.authorization, expected.authorization); + assert.deepStrictEqual( + actual.authorization.proof, + expected.authorization.proof + ); + assert.deepStrictEqual( + actual.authorization.signature, + expected.authorization.signature + ); assert.deepStrictEqual(actualHash, expectedHash); } From ad069e5585648dc7820c3fa4a9f9f83d09d20afd Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 16:35:01 +0100 Subject: [PATCH 28/57] finish unit tests for call forest iteration --- src/lib/mina/token/call-forest.ts | 6 +- src/lib/mina/token/call-forest.unit-test.ts | 214 ++++++++++++-------- 2 files changed, 137 insertions(+), 83 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 607a1c365a..146f97d1ff 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -88,7 +88,7 @@ class PartialCallForest { * However, neither can be ruled out. We're returning { update, usesThisToken: Bool } and let the * caller handle the irrelevant case where `usesThisToken` is false. */ - next({ skipSubtrees = true } = {}) { + next() { // get next account update from the current forest (might be a dummy) // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); @@ -117,9 +117,7 @@ class PartialCallForest { .and(canAccessThisToken); // if we don't have to check the children, ignore the forest by jumping to its end - let skipSubtree = skipSubtrees - ? canAccessThisToken.not().or(isSelf) - : new Bool(false); + let skipSubtree = canAccessThisToken.not().or(isSelf); forest.jumpToEndIf(skipSubtree); // if we're at the end of the current layer, step up to the next unfinished parent layer diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index ae5f79abb6..de62ceb354 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -2,8 +2,7 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; import { CallForest, - HashedAccountUpdate, - PartialCallForest, + PartialCallForest as CallForestIterator, hashAccountUpdate, } from './call-forest.js'; import { @@ -19,7 +18,9 @@ import { CallForest as SimpleCallForest, } from '../../../mina-signer/src/sign-zkapp-command.js'; import assert from 'assert'; -import { Field } from '../../field.js'; +import { Field, Bool } from '../../core.js'; +import { Bool as BoolB } from '../../../provable/field-bigint.js'; +import { PublicKey } from '../../signature.js'; // RANDOM NUMBER GENERATORS for account updates @@ -32,6 +33,12 @@ const accountUpdateBigint = Random.map( // fix verification key if (a.body.update.verificationKey.isSome) a.body.update.verificationKey.value = dummyVerificationKey; + + // ensure that, by default, all account updates are token-accessible + a.body.mayUseToken = + a.body.callDepth === 0 + ? { parentsOwnToken: BoolB(true), inheritFromParent: BoolB(false) } + : { parentsOwnToken: BoolB(false), inheritFromParent: BoolB(true) }; return a; } ); @@ -48,7 +55,7 @@ const flatAccountUpdates = arrayOf(accountUpdate); // correctly hashes a call forest -test.custom({ timeBudget: 1000, logFailures: false })( +test.custom({ timeBudget: 1000 })( flatAccountUpdatesBigint, (flatUpdatesBigint) => { // reference: bigint callforest hash from mina-signer @@ -66,114 +73,163 @@ test.custom({ timeBudget: 1000, logFailures: false })( } ); +// can recover flat account updates from nested updates +// this is here to assert that we compute `updates` correctly in the other tests + +test(flatAccountUpdates, (flatUpdates) => { + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + let n = flatUpdates.length; + for (let i = 0; i < n; i++) { + assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); + } +}); + // traverses the top level of a call forest in correct order // i.e., CallForest.next() works -test.custom({ timeBudget: 1000, logFailures: false })( - flatAccountUpdates, - (flatUpdates) => { - // convert to o1js-style list of nested `AccountUpdate`s - let updates = callForestToNestedArray( - accountUpdatesToCallForest(flatUpdates) - ); - let forest = CallForest.fromAccountUpdates(updates); - - let n = updates.length; - for (let i = 0; i < n; i++) { - let expected = updates[i]; - let actual = forest.next().accountUpdate.value.get(); - assertEqual(actual, expected); - } +test.custom({ timeBudget: 1000 })(flatAccountUpdates, (flatUpdates) => { + // prepare call forest from flat account updates + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + let forest = CallForest.fromAccountUpdates(updates); - // doing next() again should return a dummy + // step through top-level by calling forest.next() repeatedly + let n = updates.length; + for (let i = 0; i < n; i++) { + let expected = updates[i]; let actual = forest.next().accountUpdate.value.get(); - assertEqual(actual, AccountUpdate.dummy()); + assertEqual(actual, expected); } -); + + // doing next() again should return a dummy + let actual = forest.next().accountUpdate.value.get(); + assertEqual(actual, AccountUpdate.dummy()); +}); // traverses a call forest in correct depth-first order, -// assuming we don't skip any subtrees +// when no subtrees are skipped + +test.custom({ timeBudget: 5000 })(flatAccountUpdates, (flatUpdates) => { + // with default token id, no subtrees will be skipped + let tokenId = TokenId.default; -test.custom({ timeBudget: 10000, logFailures: false })( + // prepare forest iterator from flat account updates + let updates = callForestToNestedArray( + accountUpdatesToCallForest(flatUpdates) + ); + let forest = CallForest.fromAccountUpdates(updates); + let forestIterator = CallForestIterator.create(forest, tokenId); + + // step through forest iterator and compare against expected updates + let expectedUpdates = flatUpdates; + + let n = flatUpdates.length; + for (let i = 0; i < n; i++) { + let expected = expectedUpdates[i]; + let actual = forestIterator.next().accountUpdate; + assertEqual(actual, expected); + } + + // doing next() again should return a dummy + let actual = forestIterator.next().accountUpdate; + assertEqual(actual, AccountUpdate.dummy()); +}); + +// correctly skips subtrees for various reasons + +// in this test, we make all updates inaccessible by setting the top level to `no` or `inherit`, or to the token owner +// this means we wouldn't need to traverse any update in the whole tree +// but we only notice inaccessibility when we have already traversed the inaccessible update +// so, the result should be that we traverse the top level and nothing else +test.custom({ timeBudget: 5000 })( flatAccountUpdates, - (flatUpdates) => { - // convert to o1js-style list of nested `AccountUpdate`s + Random.publicKey, + (flatUpdates, publicKey) => { + // create token owner and derive token id + let tokenOwner = PublicKey.fromObject({ + x: Field.from(publicKey.x), + isOdd: Bool(!!publicKey.isOdd), + }); + let tokenId = TokenId.derive(tokenOwner); + + // prepare forest iterator from flat account updates let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); + // make all top-level updates inaccessible + updates.forEach((u, i) => { + if (i % 3 === 0) { + u.body.mayUseToken = AccountUpdate.MayUseToken.No; + } else if (i % 3 === 1) { + u.body.mayUseToken = AccountUpdate.MayUseToken.InheritFromParent; + } else { + u.body.publicKey = tokenOwner; + u.body.tokenId = TokenId.default; + } + }); + let forest = CallForest.fromAccountUpdates(updates); - let tokenId = Field.random(); - let partialForest = PartialCallForest.create(forest, tokenId); + let forestIterator = CallForestIterator.create(forest, tokenId); - let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + // step through forest iterator and compare against expected updates + let expectedUpdates = updates; let n = flatUpdates.length; for (let i = 0; i < n; i++) { - assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); - - let expected = flatUpdates[i]; - let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; + let expected = expectedUpdates[i] ?? AccountUpdate.dummy(); + let actual = forestIterator.next().accountUpdate; assertEqual(actual, expected); } - - // doing next() again should return a dummy - let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; - assertEqual(actual, AccountUpdate.dummy()); } ); -// TODO -// traverses a call forest in correct depth-first order, when skipping subtrees - -test.custom({ timeBudget: 10000, logFailures: false, minRuns: 1, maxRuns: 1 })( +// similar to the test before, but now we make all updates in the second layer inaccessible +// so the iteration should walk through the first and second layer +test.custom({ timeBudget: 5000 })( flatAccountUpdates, - (flatUpdates) => { - // convert to o1js-style list of nested `AccountUpdate`s + Random.publicKey, + (flatUpdates, publicKey) => { + // create token owner and derive token id + let tokenOwner = PublicKey.fromObject({ + x: Field.from(publicKey.x), + isOdd: Bool(!!publicKey.isOdd), + }); + let tokenId = TokenId.derive(tokenOwner); + + // make all second-level updates inaccessible + flatUpdates + .filter((u) => u.body.callDepth === 1) + .forEach((u, i) => { + if (i % 3 === 0) { + u.body.mayUseToken = AccountUpdate.MayUseToken.No; + } else if (i % 3 === 1) { + u.body.mayUseToken = AccountUpdate.MayUseToken.ParentsOwnToken; + } else { + u.body.publicKey = tokenOwner; + u.body.tokenId = TokenId.default; + } + }); + + // prepare forest iterator from flat account updates let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - - let dummyParent = AccountUpdate.dummy(); - dummyParent.children.accountUpdates = updates; - console.log(dummyParent.toPrettyLayout()); - let forest = CallForest.fromAccountUpdates(updates); - let tokenId = Field.random(); - let partialForest = PartialCallForest.create(forest, tokenId); + let forestIterator = CallForestIterator.create(forest, tokenId); - let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + // step through forest iterator and compare against expected updates + let expectedUpdates = flatUpdates.filter((u) => u.body.callDepth <= 1); let n = flatUpdates.length; for (let i = 0; i < n; i++) { - assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); - - let expected = flatUpdates[i]; - let expectedHash = hashAccountUpdate(expected).toBigInt(); - let actual = partialForest.next({ skipSubtrees: false }).accountUpdate; - let actualHash = hashAccountUpdate(actual).toBigInt(); - - let isCorrect = String(expectedHash === actualHash).padStart(5, ' '); - - console.log( - 'actual: ', - actual.body.callDepth, - isCorrect, - actual.body.publicKey.toBase58(), - actualHash - ); - } - console.log(); - - for (let i = 0; i < n; i++) { - let expected = flatUpdates[i]; - console.log( - 'expected: ', - expected.body.callDepth, - ' true', - expected.body.publicKey.toBase58(), - hashAccountUpdate(expected).toBigInt() - ); + let expected = expectedUpdates[i] ?? AccountUpdate.dummy(); + let actual = forestIterator.next().accountUpdate; + assertEqual(actual, expected); } } ); From 2803ed904f9b4694b5984b943a25bbf34dd38e32 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 24 Jan 2024 18:16:02 +0100 Subject: [PATCH 29/57] rename, doccomments --- src/lib/mina/token/call-forest.ts | 37 +++++++++++++++------ src/lib/mina/token/call-forest.unit-test.ts | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 146f97d1ff..8b2d953d65 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -14,9 +14,9 @@ import { ProvableHashable, genericHash, } from './merkle-list.js'; -import { Field, Bool } from '../../core.js'; +import { Field } from '../../core.js'; -export { CallForest, PartialCallForest, hashAccountUpdate }; +export { CallForest, CallForestIterator, hashAccountUpdate }; export { HashedAccountUpdate }; @@ -55,7 +55,13 @@ const ParentLayers = MerkleList.create(Layer); type MayUseToken = AccountUpdate['body']['mayUseToken']; const MayUseToken = AccountUpdate.MayUseToken; -class PartialCallForest { +/** + * Data structure to represent a forest tree of account updates that is being iterated over. + * + * Important: Since this is to be used for token manager contracts to process it's entire subtree + * of account updates, the iterator skips subtrees that don't inherit token permissions. + */ +class CallForestIterator { currentLayer: Layer; unfinishedParentLayers: MerkleList; selfToken: Field; @@ -67,7 +73,7 @@ class PartialCallForest { } static create(forest: CallForest, selfToken: Field) { - return new PartialCallForest( + return new CallForestIterator( forest, MayUseToken.ParentsOwnToken, selfToken @@ -80,9 +86,6 @@ class PartialCallForest { * This function is guaranteed to visit each account update in the tree that uses the token * exactly once, when called repeatedly. * - * The internal state of `PartialCallForest` represents the work still to be done, and - * can be passed from one proof to the next. - * * The method makes a best effort to avoid visiting account updates that are not using the token, * and in particular, to avoid returning dummy updates. * However, neither can be ruled out. We're returning { update, usesThisToken: Bool } and let the @@ -112,9 +115,7 @@ class PartialCallForest { this.selfToken ); - let usesThisToken = update.tokenId - .equals(this.selfToken) - .and(canAccessThisToken); + let usesThisToken = update.tokenId.equals(this.selfToken); // if we don't have to check the children, ignore the forest by jumping to its end let skipSubtree = canAccessThisToken.not().or(isSelf); @@ -147,6 +148,22 @@ class PartialCallForest { } } +// helper class to represent the position in a tree = the last visited node + +// every entry in the array is a layer +// so if there are two entries, we last visited a node in the second layer +// this index is the index of the node in that layer +type TreePosition = { index: number; isDone: boolean }[]; +// const TreePosition = { +// stepDown(position: TreePosition, numberOfChildren: number) { +// position.push({ index: 0, isDone: false }); +// }, +// stepUp(position: TreePosition) { +// position.pop(); +// position[position.length - 1].index++; +// }, +// }; + // how to hash a forest function merkleListHash(forestHash: Field, tree: CallTree) { diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index de62ceb354..11b1ea3592 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -2,7 +2,7 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; import { CallForest, - PartialCallForest as CallForestIterator, + CallForestIterator, hashAccountUpdate, } from './call-forest.js'; import { From 18fb5699a04788ab00689fb39bf3ecb342e978b3 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 11:09:41 +0100 Subject: [PATCH 30/57] unify base types of merklelist/array --- src/lib/mina/token/call-forest.ts | 6 +- src/lib/mina/token/merkle-list.ts | 164 +++++++++++++++--------------- 2 files changed, 83 insertions(+), 87 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 8b2d953d65..36d0840578 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -9,7 +9,7 @@ import { } from '../../../index.js'; import { MerkleArray, - MerkleArrayBase, + MerkleListBase, MerkleList, ProvableHashable, genericHash, @@ -27,11 +27,11 @@ class HashedAccountUpdate extends Hashed.create( type CallTree = { accountUpdate: Hashed; - calls: MerkleArrayBase; + calls: MerkleListBase; }; const CallTree: ProvableHashable = Struct({ accountUpdate: HashedAccountUpdate.provable, - calls: MerkleArrayBase(), + calls: MerkleListBase(), }); class CallForest extends MerkleArray.create(CallTree, merkleListHash) { diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index b5cca46e5a..ae0164f5c3 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -11,42 +11,49 @@ import { provableFromClass } from '../../../bindings/lib/provable-snarky.js'; import { packToFields } from '../../hash.js'; export { - MerkleArray, - MerkleArrayIterator, - MerkleArrayBase, + MerkleListBase, MerkleList, + MerkleListIterator, + MerkleArray, WithHash, - WithStackHash, emptyHash, ProvableHashable, genericHash, + merkleListHash, }; -function genericHash( - provable: ProvableHashable, - prefix: string, - value: T -) { - let input = provable.toInput(value); - let packed = packToFields(input); - return Poseidon.hashWithPrefix(prefix, packed); +// common base types for both MerkleList and MerkleArray + +const emptyHash = Field(0); + +type WithHash = { previousHash: Field; element: T }; + +function WithHash(type: ProvableHashable): ProvableHashable> { + return Struct({ previousHash: Field, element: type }); } -function merkleListHash(provable: ProvableHashable, prefix = '') { - return function nextHash(hash: Field, value: T) { - let input = provable.toInput(value); - let packed = packToFields(input); - return Poseidon.hashWithPrefix(prefix, [hash, ...packed]); +type MerkleListBase = { + hash: Field; + data: Unconstrained[]>; +}; + +function MerkleListBase(): ProvableHashable> { + return class extends Struct({ hash: Field, data: Unconstrained.provable }) { + static empty(): MerkleListBase { + return { hash: emptyHash, data: Unconstrained.from([]) }; + } }; } -class MerkleList { +// merkle list + +class MerkleList implements MerkleListBase { hash: Field; - stack: Unconstrained[]>; + data: Unconstrained[]>; - constructor({ hash, stack }: WithStackHash) { + constructor({ hash, data }: MerkleListBase) { this.hash = hash; - this.stack = stack; + this.data = data; } isEmpty() { @@ -60,7 +67,7 @@ class MerkleList { let previousHash = this.hash; this.hash = this.nextHash(previousHash, element); Provable.asProver(() => { - this.stack.set([...this.stack.get(), { previousHash, element }]); + this.data.set([...this.data.get(), { previousHash, element }]); }); } @@ -73,19 +80,19 @@ class MerkleList { ); Provable.asProver(() => { if (condition.toBoolean()) { - this.stack.set([...this.stack.get(), { previousHash, element }]); + this.data.set([...this.data.get(), { previousHash, element }]); } }); } private popWitness() { return Provable.witness(WithHash(this.innerProvable), () => { - let value = this.stack.get(); + let value = this.data.get(); let head = value.at(-1) ?? { previousHash: emptyHash, element: this.innerProvable.empty(), }; - this.stack.set(value.slice(0, -1)); + this.data.set(value.slice(0, -1)); return head; }); } @@ -114,8 +121,8 @@ class MerkleList { } clone(): MerkleList { - let stack = Unconstrained.witness(() => [...this.stack.get()]); - return new this.Constructor({ hash: this.hash, stack }); + let data = Unconstrained.witness(() => [...this.data.get()]); + return new this.Constructor({ hash: this.hash, data }); } /** @@ -125,6 +132,7 @@ class MerkleList { type: ProvableHashable, nextHash: (hash: Field, value: T) => Field = merkleListHash(type) ): typeof MerkleList & { + // override static methods with strict types empty: () => MerkleList; provable: ProvableHashable>; } { @@ -133,13 +141,13 @@ class MerkleList { static _provable = provableFromClass(MerkleList_, { hash: Field, - stack: Unconstrained.provable, + data: Unconstrained.provable, }) as ProvableHashable>; static _nextHash = nextHash; static empty(): MerkleList { - return new this({ hash: emptyHash, stack: Unconstrained.from([]) }); + return new this({ hash: emptyHash, data: Unconstrained.from([]) }); } static get provable(): ProvableHashable> { @@ -176,25 +184,6 @@ class MerkleList { } } -type WithHash = { previousHash: Field; element: T }; -function WithHash(type: ProvableHashable): Provable> { - return Struct({ previousHash: Field, element: type }); -} - -const emptyHash = Field(0); - -type WithStackHash = { - hash: Field; - stack: Unconstrained[]>; -}; -function WithStackHash(): ProvableHashable> { - return class extends Struct({ hash: Field, stack: Unconstrained.provable }) { - static empty(): WithStackHash { - return { hash: emptyHash, stack: Unconstrained.from([]) }; - } - }; -} - type HashInput = { fields?: Field[]; packed?: [Field, number][] }; type ProvableHashable = Provable & { toInput: (x: T) => HashInput; @@ -203,38 +192,25 @@ type ProvableHashable = Provable & { // merkle array -type MerkleArrayBase = { - readonly array: Unconstrained[]>; - readonly hash: Field; -}; - -function MerkleArrayBase(): ProvableHashable> { - return class extends Struct({ array: Unconstrained.provable, hash: Field }) { - static empty(): MerkleArrayBase { - return { array: Unconstrained.from([]), hash: emptyHash }; - } - }; -} - -type MerkleArrayIterator = { - readonly array: Unconstrained[]>; +type MerkleListIterator = { readonly hash: Field; + readonly data: Unconstrained[]>; currentHash: Field; currentIndex: Unconstrained; }; -function MerkleArrayIterator(): ProvableHashable> { +function MerkleListIterator(): ProvableHashable> { return class extends Struct({ - array: Unconstrained.provable, hash: Field, + data: Unconstrained.provable, currentHash: Field, currentIndex: Unconstrained.provable, }) { - static empty(): MerkleArrayIterator { + static empty(): MerkleListIterator { return { - array: Unconstrained.from([]), hash: emptyHash, + data: Unconstrained.from([]), currentHash: emptyHash, currentIndex: Unconstrained.from(0), }; @@ -244,28 +220,28 @@ function MerkleArrayIterator(): ProvableHashable> { /** * MerkleArray is similar to a MerkleList, but it maintains the entire array througout a computation, - * instead of needlessly mutating itself / throwing away context while stepping through it. + * instead of mutating itself / throwing away context while stepping through it. * * We maintain two commitments, both of which are equivalent to a Merkle list hash starting _from the end_ of the array: * - One to the entire array, to prove that we start iterating at the beginning. * - One to the array from the current index until the end, to efficiently step forward. */ -class MerkleArray implements MerkleArrayIterator { +class MerkleArray implements MerkleListIterator { // fixed parts - readonly array: Unconstrained[]>; + readonly data: Unconstrained[]>; readonly hash: Field; // mutable parts currentHash: Field; currentIndex: Unconstrained; - constructor(value: MerkleArrayIterator) { + constructor(value: MerkleListIterator) { Object.assign(this, value); } - static startIterating({ array, hash }: MerkleArrayBase) { + static startIterating({ data, hash }: MerkleListBase) { return new this({ - array, + data, hash, currentHash: hash, currentIndex: Unconstrained.from(-1), @@ -280,14 +256,14 @@ class MerkleArray implements MerkleArrayIterator { } jumpToEnd() { this.currentIndex.setTo( - Unconstrained.witness(() => this.array.get().length) + Unconstrained.witness(() => this.data.get().length) ); this.currentHash = emptyHash; } jumpToEndIf(condition: Bool) { Provable.asProver(() => { if (condition.toBoolean()) { - this.currentIndex.set(this.array.get().length); + this.currentIndex.set(this.data.get().length); } }); this.currentHash = Provable.if(condition, emptyHash, this.currentHash); @@ -301,7 +277,7 @@ class MerkleArray implements MerkleArrayIterator { let { previousHash, element } = Provable.witness( WithHash(this.innerProvable), () => - this.array.get()[index.get()] ?? { + this.data.get()[index.get()] ?? { previousHash: emptyHash, element: this.innerProvable.empty(), } @@ -324,10 +300,10 @@ class MerkleArray implements MerkleArrayIterator { } clone(): MerkleArray { - let array = Unconstrained.witness(() => [...this.array.get()]); + let data = Unconstrained.witness(() => [...this.data.get()]); let currentIndex = Unconstrained.witness(() => this.currentIndex.get()); return new this.Constructor({ - array, + data, hash: this.hash, currentHash: this.currentHash, currentIndex, @@ -335,7 +311,7 @@ class MerkleArray implements MerkleArrayIterator { } /** - * Create a Merkle list type + * Create a Merkle array type */ static create( type: ProvableHashable, @@ -349,8 +325,8 @@ class MerkleArray implements MerkleArrayIterator { static _innerProvable = type; static _provable = provableFromClass(MerkleArray_, { - array: Unconstrained.provable, hash: Field, + data: Unconstrained.provable, currentHash: Field, currentIndex: Unconstrained.provable, }) satisfies ProvableHashable> as ProvableHashable< @@ -370,7 +346,7 @@ class MerkleArray implements MerkleArrayIterator { } return new this({ - array: Unconstrained.from(arrayWithHashes), + data: Unconstrained.from(arrayWithHashes), hash: currentHash, currentHash, currentIndex: Unconstrained.from(-1), @@ -389,7 +365,7 @@ class MerkleArray implements MerkleArrayIterator { } // dynamic subclassing infra - static _nextHash: ((hash: Field, t: any) => Field) | undefined; + static _nextHash: ((hash: Field, value: any) => Field) | undefined; static _provable: ProvableHashable> | undefined; static _innerProvable: ProvableHashable | undefined; @@ -398,12 +374,12 @@ class MerkleArray implements MerkleArrayIterator { return this.constructor as typeof MerkleArray; } - nextHash(hash: Field, t: T): Field { + nextHash(hash: Field, value: T): Field { assert( this.Constructor._nextHash !== undefined, 'MerkleArray not initialized' ); - return this.Constructor._nextHash(hash, t); + return this.Constructor._nextHash(hash, value); } get innerProvable(): ProvableHashable { @@ -414,3 +390,23 @@ class MerkleArray implements MerkleArrayIterator { return this.Constructor._innerProvable; } } + +// hash helpers + +function genericHash( + provable: ProvableHashable, + prefix: string, + value: T +) { + let input = provable.toInput(value); + let packed = packToFields(input); + return Poseidon.hashWithPrefix(prefix, packed); +} + +function merkleListHash(provable: ProvableHashable, prefix = '') { + return function nextHash(hash: Field, value: T) { + let input = provable.toInput(value); + let packed = packToFields(input); + return Poseidon.hashWithPrefix(prefix, [hash, ...packed]); + }; +} From d3d777bcb391dca29342b30c3e090aec09e42f46 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 11:35:41 +0100 Subject: [PATCH 31/57] simple way to update unconstrained --- src/lib/circuit_value.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lib/circuit_value.ts b/src/lib/circuit_value.ts index adccc9a249..67bc6515af 100644 --- a/src/lib/circuit_value.ts +++ b/src/lib/circuit_value.ts @@ -562,6 +562,16 @@ and Provable.asProver() blocks, which execute outside the proof. ); } + /** + * Update an `Unconstrained` by a witness computation. + */ + updateAsProver(compute: (value: T) => T) { + return Provable.asProver(() => { + let value = this.get(); + this.set(compute(value)); + }); + } + static provable: Provable> & { toInput: (x: Unconstrained) => { fields?: Field[]; From a970de2391d94e79e00eb889736098868d698c40 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 11:38:21 +0100 Subject: [PATCH 32/57] change merkle array start index from -1 to 0 --- src/lib/mina/token/merkle-list.ts | 52 ++++++++++++------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index ae0164f5c3..0e621f7279 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -66,9 +66,7 @@ class MerkleList implements MerkleListBase { push(element: T) { let previousHash = this.hash; this.hash = this.nextHash(previousHash, element); - Provable.asProver(() => { - this.data.set([...this.data.get(), { previousHash, element }]); - }); + this.data.updateAsProver((data) => [...data, { previousHash, element }]); } pushIf(condition: Bool, element: T) { @@ -78,11 +76,9 @@ class MerkleList implements MerkleListBase { this.nextHash(previousHash, element), previousHash ); - Provable.asProver(() => { - if (condition.toBoolean()) { - this.data.set([...this.data.get(), { previousHash, element }]); - } - }); + this.data.updateAsProver((data) => + condition.toBoolean() ? [...data, { previousHash, element }] : data + ); } private popWitness() { @@ -196,28 +192,20 @@ type MerkleListIterator = { readonly hash: Field; readonly data: Unconstrained[]>; + /** + * The merkle list hash of `[data[currentIndex], ..., data[length-1]]` (when hashing from right to left). + * + * For example: + * - If `currentIndex === 0`, then `currentHash === this.hash` is the hash of the entire array. + * - If `currentIndex === length`, then `currentHash === emptyHash` is the hash of an empty array. + */ currentHash: Field; + /** + * The index of the element that will be returned by the next call to `next()`. + */ currentIndex: Unconstrained; }; -function MerkleListIterator(): ProvableHashable> { - return class extends Struct({ - hash: Field, - data: Unconstrained.provable, - currentHash: Field, - currentIndex: Unconstrained.provable, - }) { - static empty(): MerkleListIterator { - return { - hash: emptyHash, - data: Unconstrained.from([]), - currentHash: emptyHash, - currentIndex: Unconstrained.from(0), - }; - } - }; -} - /** * MerkleArray is similar to a MerkleList, but it maintains the entire array througout a computation, * instead of mutating itself / throwing away context while stepping through it. @@ -244,7 +232,7 @@ class MerkleArray implements MerkleListIterator { data, hash, currentHash: hash, - currentIndex: Unconstrained.from(-1), + currentIndex: Unconstrained.from(0), }); } assertAtStart() { @@ -272,12 +260,10 @@ class MerkleArray implements MerkleListIterator { next() { // next corresponds to `pop()` in MerkleList // it returns a dummy element if we're at the end of the array - let index = Unconstrained.witness(() => this.currentIndex.get() + 1); - let { previousHash, element } = Provable.witness( WithHash(this.innerProvable), () => - this.data.get()[index.get()] ?? { + this.data.get()[this.currentIndex.get()] ?? { previousHash: emptyHash, element: this.innerProvable.empty(), } @@ -288,7 +274,9 @@ class MerkleArray implements MerkleListIterator { let requiredHash = Provable.if(isDummy, emptyHash, correctHash); this.currentHash.assertEquals(requiredHash); - this.currentIndex.setTo(index); + this.currentIndex.updateAsProver((i) => + Math.min(i + 1, this.data.get().length) + ); this.currentHash = Provable.if(isDummy, emptyHash, previousHash); return Provable.if( @@ -349,7 +337,7 @@ class MerkleArray implements MerkleListIterator { data: Unconstrained.from(arrayWithHashes), hash: currentHash, currentHash, - currentIndex: Unconstrained.from(-1), + currentIndex: Unconstrained.from(0), }); } From 42fcb39830975ac94631b0f21c573c34c0cb7c5d Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 12:37:26 +0100 Subject: [PATCH 33/57] invert internal order in merkle list --- src/lib/mina/token/merkle-list.ts | 70 ++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index 0e621f7279..a3ded82ab3 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -6,9 +6,9 @@ import { Struct, Unconstrained, assert, -} from 'o1js'; +} from '../../../index.js'; import { provableFromClass } from '../../../bindings/lib/provable-snarky.js'; -import { packToFields } from '../../hash.js'; +import { packToFields, ProvableHashable } from '../../hash.js'; export { MerkleListBase, @@ -32,6 +32,9 @@ function WithHash(type: ProvableHashable): ProvableHashable> { return Struct({ previousHash: Field, element: type }); } +/** + * Common base type for {@link MerkleList} and {@link MerkleArray} + */ type MerkleListBase = { hash: Field; data: Unconstrained[]>; @@ -47,6 +50,25 @@ function MerkleListBase(): ProvableHashable> { // merkle list +/** + * Dynamic-length list which is represented as a single hash + * + * Supported operations are {@link push()} and {@link pop()} and some variants thereof. + * + * **Important:** `push()` adds elements to the _start_ of the internal array and `pop()` removes them from the start. + * This is so that the hash which represents the list is consistent with {@link MerkleArray}, + * and so a `MerkleList` can be used as input to `MerkleArray.startIterating(list)` (which will then iterate starting from the last pushed element). + * + * A Merkle list is generic over its element types, so before using it you must create a subclass for your element type: + * + * ```ts + * class MyList extends MerkleList.create(MyType) {} + * + * // now use it + * let list = MyList.empty(); + * list.push(new MyType(...)); + * ``` + */ class MerkleList implements MerkleListBase { hash: Field; data: Unconstrained[]>; @@ -59,14 +81,11 @@ class MerkleList implements MerkleListBase { isEmpty() { return this.hash.equals(emptyHash); } - notEmpty() { - return this.hash.equals(emptyHash).not(); - } push(element: T) { let previousHash = this.hash; this.hash = this.nextHash(previousHash, element); - this.data.updateAsProver((data) => [...data, { previousHash, element }]); + this.data.updateAsProver((data) => [{ previousHash, element }, ...data]); } pushIf(condition: Bool, element: T) { @@ -77,18 +96,18 @@ class MerkleList implements MerkleListBase { previousHash ); this.data.updateAsProver((data) => - condition.toBoolean() ? [...data, { previousHash, element }] : data + condition.toBoolean() ? [{ previousHash, element }, ...data] : data ); } private popWitness() { return Provable.witness(WithHash(this.innerProvable), () => { - let value = this.data.get(); - let head = value.at(-1) ?? { + let [value, ...data] = this.data.get(); + let head = value ?? { previousHash: emptyHash, element: this.innerProvable.empty(), }; - this.data.set(value.slice(0, -1)); + this.data.set(data); return head; }); } @@ -96,8 +115,8 @@ class MerkleList implements MerkleListBase { popExn(): T { let { previousHash, element } = this.popWitness(); - let requiredHash = this.nextHash(previousHash, element); - this.hash.assertEquals(requiredHash); + let currentHash = this.nextHash(previousHash, element); + this.hash.assertEquals(currentHash); this.hash = previousHash; return element; @@ -105,11 +124,11 @@ class MerkleList implements MerkleListBase { pop(): T { let { previousHash, element } = this.popWitness(); - let isEmpty = this.isEmpty(); - let correctHash = this.nextHash(previousHash, element); - let requiredHash = Provable.if(isEmpty, emptyHash, correctHash); - this.hash.assertEquals(requiredHash); + + let currentHash = this.nextHash(previousHash, element); + currentHash = Provable.if(isEmpty, emptyHash, currentHash); + this.hash.assertEquals(currentHash); this.hash = Provable.if(isEmpty, emptyHash, previousHash); let provable = this.innerProvable; @@ -123,6 +142,15 @@ class MerkleList implements MerkleListBase { /** * Create a Merkle list type + * + * Optionally, you can tell `create()` how to do the hash that pushed a new list element, by passing a `nextHash` function. + * + * @example + * ```ts + * class MyList extends MerkleList.create(Field, (hash, x) => + * Poseidon.hashWithPrefix('custom', [hash, x]) + * ) {} + * ``` */ static create( type: ProvableHashable, @@ -163,12 +191,12 @@ class MerkleList implements MerkleListBase { return this.constructor as typeof MerkleList; } - nextHash(hash: Field, t: T): Field { + nextHash(hash: Field, value: T): Field { assert( this.Constructor._nextHash !== undefined, 'MerkleList not initialized' ); - return this.Constructor._nextHash(hash, t); + return this.Constructor._nextHash(hash, value); } get innerProvable(): ProvableHashable { @@ -180,12 +208,6 @@ class MerkleList implements MerkleListBase { } } -type HashInput = { fields?: Field[]; packed?: [Field, number][] }; -type ProvableHashable = Provable & { - toInput: (x: T) => HashInput; - empty: () => T; -}; - // merkle array type MerkleListIterator = { From 28e906bb84493fba58d31be90bd6ae464fc77795 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 13:51:04 +0100 Subject: [PATCH 34/57] make merkle list the main callforest intf --- src/lib/mina/token/call-forest.ts | 18 ++++-- src/lib/mina/token/call-forest.unit-test.ts | 8 +-- src/lib/mina/token/merkle-list.ts | 61 ++++++++++++++------- 3 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 36d0840578..a4b070481d 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -16,7 +16,7 @@ import { } from './merkle-list.js'; import { Field } from '../../core.js'; -export { CallForest, CallForestIterator, hashAccountUpdate }; +export { CallForest, CallForestArray, CallForestIterator, hashAccountUpdate }; export { HashedAccountUpdate }; @@ -34,7 +34,7 @@ const CallTree: ProvableHashable = Struct({ calls: MerkleListBase(), }); -class CallForest extends MerkleArray.create(CallTree, merkleListHash) { +class CallForest extends MerkleList.create(CallTree, merkleListHash) { static fromAccountUpdates(updates: AccountUpdate[]): CallForest { let nodes = updates.map((update) => { let accountUpdate = HashedAccountUpdate.hash(update); @@ -46,8 +46,10 @@ class CallForest extends MerkleArray.create(CallTree, merkleListHash) { } } +class CallForestArray extends MerkleArray.createFromList(CallForest) {} + class Layer extends Struct({ - forest: CallForest.provable, + forest: CallForestArray.provable, mayUseToken: AccountUpdate.MayUseToken.type, }) {} const ParentLayers = MerkleList.create(Layer); @@ -66,7 +68,11 @@ class CallForestIterator { unfinishedParentLayers: MerkleList; selfToken: Field; - constructor(forest: CallForest, mayUseToken: MayUseToken, selfToken: Field) { + constructor( + forest: CallForestArray, + mayUseToken: MayUseToken, + selfToken: Field + ) { this.currentLayer = { forest, mayUseToken }; this.unfinishedParentLayers = ParentLayers.empty(); this.selfToken = selfToken; @@ -74,7 +80,7 @@ class CallForestIterator { static create(forest: CallForest, selfToken: Field) { return new CallForestIterator( - forest, + CallForestArray.startIterating(forest), MayUseToken.ParentsOwnToken, selfToken ); @@ -95,7 +101,7 @@ class CallForestIterator { // get next account update from the current forest (might be a dummy) // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); - let forest = CallForest.startIterating(calls); + let forest = CallForestArray.startIterating(calls); let parentForest = this.currentLayer.forest; this.unfinishedParentLayers.pushIf(parentForest.isAtEnd().not(), { diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index 11b1ea3592..a9ca56bc56 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -88,25 +88,25 @@ test(flatAccountUpdates, (flatUpdates) => { }); // traverses the top level of a call forest in correct order -// i.e., CallForest.next() works +// i.e., CallForestArray works test.custom({ timeBudget: 1000 })(flatAccountUpdates, (flatUpdates) => { // prepare call forest from flat account updates let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let forest = CallForest.fromAccountUpdates(updates); + let forest = CallForest.fromAccountUpdates(updates).startIterating(); // step through top-level by calling forest.next() repeatedly let n = updates.length; for (let i = 0; i < n; i++) { let expected = updates[i]; - let actual = forest.next().accountUpdate.value.get(); + let actual = forest.next().accountUpdate.unhash(); assertEqual(actual, expected); } // doing next() again should return a dummy - let actual = forest.next().accountUpdate.value.get(); + let actual = forest.next().accountUpdate.unhash(); assertEqual(actual, AccountUpdate.dummy()); }); diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index a3ded82ab3..60139dca77 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -140,6 +140,11 @@ class MerkleList implements MerkleListBase { return new this.Constructor({ hash: this.hash, data }); } + startIterating(): MerkleArray { + let merkleArray = MerkleArray.createFromList(this.Constructor); + return merkleArray.startIterating(this); + } + /** * Create a Merkle list type * @@ -158,6 +163,7 @@ class MerkleList implements MerkleListBase { ): typeof MerkleList & { // override static methods with strict types empty: () => MerkleList; + from: (array: T[]) => MerkleList; provable: ProvableHashable>; } { return class MerkleList_ extends MerkleList { @@ -174,6 +180,11 @@ class MerkleList implements MerkleListBase { return new this({ hash: emptyHash, data: Unconstrained.from([]) }); } + static from(array: T[]): MerkleList { + let { hash, data } = withHashes(array, nextHash); + return new this({ data: Unconstrained.from(data), hash }); + } + static get provable(): ProvableHashable> { assert(this._provable !== undefined, 'MerkleList not initialized'); return this._provable; @@ -249,14 +260,6 @@ class MerkleArray implements MerkleListIterator { Object.assign(this, value); } - static startIterating({ data, hash }: MerkleListBase) { - return new this({ - data, - hash, - currentHash: hash, - currentIndex: Unconstrained.from(0), - }); - } assertAtStart() { return this.currentHash.assertEquals(this.hash); } @@ -328,6 +331,7 @@ class MerkleArray implements MerkleListIterator { nextHash: (hash: Field, value: T) => Field = merkleListHash(type) ): typeof MerkleArray & { from: (array: T[]) => MerkleArray; + startIterating: (list: MerkleListBase) => MerkleArray; empty: () => MerkleArray; provable: ProvableHashable>; } { @@ -346,19 +350,15 @@ class MerkleArray implements MerkleListIterator { static _nextHash = nextHash; static from(array: T[]): MerkleArray { - let n = array.length; - let arrayWithHashes = Array>(n); - let currentHash = emptyHash; - - for (let i = n - 1; i >= 0; i--) { - arrayWithHashes[i] = { previousHash: currentHash, element: array[i] }; - currentHash = nextHash(currentHash, array[i]); - } + let { hash, data } = withHashes(array, nextHash); + return this.startIterating({ data: Unconstrained.from(data), hash }); + } + static startIterating({ data, hash }: MerkleListBase): MerkleArray { return new this({ - data: Unconstrained.from(arrayWithHashes), - hash: currentHash, - currentHash, + data, + hash, + currentHash: hash, currentIndex: Unconstrained.from(0), }); } @@ -374,6 +374,13 @@ class MerkleArray implements MerkleListIterator { }; } + static createFromList(merkleList: typeof MerkleList) { + return this.create( + merkleList.prototype.innerProvable, + merkleList._nextHash + ); + } + // dynamic subclassing infra static _nextHash: ((hash: Field, value: any) => Field) | undefined; @@ -420,3 +427,19 @@ function merkleListHash(provable: ProvableHashable, prefix = '') { return Poseidon.hashWithPrefix(prefix, [hash, ...packed]); }; } + +function withHashes( + data: T[], + nextHash: (hash: Field, value: T) => Field +): { data: WithHash[]; hash: Field } { + let n = data.length; + let arrayWithHashes = Array>(n); + let currentHash = emptyHash; + + for (let i = n - 1; i >= 0; i--) { + arrayWithHashes[i] = { previousHash: currentHash, element: data[i] }; + currentHash = nextHash(currentHash, data[i]); + } + + return { data: arrayWithHashes, hash: currentHash }; +} From e0c44fd0961a245d4359120a4002290550a6609c Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:05:25 +0100 Subject: [PATCH 35/57] lower level deps for merkle list --- src/lib/mina/token/merkle-list.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/mina/token/merkle-list.ts index 60139dca77..7fc72b1592 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/mina/token/merkle-list.ts @@ -1,14 +1,9 @@ -import { - Bool, - Field, - Poseidon, - Provable, - Struct, - Unconstrained, - assert, -} from '../../../index.js'; +import { Bool, Field } from '../../core.js'; +import { Provable } from '../../provable.js'; +import { Struct, Unconstrained } from '../../circuit_value.js'; +import { assert } from '../../gadgets/common.js'; import { provableFromClass } from '../../../bindings/lib/provable-snarky.js'; -import { packToFields, ProvableHashable } from '../../hash.js'; +import { Poseidon, packToFields, ProvableHashable } from '../../hash.js'; export { MerkleListBase, From cbde4ff6e70d56e563890c251397d8b34b9456d0 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:06:32 +0100 Subject: [PATCH 36/57] move merkle list --- src/lib/mina/token/call-forest.ts | 2 +- .../{mina/token => provable-types}/merkle-list.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) rename src/lib/{mina/token => provable-types}/merkle-list.ts (97%) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index a4b070481d..a589546838 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -13,7 +13,7 @@ import { MerkleList, ProvableHashable, genericHash, -} from './merkle-list.js'; +} from '../../provable-types/merkle-list.js'; import { Field } from '../../core.js'; export { CallForest, CallForestArray, CallForestIterator, hashAccountUpdate }; diff --git a/src/lib/mina/token/merkle-list.ts b/src/lib/provable-types/merkle-list.ts similarity index 97% rename from src/lib/mina/token/merkle-list.ts rename to src/lib/provable-types/merkle-list.ts index 7fc72b1592..3e299f8c0c 100644 --- a/src/lib/mina/token/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -1,9 +1,9 @@ -import { Bool, Field } from '../../core.js'; -import { Provable } from '../../provable.js'; -import { Struct, Unconstrained } from '../../circuit_value.js'; -import { assert } from '../../gadgets/common.js'; -import { provableFromClass } from '../../../bindings/lib/provable-snarky.js'; -import { Poseidon, packToFields, ProvableHashable } from '../../hash.js'; +import { Bool, Field } from '../core.js'; +import { Provable } from '../provable.js'; +import { Struct, Unconstrained } from '../circuit_value.js'; +import { assert } from '../gadgets/common.js'; +import { provableFromClass } from '../../bindings/lib/provable-snarky.js'; +import { Poseidon, packToFields, ProvableHashable } from '../hash.js'; export { MerkleListBase, From c6813a4e7f37c2d4bba212fb5c00ec1eaf72eee2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:06:40 +0100 Subject: [PATCH 37/57] expose merkle list/array --- src/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/index.ts b/src/index.ts index 8b8aa0672e..81f3bb4193 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,6 +40,12 @@ export { Packed, Hashed } from './lib/provable-types/packed.js'; export { Gadgets } from './lib/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; +export { + MerkleList, + MerkleArray, + ProvableHashable, +} from './lib/provable-types/merkle-list.js'; + export * as Mina from './lib/mina.js'; export type { DeployArgs } from './lib/zkapp.js'; export { From de2f3ca50b81a8fa2f8bfe503c2c68f30da62b4e Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:17:24 +0100 Subject: [PATCH 38/57] remove unnecessary code --- src/lib/mina/token/call-forest.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index a589546838..84fa0f1a41 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -154,22 +154,6 @@ class CallForestIterator { } } -// helper class to represent the position in a tree = the last visited node - -// every entry in the array is a layer -// so if there are two entries, we last visited a node in the second layer -// this index is the index of the node in that layer -type TreePosition = { index: number; isDone: boolean }[]; -// const TreePosition = { -// stepDown(position: TreePosition, numberOfChildren: number) { -// position.push({ index: 0, isDone: false }); -// }, -// stepUp(position: TreePosition) { -// position.pop(); -// position[position.length - 1].index++; -// }, -// }; - // how to hash a forest function merkleListHash(forestHash: Field, tree: CallTree) { From dc09c20ff2784355734bf73b4918fa18f6e5f408 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:41:20 +0100 Subject: [PATCH 39/57] fix dependencies --- src/lib/mina/token/call-forest.ts | 17 +++++++---------- src/lib/provable-types/merkle-list.ts | 1 - 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 84fa0f1a41..2083842f5a 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -1,20 +1,17 @@ import { prefixes } from '../../../provable/poseidon-bigint.js'; -import { - AccountUpdate, - Hashed, - Poseidon, - Provable, - Struct, - TokenId, -} from '../../../index.js'; +import { AccountUpdate, TokenId } from '../../account_update.js'; +import { Field } from '../../core.js'; +import { Provable } from '../../provable.js'; +import { Struct } from '../../circuit_value.js'; +import { assert } from '../../gadgets/common.js'; +import { Poseidon, ProvableHashable } from '../../hash.js'; +import { Hashed } from '../../provable-types/packed.js'; import { MerkleArray, MerkleListBase, MerkleList, - ProvableHashable, genericHash, } from '../../provable-types/merkle-list.js'; -import { Field } from '../../core.js'; export { CallForest, CallForestArray, CallForestIterator, hashAccountUpdate }; diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index 3e299f8c0c..72dee6d025 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -12,7 +12,6 @@ export { MerkleArray, WithHash, emptyHash, - ProvableHashable, genericHash, merkleListHash, }; From bd8f5359e5d5cbb91ea130483aea201d2f73e01b Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 25 Jan 2024 21:42:11 +0100 Subject: [PATCH 40/57] fix build --- src/index.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index 81f3bb4193..d18500bfb7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ export { } from './lib/foreign-field.js'; export { createForeignCurve, ForeignCurve } from './lib/foreign-curve.js'; export { createEcdsa, EcdsaSignature } from './lib/foreign-ecdsa.js'; -export { Poseidon, TokenSymbol } from './lib/hash.js'; +export { Poseidon, TokenSymbol, ProvableHashable } from './lib/hash.js'; export { Keccak } from './lib/keccak.js'; export { Hash } from './lib/hashes-combined.js'; @@ -40,11 +40,7 @@ export { Packed, Hashed } from './lib/provable-types/packed.js'; export { Gadgets } from './lib/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; -export { - MerkleList, - MerkleArray, - ProvableHashable, -} from './lib/provable-types/merkle-list.js'; +export { MerkleList, MerkleArray } from './lib/provable-types/merkle-list.js'; export * as Mina from './lib/mina.js'; export type { DeployArgs } from './lib/zkapp.js'; From 985f5708b8c4a7441fc15d81858ecd1fe321c9f4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Sat, 27 Jan 2024 18:03:02 +0100 Subject: [PATCH 41/57] add edge case to investigate --- src/lib/gadgets/ecdsa.unit-test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/gadgets/ecdsa.unit-test.ts b/src/lib/gadgets/ecdsa.unit-test.ts index 250b088709..f21adb722f 100644 --- a/src/lib/gadgets/ecdsa.unit-test.ts +++ b/src/lib/gadgets/ecdsa.unit-test.ts @@ -43,6 +43,17 @@ for (let Curve of curves) { msg: scalar, publicKey: record({ x: field, y: field }), }); + badSignature.rng = Random.withHardCoded(badSignature.rng, { + signature: { + r: 3243632040670678816425112099743675011873398345579979202080647260629177216981n, + s: 0n, + }, + msg: 0n, + publicKey: { + x: 28948022309329048855892746252171976963363056481941560715954676764349967630336n, + y: 2n, + }, + }); let signatureInputs = record({ privateKey, msg: scalar }); From 09f9fb9b401a4a36f360a4b26427321be5614788 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 29 Jan 2024 12:13:41 +0100 Subject: [PATCH 42/57] export token contract --- src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/index.ts b/src/index.ts index d18500bfb7..c2eac0ee4a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,6 +41,10 @@ export { Gadgets } from './lib/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; export { MerkleList, MerkleArray } from './lib/provable-types/merkle-list.js'; +export { + CallForest, + CallForestIterator, +} from './lib/mina/token/call-forest.js'; export * as Mina from './lib/mina.js'; export type { DeployArgs } from './lib/zkapp.js'; From 05b300b8cf0f9c736a9fe701efe9a4499b224fc7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 31 Jan 2024 12:05:05 +0100 Subject: [PATCH 43/57] move call forest code next to account update --- src/index.ts | 7 +- src/lib/account_update.ts | 201 ++++++++++++++++++-- src/lib/mina.ts | 9 +- src/lib/mina/token/call-forest.ts | 65 +------ src/lib/mina/token/call-forest.unit-test.ts | 12 +- src/lib/provable-types/merkle-list.ts | 1 + src/lib/zkapp.ts | 4 + 7 files changed, 204 insertions(+), 95 deletions(-) diff --git a/src/index.ts b/src/index.ts index c2eac0ee4a..dc7607afa7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,10 +41,6 @@ export { Gadgets } from './lib/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; export { MerkleList, MerkleArray } from './lib/provable-types/merkle-list.js'; -export { - CallForest, - CallForestIterator, -} from './lib/mina/token/call-forest.js'; export * as Mina from './lib/mina.js'; export type { DeployArgs } from './lib/zkapp.js'; @@ -75,8 +71,11 @@ export { AccountUpdate, Permissions, ZkappPublicInput, + CallForest, } from './lib/account_update.js'; +export { CallForestIterator } from './lib/mina/token/call-forest.js'; + export type { TransactionStatus } from './lib/fetch.js'; export { fetchAccount, diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index f585fed1cd..22e3d6ea9f 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -3,6 +3,8 @@ import { FlexibleProvable, provable, provablePure, + Struct, + Unconstrained, } from './circuit_value.js'; import { memoizationContext, memoizeWitness, Provable } from './provable.js'; import { Field, Bool } from './core.js'; @@ -25,7 +27,12 @@ import { Actions, } from '../bindings/mina-transaction/transaction-leaves.js'; import { TokenId as Base58TokenId } from './base58-encodings.js'; -import { hashWithPrefix, packToFields } from './hash.js'; +import { + hashWithPrefix, + packToFields, + Poseidon, + ProvableHashable, +} from './hash.js'; import { mocks, prefixes } from '../bindings/crypto/constants.js'; import { Context } from './global-context.js'; import { assert } from './errors.js'; @@ -33,9 +40,16 @@ import { MlArray } from './ml/base.js'; import { Signature, signFieldElement } from '../mina-signer/src/signature.js'; import { MlFieldConstArray } from './ml/fields.js'; import { transactionCommitments } from '../mina-signer/src/sign-zkapp-command.js'; +import { + genericHash, + MerkleList, + MerkleListBase, + withHashes, +} from './provable-types/merkle-list.js'; +import { Hashed } from './provable-types/packed.js'; // external API -export { AccountUpdate, Permissions, ZkappPublicInput }; +export { AccountUpdate, Permissions, ZkappPublicInput, CallForest }; // internal API export { smartContractContext, @@ -53,12 +67,16 @@ export { Actions, TokenId, Token, - CallForest, + CallForestHelpers, createChildAccountUpdate, AccountUpdatesLayout, zkAppProver, SmartContractContext, dummySignature, + LazyProof, + CallTree, + CallForestUnderConstruction, + hashAccountUpdate, }; const ZkappStateLength = 8; @@ -1045,7 +1063,7 @@ class AccountUpdate implements Types.AccountUpdate { if (isSameAsFeePayer) nonce++; // now, we check how often this account update already updated its nonce in // this tx, and increase nonce from `getAccount` by that amount - CallForest.forEachPredecessor( + CallForestHelpers.forEachPredecessor( Mina.currentTransaction.get().accountUpdates, update as AccountUpdate, (otherUpdate) => { @@ -1092,7 +1110,7 @@ class AccountUpdate implements Types.AccountUpdate { toPublicInput(): ZkappPublicInput { let accountUpdate = this.hash(); - let calls = CallForest.hashChildren(this); + let calls = CallForestHelpers.hashChildren(this); return { accountUpdate, calls }; } @@ -1368,7 +1386,7 @@ class AccountUpdate implements Types.AccountUpdate { if (n === 0) { accountUpdate.children.callsType = { type: 'Equals', - value: CallForest.emptyHash(), + value: CallForestHelpers.emptyHash(), }; } } @@ -1573,7 +1591,148 @@ type WithCallers = { children: WithCallers[]; }; -const CallForest = { +// call forest stuff + +function hashAccountUpdate(update: AccountUpdate) { + return genericHash(AccountUpdate, prefixes.body, update); +} + +class HashedAccountUpdate extends Hashed.create( + AccountUpdate, + hashAccountUpdate +) {} + +type CallTree = { + accountUpdate: Hashed; + calls: MerkleListBase; +}; +const CallTree: ProvableHashable = Struct({ + accountUpdate: HashedAccountUpdate.provable, + calls: MerkleListBase(), +}); + +class CallForest extends MerkleList.create(CallTree, merkleListHash) { + static fromAccountUpdates(updates: AccountUpdate[]): CallForest { + let nodes = updates.map((update) => { + let accountUpdate = HashedAccountUpdate.hash(update); + let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); + return { accountUpdate, calls }; + }); + + return CallForest.from(nodes); + } +} + +// how to hash a forest + +function merkleListHash(forestHash: Field, tree: CallTree) { + return hashCons(forestHash, hashNode(tree)); +} +function hashNode(tree: CallTree) { + return Poseidon.hashWithPrefix(prefixes.accountUpdateNode, [ + tree.accountUpdate.hash, + tree.calls.hash, + ]); +} +function hashCons(forestHash: Field, nodeHash: Field) { + return Poseidon.hashWithPrefix(prefixes.accountUpdateCons, [ + nodeHash, + forestHash, + ]); +} + +/** + * Structure for constructing a call forest from a circuit. + * + * The circuit can mutate account updates and change their array of children, so here we can't hash + * everything immediately. Instead, we maintain a structure consisting of either hashes or full account + * updates that can be hashed into a final call forest at the end. + */ +type CallForestUnderConstruction = HashOrValue< + { + accountUpdate: HashOrValue; + calls: CallForestUnderConstruction; + }[] +>; + +type HashOrValue = + | { useHash: true; hash: Field; value: T } + | { useHash: false; value: T }; + +const CallForestUnderConstruction = { + empty(): CallForestUnderConstruction { + return { useHash: false, value: [] }; + }, + + setHash(forest: CallForestUnderConstruction, hash: Field) { + Object.assign(forest, { useHash: true, hash }); + }, + + witnessHash(forest: CallForestUnderConstruction) { + let hash = Provable.witness(Field, () => { + let nodes = forest.value.map(toCallTree); + return withHashes(nodes, merkleListHash).hash; + }); + CallForestUnderConstruction.setHash(forest, hash); + }, + + push( + forest: CallForestUnderConstruction, + accountUpdate: HashOrValue, + calls?: CallForestUnderConstruction + ) { + forest.value.push({ + accountUpdate, + calls: calls ?? CallForestUnderConstruction.empty(), + }); + }, + + remove(forest: CallForestUnderConstruction, accountUpdate: AccountUpdate) { + // find account update by .id + let index = forest.value.findIndex( + (node) => node.accountUpdate.value.id === accountUpdate.id + ); + + // nothing to do if it's not there + if (index === -1) return; + + // remove it + forest.value.splice(index, 1); + }, + + finalize(forest: CallForestUnderConstruction): CallForest { + if (forest.useHash) { + let data = Unconstrained.witness(() => { + let nodes = forest.value.map(toCallTree); + return withHashes(nodes, merkleListHash).data; + }); + return new CallForest({ hash: forest.hash, data }); + } + + // not using the hash means we calculate it in-circuit + let nodes = forest.value.map(toCallTree); + return CallForest.from(nodes); + }, +}; + +function toCallTree(node: { + accountUpdate: HashOrValue; + calls: CallForestUnderConstruction; +}): CallTree { + let accountUpdate = node.accountUpdate.useHash + ? new HashedAccountUpdate( + node.accountUpdate.hash, + Unconstrained.from(node.accountUpdate.value) + ) + : HashedAccountUpdate.hash(node.accountUpdate.value); + + return { + accountUpdate, + calls: CallForestUnderConstruction.finalize(node.calls), + }; +} + +const CallForestHelpers = { // similar to Mina_base.ZkappCommand.Call_forest.to_account_updates_list // takes a list of accountUpdates, which each can have children, so they form a "forest" (list of trees) // returns a flattened list, with `accountUpdate.body.callDepth` specifying positions in the forest @@ -1590,7 +1749,7 @@ const CallForest = { let children = accountUpdate.children.accountUpdates; accountUpdates.push( accountUpdate, - ...CallForest.toFlatList(children, mutate, depth + 1) + ...CallForestHelpers.toFlatList(children, mutate, depth + 1) ); } return accountUpdates; @@ -1605,23 +1764,29 @@ const CallForest = { // hashes a accountUpdate's children (and their children, and ...) to compute // the `calls` field of ZkappPublicInput hashChildren(update: AccountUpdate): Field { + if (!Provable.inCheckedComputation()) { + return CallForestHelpers.hashChildrenBase(update); + } + let { callsType } = update.children; // compute hash outside the circuit if callsType is "Witness" // i.e., allowing accountUpdates with arbitrary children if (callsType.type === 'Witness') { - return Provable.witness(Field, () => CallForest.hashChildrenBase(update)); + return Provable.witness(Field, () => + CallForestHelpers.hashChildrenBase(update) + ); } - let calls = CallForest.hashChildrenBase(update); - if (callsType.type === 'Equals' && Provable.inCheckedComputation()) { + let calls = CallForestHelpers.hashChildrenBase(update); + if (callsType.type === 'Equals') { calls.assertEquals(callsType.value); } return calls; }, hashChildrenBase({ children }: AccountUpdate) { - let stackHash = CallForest.emptyHash(); + let stackHash = CallForestHelpers.emptyHash(); for (let accountUpdate of [...children.accountUpdates].reverse()) { - let calls = CallForest.hashChildren(accountUpdate); + let calls = CallForestHelpers.hashChildren(accountUpdate); let nodeHash = hashWithPrefix(prefixes.accountUpdateNode, [ accountUpdate.hash(), calls, @@ -1663,7 +1828,7 @@ const CallForest = { withCallers.push({ accountUpdate: update, caller, - children: CallForest.addCallers( + children: CallForestHelpers.addCallers( update.children.accountUpdates, childContext ), @@ -1711,7 +1876,7 @@ const CallForest = { let newUpdates: AccountUpdate[] = []; for (let update of updates) { let newUpdate = map(update); - newUpdate.children.accountUpdates = CallForest.map( + newUpdate.children.accountUpdates = CallForestHelpers.map( update.children.accountUpdates, map ); @@ -1723,7 +1888,7 @@ const CallForest = { forEach(updates: AccountUpdate[], callback: (update: AccountUpdate) => void) { for (let update of updates) { callback(update); - CallForest.forEach(update.children.accountUpdates, callback); + CallForestHelpers.forEach(update.children.accountUpdates, callback); } }, @@ -1733,7 +1898,7 @@ const CallForest = { callback: (update: AccountUpdate) => void ) { let isPredecessor = true; - CallForest.forEach(updates, (otherUpdate) => { + CallForestHelpers.forEach(updates, (otherUpdate) => { if (otherUpdate.id === update.id) isPredecessor = false; if (isPredecessor) callback(otherUpdate); }); @@ -1860,7 +2025,7 @@ const Authorization = { priorAccountUpdates = priorAccountUpdates.filter( (a) => a.id !== myAccountUpdateId ); - let priorAccountUpdatesFlat = CallForest.toFlatList( + let priorAccountUpdatesFlat = CallForestHelpers.toFlatList( priorAccountUpdates, false ); diff --git a/src/lib/mina.ts b/src/lib/mina.ts index fbac14657b..78ad16ecd4 100644 --- a/src/lib/mina.ts +++ b/src/lib/mina.ts @@ -10,7 +10,7 @@ import { AccountUpdate, ZkappPublicInput, TokenId, - CallForest, + CallForestHelpers, Authorization, Actions, Events, @@ -242,8 +242,9 @@ function createTransaction( f(); Provable.asProver(() => { let tx = currentTransaction.get(); - tx.accountUpdates = CallForest.map(tx.accountUpdates, (a) => - toConstant(AccountUpdate, a) + tx.accountUpdates = CallForestHelpers.map( + tx.accountUpdates, + (a) => toConstant(AccountUpdate, a) ); }); }); @@ -263,7 +264,7 @@ function createTransaction( let accountUpdates = currentTransaction.get().accountUpdates; // TODO: I'll be back // CallForest.addCallers(accountUpdates); - accountUpdates = CallForest.toFlatList(accountUpdates); + accountUpdates = CallForestHelpers.toFlatList(accountUpdates); try { // check that on-chain values weren't used without setting a precondition diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 2083842f5a..624a865938 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -1,47 +1,11 @@ -import { prefixes } from '../../../provable/poseidon-bigint.js'; -import { AccountUpdate, TokenId } from '../../account_update.js'; +import { AccountUpdate, CallForest, TokenId } from '../../account_update.js'; import { Field } from '../../core.js'; import { Provable } from '../../provable.js'; import { Struct } from '../../circuit_value.js'; import { assert } from '../../gadgets/common.js'; -import { Poseidon, ProvableHashable } from '../../hash.js'; -import { Hashed } from '../../provable-types/packed.js'; -import { - MerkleArray, - MerkleListBase, - MerkleList, - genericHash, -} from '../../provable-types/merkle-list.js'; - -export { CallForest, CallForestArray, CallForestIterator, hashAccountUpdate }; - -export { HashedAccountUpdate }; - -class HashedAccountUpdate extends Hashed.create( - AccountUpdate, - hashAccountUpdate -) {} - -type CallTree = { - accountUpdate: Hashed; - calls: MerkleListBase; -}; -const CallTree: ProvableHashable = Struct({ - accountUpdate: HashedAccountUpdate.provable, - calls: MerkleListBase(), -}); - -class CallForest extends MerkleList.create(CallTree, merkleListHash) { - static fromAccountUpdates(updates: AccountUpdate[]): CallForest { - let nodes = updates.map((update) => { - let accountUpdate = HashedAccountUpdate.hash(update); - let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); - return { accountUpdate, calls }; - }); +import { MerkleArray, MerkleList } from '../../provable-types/merkle-list.js'; - return CallForest.from(nodes); - } -} +export { CallForestArray, CallForestIterator }; class CallForestArray extends MerkleArray.createFromList(CallForest) {} @@ -150,26 +114,3 @@ class CallForestIterator { return { accountUpdate: update, usesThisToken }; } } - -// how to hash a forest - -function merkleListHash(forestHash: Field, tree: CallTree) { - return hashCons(forestHash, hashNode(tree)); -} - -function hashNode(tree: CallTree) { - return Poseidon.hashWithPrefix(prefixes.accountUpdateNode, [ - tree.accountUpdate.hash, - tree.calls.hash, - ]); -} -function hashCons(forestHash: Field, nodeHash: Field) { - return Poseidon.hashWithPrefix(prefixes.accountUpdateCons, [ - nodeHash, - forestHash, - ]); -} - -function hashAccountUpdate(update: AccountUpdate) { - return genericHash(AccountUpdate, prefixes.body, update); -} diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index a9ca56bc56..fa5d46b508 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -1,14 +1,12 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; -import { - CallForest, - CallForestIterator, - hashAccountUpdate, -} from './call-forest.js'; +import { CallForestIterator } from './call-forest.js'; import { AccountUpdate, - CallForest as ProvableCallForest, + CallForest, + CallForestHelpers, TokenId, + hashAccountUpdate, } from '../../account_update.js'; import { TypesBigint } from '../../../bindings/mina-transaction/types.js'; import { Pickles } from '../../../snarky.js'; @@ -80,7 +78,7 @@ test(flatAccountUpdates, (flatUpdates) => { let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let flatUpdates2 = ProvableCallForest.toFlatList(updates, false); + let flatUpdates2 = CallForestHelpers.toFlatList(updates, false); let n = flatUpdates.length; for (let i = 0; i < n; i++) { assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index 72dee6d025..0ee9c1500d 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -14,6 +14,7 @@ export { emptyHash, genericHash, merkleListHash, + withHashes, }; // common base types for both MerkleList and MerkleArray diff --git a/src/lib/zkapp.ts b/src/lib/zkapp.ts index 9346973f72..cd4d662cc2 100644 --- a/src/lib/zkapp.ts +++ b/src/lib/zkapp.ts @@ -16,6 +16,9 @@ import { ZkappPublicInput, ZkappStateLength, SmartContractContext, + LazyProof, + CallForestHelpers, + CallForestUnderConstruction, } from './account_update.js'; import { cloneCircuitValue, @@ -56,6 +59,7 @@ import { snarkContext, } from './provable-context.js'; import { Cache } from './proof-system/cache.js'; +import { assert } from './gadgets/common.js'; // external API export { From 7cf2a85768dd38b7c5200dd697d6ce9dfb97014d Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 11:38:19 +0100 Subject: [PATCH 44/57] remove accidental/premature changes --- src/lib/account_update.ts | 92 ------------------------------ src/lib/gadgets/ecdsa.unit-test.ts | 11 ---- src/lib/zkapp.ts | 4 -- 3 files changed, 107 deletions(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 22e3d6ea9f..8ee3739b4c 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -75,7 +75,6 @@ export { dummySignature, LazyProof, CallTree, - CallForestUnderConstruction, hashAccountUpdate, }; @@ -1641,97 +1640,6 @@ function hashCons(forestHash: Field, nodeHash: Field) { ]); } -/** - * Structure for constructing a call forest from a circuit. - * - * The circuit can mutate account updates and change their array of children, so here we can't hash - * everything immediately. Instead, we maintain a structure consisting of either hashes or full account - * updates that can be hashed into a final call forest at the end. - */ -type CallForestUnderConstruction = HashOrValue< - { - accountUpdate: HashOrValue; - calls: CallForestUnderConstruction; - }[] ->; - -type HashOrValue = - | { useHash: true; hash: Field; value: T } - | { useHash: false; value: T }; - -const CallForestUnderConstruction = { - empty(): CallForestUnderConstruction { - return { useHash: false, value: [] }; - }, - - setHash(forest: CallForestUnderConstruction, hash: Field) { - Object.assign(forest, { useHash: true, hash }); - }, - - witnessHash(forest: CallForestUnderConstruction) { - let hash = Provable.witness(Field, () => { - let nodes = forest.value.map(toCallTree); - return withHashes(nodes, merkleListHash).hash; - }); - CallForestUnderConstruction.setHash(forest, hash); - }, - - push( - forest: CallForestUnderConstruction, - accountUpdate: HashOrValue, - calls?: CallForestUnderConstruction - ) { - forest.value.push({ - accountUpdate, - calls: calls ?? CallForestUnderConstruction.empty(), - }); - }, - - remove(forest: CallForestUnderConstruction, accountUpdate: AccountUpdate) { - // find account update by .id - let index = forest.value.findIndex( - (node) => node.accountUpdate.value.id === accountUpdate.id - ); - - // nothing to do if it's not there - if (index === -1) return; - - // remove it - forest.value.splice(index, 1); - }, - - finalize(forest: CallForestUnderConstruction): CallForest { - if (forest.useHash) { - let data = Unconstrained.witness(() => { - let nodes = forest.value.map(toCallTree); - return withHashes(nodes, merkleListHash).data; - }); - return new CallForest({ hash: forest.hash, data }); - } - - // not using the hash means we calculate it in-circuit - let nodes = forest.value.map(toCallTree); - return CallForest.from(nodes); - }, -}; - -function toCallTree(node: { - accountUpdate: HashOrValue; - calls: CallForestUnderConstruction; -}): CallTree { - let accountUpdate = node.accountUpdate.useHash - ? new HashedAccountUpdate( - node.accountUpdate.hash, - Unconstrained.from(node.accountUpdate.value) - ) - : HashedAccountUpdate.hash(node.accountUpdate.value); - - return { - accountUpdate, - calls: CallForestUnderConstruction.finalize(node.calls), - }; -} - const CallForestHelpers = { // similar to Mina_base.ZkappCommand.Call_forest.to_account_updates_list // takes a list of accountUpdates, which each can have children, so they form a "forest" (list of trees) diff --git a/src/lib/gadgets/ecdsa.unit-test.ts b/src/lib/gadgets/ecdsa.unit-test.ts index f21adb722f..250b088709 100644 --- a/src/lib/gadgets/ecdsa.unit-test.ts +++ b/src/lib/gadgets/ecdsa.unit-test.ts @@ -43,17 +43,6 @@ for (let Curve of curves) { msg: scalar, publicKey: record({ x: field, y: field }), }); - badSignature.rng = Random.withHardCoded(badSignature.rng, { - signature: { - r: 3243632040670678816425112099743675011873398345579979202080647260629177216981n, - s: 0n, - }, - msg: 0n, - publicKey: { - x: 28948022309329048855892746252171976963363056481941560715954676764349967630336n, - y: 2n, - }, - }); let signatureInputs = record({ privateKey, msg: scalar }); diff --git a/src/lib/zkapp.ts b/src/lib/zkapp.ts index cd4d662cc2..9346973f72 100644 --- a/src/lib/zkapp.ts +++ b/src/lib/zkapp.ts @@ -16,9 +16,6 @@ import { ZkappPublicInput, ZkappStateLength, SmartContractContext, - LazyProof, - CallForestHelpers, - CallForestUnderConstruction, } from './account_update.js'; import { cloneCircuitValue, @@ -59,7 +56,6 @@ import { snarkContext, } from './provable-context.js'; import { Cache } from './proof-system/cache.js'; -import { assert } from './gadgets/common.js'; // external API export { From 4632f93f578fa6f12bd87011cac9301d6315adb2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 11:40:51 +0100 Subject: [PATCH 45/57] prune import not needed yet --- src/lib/account_update.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 8ee3739b4c..4ab04e792b 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -4,7 +4,6 @@ import { provable, provablePure, Struct, - Unconstrained, } from './circuit_value.js'; import { memoizationContext, memoizeWitness, Provable } from './provable.js'; import { Field, Bool } from './core.js'; @@ -44,7 +43,6 @@ import { genericHash, MerkleList, MerkleListBase, - withHashes, } from './provable-types/merkle-list.js'; import { Hashed } from './provable-types/packed.js'; From b8b5cf6c2fc95ba7f5eee1481f8c2e55ed49097c Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 12:18:58 +0100 Subject: [PATCH 46/57] renames and changelog --- CHANGELOG.md | 3 + src/index.ts | 9 ++- src/lib/account_update.ts | 22 ++++-- src/lib/mina/token/call-forest.ts | 56 ++++++++++----- src/lib/mina/token/call-forest.unit-test.ts | 20 +++--- src/lib/provable-types/merkle-list.ts | 75 ++++++++++++--------- 6 files changed, 122 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 433daf8f5e..1f9f78f9a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,9 +23,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added +- `MerkleList` to enable provable operations on a dynamically-sized list https://github.com/o1-labs/o1js/pull/1398 + - including `MerkleListIterator` to iterate over a merkle list - Provable type `Packed` to pack small field elements into fewer field elements https://github.com/o1-labs/o1js/pull/1376 - Provable type `Hashed` to represent provable types by their hash https://github.com/o1-labs/o1js/pull/1377 - This also exposes `Poseidon.hashPacked()` to efficiently hash an arbitrary type +- `TokenAccountUpdateIterator`, a primitive for token contracts to iterate over all token account updates in a transaction. https://github.com/o1-labs/o1js/pull/1398 ## [0.15.4](https://github.com/o1-labs/o1js/compare/be748e42e...e5d1e0f) diff --git a/src/index.ts b/src/index.ts index dc7607afa7..405edf638c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,7 +40,10 @@ export { Packed, Hashed } from './lib/provable-types/packed.js'; export { Gadgets } from './lib/gadgets/gadgets.js'; export { Types } from './bindings/mina-transaction/types.js'; -export { MerkleList, MerkleArray } from './lib/provable-types/merkle-list.js'; +export { + MerkleList, + MerkleListIterator, +} from './lib/provable-types/merkle-list.js'; export * as Mina from './lib/mina.js'; export type { DeployArgs } from './lib/zkapp.js'; @@ -71,10 +74,10 @@ export { AccountUpdate, Permissions, ZkappPublicInput, - CallForest, + AccountUpdateForest, } from './lib/account_update.js'; -export { CallForestIterator } from './lib/mina/token/call-forest.js'; +export { TokenAccountUpdateIterator } from './lib/mina/token/call-forest.js'; export type { TransactionStatus } from './lib/fetch.js'; export { diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 4ab04e792b..5e3332f7af 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -47,7 +47,7 @@ import { import { Hashed } from './provable-types/packed.js'; // external API -export { AccountUpdate, Permissions, ZkappPublicInput, CallForest }; +export { AccountUpdate, Permissions, ZkappPublicInput, AccountUpdateForest }; // internal API export { smartContractContext, @@ -1608,15 +1608,27 @@ const CallTree: ProvableHashable = Struct({ calls: MerkleListBase(), }); -class CallForest extends MerkleList.create(CallTree, merkleListHash) { - static fromAccountUpdates(updates: AccountUpdate[]): CallForest { +/** + * Class which represents a forest (list of trees) of account updates, + * in a compressed way which allows iterating and selectively witnessing the account updates. + * + * The (recursive) type signature is: + * ``` + * type AccountUpdateForest = MerkleList<{ + * accountUpdate: Hashed; + * calls: AccountUpdateForest; + * }>; + * ``` + */ +class AccountUpdateForest extends MerkleList.create(CallTree, merkleListHash) { + static fromArray(updates: AccountUpdate[]): AccountUpdateForest { let nodes = updates.map((update) => { let accountUpdate = HashedAccountUpdate.hash(update); - let calls = CallForest.fromAccountUpdates(update.children.accountUpdates); + let calls = AccountUpdateForest.fromArray(update.children.accountUpdates); return { accountUpdate, calls }; }); - return CallForest.from(nodes); + return AccountUpdateForest.from(nodes); } } diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/call-forest.ts index 624a865938..c5878991db 100644 --- a/src/lib/mina/token/call-forest.ts +++ b/src/lib/mina/token/call-forest.ts @@ -1,16 +1,24 @@ -import { AccountUpdate, CallForest, TokenId } from '../../account_update.js'; +import { + AccountUpdate, + AccountUpdateForest, + TokenId, +} from '../../account_update.js'; import { Field } from '../../core.js'; import { Provable } from '../../provable.js'; import { Struct } from '../../circuit_value.js'; -import { assert } from '../../gadgets/common.js'; -import { MerkleArray, MerkleList } from '../../provable-types/merkle-list.js'; +import { + MerkleListIterator, + MerkleList, +} from '../../provable-types/merkle-list.js'; -export { CallForestArray, CallForestIterator }; +export { AccountUpdateIterator, TokenAccountUpdateIterator }; -class CallForestArray extends MerkleArray.createFromList(CallForest) {} +class AccountUpdateIterator extends MerkleListIterator.createFromList( + AccountUpdateForest +) {} class Layer extends Struct({ - forest: CallForestArray.provable, + forest: AccountUpdateIterator.provable, mayUseToken: AccountUpdate.MayUseToken.type, }) {} const ParentLayers = MerkleList.create(Layer); @@ -19,18 +27,36 @@ type MayUseToken = AccountUpdate['body']['mayUseToken']; const MayUseToken = AccountUpdate.MayUseToken; /** - * Data structure to represent a forest tree of account updates that is being iterated over. + * Data structure to represent a forest of account updates that is being iterated over, + * in the context of a token manager contract. * - * Important: Since this is to be used for token manager contracts to process it's entire subtree - * of account updates, the iterator skips subtrees that don't inherit token permissions. + * The iteration is done in a depth-first manner. + * + * ```ts + * let forest: AccountUpdateForest = ...; + * let tokenIterator = TokenAccountUpdateIterator.create(forest, tokenId); + * + * // process the first 5 account updates in the tree + * for (let i = 0; i < 5; i++) { + * let { accountUpdate, usesThisToken } = tokenIterator.next(); + * // ... do something with the account update ... + * } + * ``` + * + * **Important**: Since this is specifically used by token manager contracts to process their entire subtree + * of account updates, the iterator skips subtrees that don't inherit token permissions and can therefore definitely not use the token. + * + * So, the assumption is that the consumer of this iterator is only interested in account updates that use the token. + * We still can't avoid processing some account updates that don't use the token, therefore the iterator returns a boolean + * `usesThisToken` alongside each account update. */ -class CallForestIterator { +class TokenAccountUpdateIterator { currentLayer: Layer; unfinishedParentLayers: MerkleList; selfToken: Field; constructor( - forest: CallForestArray, + forest: AccountUpdateIterator, mayUseToken: MayUseToken, selfToken: Field ) { @@ -39,9 +65,9 @@ class CallForestIterator { this.selfToken = selfToken; } - static create(forest: CallForest, selfToken: Field) { - return new CallForestIterator( - CallForestArray.startIterating(forest), + static create(forest: AccountUpdateForest, selfToken: Field) { + return new TokenAccountUpdateIterator( + AccountUpdateIterator.startIterating(forest), MayUseToken.ParentsOwnToken, selfToken ); @@ -62,7 +88,7 @@ class CallForestIterator { // get next account update from the current forest (might be a dummy) // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); - let forest = CallForestArray.startIterating(calls); + let forest = AccountUpdateIterator.startIterating(calls); let parentForest = this.currentLayer.forest; this.unfinishedParentLayers.pushIf(parentForest.isAtEnd().not(), { diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/call-forest.unit-test.ts index fa5d46b508..7b093f829e 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/call-forest.unit-test.ts @@ -1,9 +1,9 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; -import { CallForestIterator } from './call-forest.js'; +import { TokenAccountUpdateIterator } from './call-forest.js'; import { AccountUpdate, - CallForest, + AccountUpdateForest, CallForestHelpers, TokenId, hashAccountUpdate, @@ -66,7 +66,7 @@ test.custom({ timeBudget: 1000 })( accountUpdatesToCallForest(flatUpdates) ); - let forest = CallForest.fromAccountUpdates(updates); + let forest = AccountUpdateForest.fromArray(updates); forest.hash.assertEquals(expectedHash); } ); @@ -93,7 +93,7 @@ test.custom({ timeBudget: 1000 })(flatAccountUpdates, (flatUpdates) => { let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let forest = CallForest.fromAccountUpdates(updates).startIterating(); + let forest = AccountUpdateForest.fromArray(updates).startIterating(); // step through top-level by calling forest.next() repeatedly let n = updates.length; @@ -119,8 +119,8 @@ test.custom({ timeBudget: 5000 })(flatAccountUpdates, (flatUpdates) => { let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let forest = CallForest.fromAccountUpdates(updates); - let forestIterator = CallForestIterator.create(forest, tokenId); + let forest = AccountUpdateForest.fromArray(updates); + let forestIterator = TokenAccountUpdateIterator.create(forest, tokenId); // step through forest iterator and compare against expected updates let expectedUpdates = flatUpdates; @@ -171,8 +171,8 @@ test.custom({ timeBudget: 5000 })( } }); - let forest = CallForest.fromAccountUpdates(updates); - let forestIterator = CallForestIterator.create(forest, tokenId); + let forest = AccountUpdateForest.fromArray(updates); + let forestIterator = TokenAccountUpdateIterator.create(forest, tokenId); // step through forest iterator and compare against expected updates let expectedUpdates = updates; @@ -217,8 +217,8 @@ test.custom({ timeBudget: 5000 })( let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let forest = CallForest.fromAccountUpdates(updates); - let forestIterator = CallForestIterator.create(forest, tokenId); + let forest = AccountUpdateForest.fromArray(updates); + let forestIterator = TokenAccountUpdateIterator.create(forest, tokenId); // step through forest iterator and compare against expected updates let expectedUpdates = flatUpdates.filter((u) => u.body.callDepth <= 1); diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index 0ee9c1500d..c74f734cf3 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -8,8 +8,8 @@ import { Poseidon, packToFields, ProvableHashable } from '../hash.js'; export { MerkleListBase, MerkleList, + MerkleListIteratorBase, MerkleListIterator, - MerkleArray, WithHash, emptyHash, genericHash, @@ -17,7 +17,7 @@ export { withHashes, }; -// common base types for both MerkleList and MerkleArray +// common base types for both MerkleList and MerkleListIterator const emptyHash = Field(0); @@ -28,7 +28,7 @@ function WithHash(type: ProvableHashable): ProvableHashable> { } /** - * Common base type for {@link MerkleList} and {@link MerkleArray} + * Common base type for {@link MerkleList} and {@link MerkleListIterator} */ type MerkleListBase = { hash: Field; @@ -51,8 +51,9 @@ function MerkleListBase(): ProvableHashable> { * Supported operations are {@link push()} and {@link pop()} and some variants thereof. * * **Important:** `push()` adds elements to the _start_ of the internal array and `pop()` removes them from the start. - * This is so that the hash which represents the list is consistent with {@link MerkleArray}, - * and so a `MerkleList` can be used as input to `MerkleArray.startIterating(list)` (which will then iterate starting from the last pushed element). + * This is so that the hash which represents the list is consistent with {@link MerkleListIterator}, + * and so a `MerkleList` can be used as input to `MerkleListIterator.startIterating(list)` + * (which will then iterate starting from the last pushed element). * * A Merkle list is generic over its element types, so before using it you must create a subclass for your element type: * @@ -135,8 +136,8 @@ class MerkleList implements MerkleListBase { return new this.Constructor({ hash: this.hash, data }); } - startIterating(): MerkleArray { - let merkleArray = MerkleArray.createFromList(this.Constructor); + startIterating(): MerkleListIterator { + let merkleArray = MerkleListIterator.createFromList(this.Constructor); return merkleArray.startIterating(this); } @@ -214,9 +215,9 @@ class MerkleList implements MerkleListBase { } } -// merkle array +// merkle list iterator -type MerkleListIterator = { +type MerkleListIteratorBase = { readonly hash: Field; readonly data: Unconstrained[]>; @@ -235,14 +236,22 @@ type MerkleListIterator = { }; /** - * MerkleArray is similar to a MerkleList, but it maintains the entire array througout a computation, + * MerkleListIterator is similar to a MerkleList, but it maintains the entire array througout a computation, * instead of mutating itself / throwing away context while stepping through it. * + * The core method that support iteration is {@link next()}. + * + * ```ts + * let iterator = MerkleListIterator.startIterating(list); + * + * let firstElement = iterator.next(); + * ``` + * * We maintain two commitments, both of which are equivalent to a Merkle list hash starting _from the end_ of the array: * - One to the entire array, to prove that we start iterating at the beginning. * - One to the array from the current index until the end, to efficiently step forward. */ -class MerkleArray implements MerkleListIterator { +class MerkleListIterator implements MerkleListIteratorBase { // fixed parts readonly data: Unconstrained[]>; readonly hash: Field; @@ -251,7 +260,7 @@ class MerkleArray implements MerkleListIterator { currentHash: Field; currentIndex: Unconstrained; - constructor(value: MerkleListIterator) { + constructor(value: MerkleListIteratorBase) { Object.assign(this, value); } @@ -307,7 +316,7 @@ class MerkleArray implements MerkleListIterator { ); } - clone(): MerkleArray { + clone(): MerkleListIterator { let data = Unconstrained.witness(() => [...this.data.get()]); let currentIndex = Unconstrained.witness(() => this.currentIndex.get()); return new this.Constructor({ @@ -324,13 +333,13 @@ class MerkleArray implements MerkleListIterator { static create( type: ProvableHashable, nextHash: (hash: Field, value: T) => Field = merkleListHash(type) - ): typeof MerkleArray & { - from: (array: T[]) => MerkleArray; - startIterating: (list: MerkleListBase) => MerkleArray; - empty: () => MerkleArray; - provable: ProvableHashable>; + ): typeof MerkleListIterator & { + from: (array: T[]) => MerkleListIterator; + startIterating: (list: MerkleListBase) => MerkleListIterator; + empty: () => MerkleListIterator; + provable: ProvableHashable>; } { - return class MerkleArray_ extends MerkleArray { + return class MerkleArray_ extends MerkleListIterator { static _innerProvable = type; static _provable = provableFromClass(MerkleArray_, { @@ -338,18 +347,21 @@ class MerkleArray implements MerkleListIterator { data: Unconstrained.provable, currentHash: Field, currentIndex: Unconstrained.provable, - }) satisfies ProvableHashable> as ProvableHashable< - MerkleArray + }) satisfies ProvableHashable> as ProvableHashable< + MerkleListIterator >; static _nextHash = nextHash; - static from(array: T[]): MerkleArray { + static from(array: T[]): MerkleListIterator { let { hash, data } = withHashes(array, nextHash); return this.startIterating({ data: Unconstrained.from(data), hash }); } - static startIterating({ data, hash }: MerkleListBase): MerkleArray { + static startIterating({ + data, + hash, + }: MerkleListBase): MerkleListIterator { return new this({ data, hash, @@ -358,12 +370,15 @@ class MerkleArray implements MerkleListIterator { }); } - static empty(): MerkleArray { + static empty(): MerkleListIterator { return this.from([]); } - static get provable(): ProvableHashable> { - assert(this._provable !== undefined, 'MerkleArray not initialized'); + static get provable(): ProvableHashable> { + assert( + this._provable !== undefined, + 'MerkleListIterator not initialized' + ); return this._provable; } }; @@ -379,17 +394,17 @@ class MerkleArray implements MerkleListIterator { // dynamic subclassing infra static _nextHash: ((hash: Field, value: any) => Field) | undefined; - static _provable: ProvableHashable> | undefined; + static _provable: ProvableHashable> | undefined; static _innerProvable: ProvableHashable | undefined; get Constructor() { - return this.constructor as typeof MerkleArray; + return this.constructor as typeof MerkleListIterator; } nextHash(hash: Field, value: T): Field { assert( this.Constructor._nextHash !== undefined, - 'MerkleArray not initialized' + 'MerkleListIterator not initialized' ); return this.Constructor._nextHash(hash, value); } @@ -397,7 +412,7 @@ class MerkleArray implements MerkleListIterator { get innerProvable(): ProvableHashable { assert( this.Constructor._innerProvable !== undefined, - 'MerkleArray not initialized' + 'MerkleListIterator not initialized' ); return this.Constructor._innerProvable; } From 12060ae92f766139e6b1b537d1559c00e3b5c876 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 12:40:48 +0100 Subject: [PATCH 47/57] better file name --- src/index.ts | 2 +- src/lib/mina/token/{call-forest.ts => forest-iterator.ts} | 0 .../{call-forest.unit-test.ts => forest-iterator.unit-test.ts} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/lib/mina/token/{call-forest.ts => forest-iterator.ts} (100%) rename src/lib/mina/token/{call-forest.unit-test.ts => forest-iterator.unit-test.ts} (99%) diff --git a/src/index.ts b/src/index.ts index 405edf638c..91ddfa7a29 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,7 +77,7 @@ export { AccountUpdateForest, } from './lib/account_update.js'; -export { TokenAccountUpdateIterator } from './lib/mina/token/call-forest.js'; +export { TokenAccountUpdateIterator } from './lib/mina/token/forest-iterator.js'; export type { TransactionStatus } from './lib/fetch.js'; export { diff --git a/src/lib/mina/token/call-forest.ts b/src/lib/mina/token/forest-iterator.ts similarity index 100% rename from src/lib/mina/token/call-forest.ts rename to src/lib/mina/token/forest-iterator.ts diff --git a/src/lib/mina/token/call-forest.unit-test.ts b/src/lib/mina/token/forest-iterator.unit-test.ts similarity index 99% rename from src/lib/mina/token/call-forest.unit-test.ts rename to src/lib/mina/token/forest-iterator.unit-test.ts index 7b093f829e..d6a18affde 100644 --- a/src/lib/mina/token/call-forest.unit-test.ts +++ b/src/lib/mina/token/forest-iterator.unit-test.ts @@ -1,6 +1,6 @@ import { Random, test } from '../../testing/property.js'; import { RandomTransaction } from '../../../mina-signer/src/random-transaction.js'; -import { TokenAccountUpdateIterator } from './call-forest.js'; +import { TokenAccountUpdateIterator } from './forest-iterator.js'; import { AccountUpdate, AccountUpdateForest, From 687891afba3ea9e4ef11aff5486d19076ba2e16b Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 12:51:30 +0100 Subject: [PATCH 48/57] revert rename which became unnecessary --- src/lib/account_update.ts | 34 +++++++++---------- src/lib/mina.ts | 20 +++++------ .../mina/token/forest-iterator.unit-test.ts | 4 +-- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 5e3332f7af..af5df67640 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -65,7 +65,7 @@ export { Actions, TokenId, Token, - CallForestHelpers, + CallForest, createChildAccountUpdate, AccountUpdatesLayout, zkAppProver, @@ -1060,7 +1060,7 @@ class AccountUpdate implements Types.AccountUpdate { if (isSameAsFeePayer) nonce++; // now, we check how often this account update already updated its nonce in // this tx, and increase nonce from `getAccount` by that amount - CallForestHelpers.forEachPredecessor( + CallForest.forEachPredecessor( Mina.currentTransaction.get().accountUpdates, update as AccountUpdate, (otherUpdate) => { @@ -1107,7 +1107,7 @@ class AccountUpdate implements Types.AccountUpdate { toPublicInput(): ZkappPublicInput { let accountUpdate = this.hash(); - let calls = CallForestHelpers.hashChildren(this); + let calls = CallForest.hashChildren(this); return { accountUpdate, calls }; } @@ -1383,7 +1383,7 @@ class AccountUpdate implements Types.AccountUpdate { if (n === 0) { accountUpdate.children.callsType = { type: 'Equals', - value: CallForestHelpers.emptyHash(), + value: CallForest.emptyHash(), }; } } @@ -1650,7 +1650,7 @@ function hashCons(forestHash: Field, nodeHash: Field) { ]); } -const CallForestHelpers = { +const CallForest = { // similar to Mina_base.ZkappCommand.Call_forest.to_account_updates_list // takes a list of accountUpdates, which each can have children, so they form a "forest" (list of trees) // returns a flattened list, with `accountUpdate.body.callDepth` specifying positions in the forest @@ -1667,7 +1667,7 @@ const CallForestHelpers = { let children = accountUpdate.children.accountUpdates; accountUpdates.push( accountUpdate, - ...CallForestHelpers.toFlatList(children, mutate, depth + 1) + ...CallForest.toFlatList(children, mutate, depth + 1) ); } return accountUpdates; @@ -1683,18 +1683,16 @@ const CallForestHelpers = { // the `calls` field of ZkappPublicInput hashChildren(update: AccountUpdate): Field { if (!Provable.inCheckedComputation()) { - return CallForestHelpers.hashChildrenBase(update); + return CallForest.hashChildrenBase(update); } let { callsType } = update.children; // compute hash outside the circuit if callsType is "Witness" // i.e., allowing accountUpdates with arbitrary children if (callsType.type === 'Witness') { - return Provable.witness(Field, () => - CallForestHelpers.hashChildrenBase(update) - ); + return Provable.witness(Field, () => CallForest.hashChildrenBase(update)); } - let calls = CallForestHelpers.hashChildrenBase(update); + let calls = CallForest.hashChildrenBase(update); if (callsType.type === 'Equals') { calls.assertEquals(callsType.value); } @@ -1702,9 +1700,9 @@ const CallForestHelpers = { }, hashChildrenBase({ children }: AccountUpdate) { - let stackHash = CallForestHelpers.emptyHash(); + let stackHash = CallForest.emptyHash(); for (let accountUpdate of [...children.accountUpdates].reverse()) { - let calls = CallForestHelpers.hashChildren(accountUpdate); + let calls = CallForest.hashChildren(accountUpdate); let nodeHash = hashWithPrefix(prefixes.accountUpdateNode, [ accountUpdate.hash(), calls, @@ -1746,7 +1744,7 @@ const CallForestHelpers = { withCallers.push({ accountUpdate: update, caller, - children: CallForestHelpers.addCallers( + children: CallForest.addCallers( update.children.accountUpdates, childContext ), @@ -1794,7 +1792,7 @@ const CallForestHelpers = { let newUpdates: AccountUpdate[] = []; for (let update of updates) { let newUpdate = map(update); - newUpdate.children.accountUpdates = CallForestHelpers.map( + newUpdate.children.accountUpdates = CallForest.map( update.children.accountUpdates, map ); @@ -1806,7 +1804,7 @@ const CallForestHelpers = { forEach(updates: AccountUpdate[], callback: (update: AccountUpdate) => void) { for (let update of updates) { callback(update); - CallForestHelpers.forEach(update.children.accountUpdates, callback); + CallForest.forEach(update.children.accountUpdates, callback); } }, @@ -1816,7 +1814,7 @@ const CallForestHelpers = { callback: (update: AccountUpdate) => void ) { let isPredecessor = true; - CallForestHelpers.forEach(updates, (otherUpdate) => { + CallForest.forEach(updates, (otherUpdate) => { if (otherUpdate.id === update.id) isPredecessor = false; if (isPredecessor) callback(otherUpdate); }); @@ -1943,7 +1941,7 @@ const Authorization = { priorAccountUpdates = priorAccountUpdates.filter( (a) => a.id !== myAccountUpdateId ); - let priorAccountUpdatesFlat = CallForestHelpers.toFlatList( + let priorAccountUpdatesFlat = CallForest.toFlatList( priorAccountUpdates, false ); diff --git a/src/lib/mina.ts b/src/lib/mina.ts index 78ad16ecd4..7961cc3943 100644 --- a/src/lib/mina.ts +++ b/src/lib/mina.ts @@ -10,7 +10,7 @@ import { AccountUpdate, ZkappPublicInput, TokenId, - CallForestHelpers, + CallForest, Authorization, Actions, Events, @@ -242,9 +242,8 @@ function createTransaction( f(); Provable.asProver(() => { let tx = currentTransaction.get(); - tx.accountUpdates = CallForestHelpers.map( - tx.accountUpdates, - (a) => toConstant(AccountUpdate, a) + tx.accountUpdates = CallForest.map(tx.accountUpdates, (a) => + toConstant(AccountUpdate, a) ); }); }); @@ -264,7 +263,7 @@ function createTransaction( let accountUpdates = currentTransaction.get().accountUpdates; // TODO: I'll be back // CallForest.addCallers(accountUpdates); - accountUpdates = CallForestHelpers.toFlatList(accountUpdates); + accountUpdates = CallForest.toFlatList(accountUpdates); try { // check that on-chain values weren't used without setting a precondition @@ -430,8 +429,8 @@ function LocalBlockchain({ getNetworkId: () => minaNetworkId, proofsEnabled, /** - * @deprecated use {@link Mina.getNetworkConstants} - */ + * @deprecated use {@link Mina.getNetworkConstants} + */ accountCreationFee: () => defaultNetworkConstants.accountCreationFee, getNetworkConstants() { return { @@ -519,7 +518,8 @@ function LocalBlockchain({ // TODO: label updates, and try to give precise explanations about what went wrong let errors = JSON.parse(err.message); err.message = invalidTransactionError(txn.transaction, errors, { - accountCreationFee: defaultNetworkConstants.accountCreationFee.toString(), + accountCreationFee: + defaultNetworkConstants.accountCreationFee.toString(), }); } finally { throw err; @@ -765,8 +765,8 @@ function Network( return { getNetworkId: () => minaNetworkId, /** - * @deprecated use {@link Mina.getNetworkConstants} - */ + * @deprecated use {@link Mina.getNetworkConstants} + */ accountCreationFee: () => defaultNetworkConstants.accountCreationFee, getNetworkConstants() { if (currentTransaction()?.fetchMode === 'test') { diff --git a/src/lib/mina/token/forest-iterator.unit-test.ts b/src/lib/mina/token/forest-iterator.unit-test.ts index d6a18affde..06c9981847 100644 --- a/src/lib/mina/token/forest-iterator.unit-test.ts +++ b/src/lib/mina/token/forest-iterator.unit-test.ts @@ -4,7 +4,7 @@ import { TokenAccountUpdateIterator } from './forest-iterator.js'; import { AccountUpdate, AccountUpdateForest, - CallForestHelpers, + CallForest, TokenId, hashAccountUpdate, } from '../../account_update.js'; @@ -78,7 +78,7 @@ test(flatAccountUpdates, (flatUpdates) => { let updates = callForestToNestedArray( accountUpdatesToCallForest(flatUpdates) ); - let flatUpdates2 = CallForestHelpers.toFlatList(updates, false); + let flatUpdates2 = CallForest.toFlatList(updates, false); let n = flatUpdates.length; for (let i = 0; i < n; i++) { assert.deepStrictEqual(flatUpdates2[i], flatUpdates[i]); From 1a4bd8a6fc5616f637fd3fab21dd3aca9dbfe692 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 12:51:33 +0100 Subject: [PATCH 49/57] bindings --- src/bindings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings b/src/bindings index 772ce4ba92..55e313bfad 160000 --- a/src/bindings +++ b/src/bindings @@ -1 +1 @@ -Subproject commit 772ce4ba92e63453253250bb706339016a8d1e8c +Subproject commit 55e313bfadc0659fb0b2e528dc43e82836ef7e7a From cfc8172f04c5249d8c62b7fd5899a8e3ecf53c18 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 13:01:48 +0100 Subject: [PATCH 50/57] more renaming --- src/lib/account_update.ts | 24 ++++++++++++++---------- src/lib/mina/token/forest-iterator.ts | 10 +++++----- src/lib/provable-types/merkle-list.ts | 4 ++-- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index af5df67640..e3caa5ca58 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -72,7 +72,7 @@ export { SmartContractContext, dummySignature, LazyProof, - CallTree, + AccountUpdateTree, hashAccountUpdate, }; @@ -1599,13 +1599,13 @@ class HashedAccountUpdate extends Hashed.create( hashAccountUpdate ) {} -type CallTree = { +type AccountUpdateTree = { accountUpdate: Hashed; - calls: MerkleListBase; + calls: MerkleListBase; }; -const CallTree: ProvableHashable = Struct({ +const AccountUpdateTree: ProvableHashable = Struct({ accountUpdate: HashedAccountUpdate.provable, - calls: MerkleListBase(), + calls: MerkleListBase(), }); /** @@ -1614,13 +1614,17 @@ const CallTree: ProvableHashable = Struct({ * * The (recursive) type signature is: * ``` - * type AccountUpdateForest = MerkleList<{ + * type AccountUpdateForest = MerkleList; + * type AccountUpdateTree = { * accountUpdate: Hashed; * calls: AccountUpdateForest; - * }>; + * }; * ``` */ -class AccountUpdateForest extends MerkleList.create(CallTree, merkleListHash) { +class AccountUpdateForest extends MerkleList.create( + AccountUpdateTree, + merkleListHash +) { static fromArray(updates: AccountUpdate[]): AccountUpdateForest { let nodes = updates.map((update) => { let accountUpdate = HashedAccountUpdate.hash(update); @@ -1634,10 +1638,10 @@ class AccountUpdateForest extends MerkleList.create(CallTree, merkleListHash) { // how to hash a forest -function merkleListHash(forestHash: Field, tree: CallTree) { +function merkleListHash(forestHash: Field, tree: AccountUpdateTree) { return hashCons(forestHash, hashNode(tree)); } -function hashNode(tree: CallTree) { +function hashNode(tree: AccountUpdateTree) { return Poseidon.hashWithPrefix(prefixes.accountUpdateNode, [ tree.accountUpdate.hash, tree.calls.hash, diff --git a/src/lib/mina/token/forest-iterator.ts b/src/lib/mina/token/forest-iterator.ts index c5878991db..5d83b9e077 100644 --- a/src/lib/mina/token/forest-iterator.ts +++ b/src/lib/mina/token/forest-iterator.ts @@ -1,6 +1,7 @@ import { AccountUpdate, AccountUpdateForest, + AccountUpdateTree, TokenId, } from '../../account_update.js'; import { Field } from '../../core.js'; @@ -11,11 +12,10 @@ import { MerkleList, } from '../../provable-types/merkle-list.js'; -export { AccountUpdateIterator, TokenAccountUpdateIterator }; +export { TokenAccountUpdateIterator }; -class AccountUpdateIterator extends MerkleListIterator.createFromList( - AccountUpdateForest -) {} +const AccountUpdateIterator = + MerkleListIterator.createFromList(AccountUpdateForest); class Layer extends Struct({ forest: AccountUpdateIterator.provable, @@ -56,7 +56,7 @@ class TokenAccountUpdateIterator { selfToken: Field; constructor( - forest: AccountUpdateIterator, + forest: MerkleListIterator, mayUseToken: MayUseToken, selfToken: Field ) { diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index c74f734cf3..3ad345f57c 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -339,10 +339,10 @@ class MerkleListIterator implements MerkleListIteratorBase { empty: () => MerkleListIterator; provable: ProvableHashable>; } { - return class MerkleArray_ extends MerkleListIterator { + return class Iterator extends MerkleListIterator { static _innerProvable = type; - static _provable = provableFromClass(MerkleArray_, { + static _provable = provableFromClass(Iterator, { hash: Field, data: Unconstrained.provable, currentHash: Field, From d1f87be3edccff7d5fdfaf44d73f159bf5c3904f Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 15:21:32 +0100 Subject: [PATCH 51/57] add popIf and doccomments --- src/lib/provable-types/merkle-list.ts | 56 +++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index 3ad345f57c..a06e19c563 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -50,10 +50,6 @@ function MerkleListBase(): ProvableHashable> { * * Supported operations are {@link push()} and {@link pop()} and some variants thereof. * - * **Important:** `push()` adds elements to the _start_ of the internal array and `pop()` removes them from the start. - * This is so that the hash which represents the list is consistent with {@link MerkleListIterator}, - * and so a `MerkleList` can be used as input to `MerkleListIterator.startIterating(list)` - * (which will then iterate starting from the last pushed element). * * A Merkle list is generic over its element types, so before using it you must create a subclass for your element type: * @@ -62,8 +58,16 @@ function MerkleListBase(): ProvableHashable> { * * // now use it * let list = MyList.empty(); + * * list.push(new MyType(...)); + * + * let element = list.pop(); * ``` + * + * Internal detail: `push()` adds elements to the _start_ of the internal array and `pop()` removes them from the start. + * This is so that the hash which represents the list is consistent with {@link MerkleListIterator}, + * and so a `MerkleList` can be used as input to `MerkleListIterator.startIterating(list)` + * (which will then iterate starting from the last pushed element). */ class MerkleList implements MerkleListBase { hash: Field; @@ -78,12 +82,18 @@ class MerkleList implements MerkleListBase { return this.hash.equals(emptyHash); } + /** + * Push a new element to the list. + */ push(element: T) { let previousHash = this.hash; this.hash = this.nextHash(previousHash, element); this.data.updateAsProver((data) => [{ previousHash, element }, ...data]); } + /** + * Push a new element to the list, if the `condition` is true. + */ pushIf(condition: Bool, element: T) { let previousHash = this.hash; this.hash = Provable.if( @@ -108,6 +118,11 @@ class MerkleList implements MerkleListBase { }); } + /** + * Remove the last element from the list and return it. + * + * This proves that the list is non-empty, and fails otherwise. + */ popExn(): T { let { previousHash, element } = this.popWitness(); @@ -118,6 +133,11 @@ class MerkleList implements MerkleListBase { return element; } + /** + * Remove the last element from the list and return it. + * + * If the list is empty, returns a dummy element. + */ pop(): T { let { previousHash, element } = this.popWitness(); let isEmpty = this.isEmpty(); @@ -131,6 +151,26 @@ class MerkleList implements MerkleListBase { return Provable.if(isEmpty, provable, provable.empty(), element); } + /** + * Return the last element, but only remove it if `condition` is true. + * + * If the list is empty, returns a dummy element. + */ + popIf(condition: Bool) { + let originalHash = this.hash; + let element = this.pop(); + + // if the condition is false, we restore the original state + this.data.updateAsProver((data) => + condition.toBoolean() + ? data + : [{ previousHash: this.hash, element }, ...data] + ); + this.hash = Provable.if(condition, this.hash, originalHash); + + return element; + } + clone(): MerkleList { let data = Unconstrained.witness(() => [...this.data.get()]); return new this.Constructor({ hash: this.hash, data }); @@ -144,7 +184,7 @@ class MerkleList implements MerkleListBase { /** * Create a Merkle list type * - * Optionally, you can tell `create()` how to do the hash that pushed a new list element, by passing a `nextHash` function. + * Optionally, you can tell `create()` how to do the hash that pushes a new list element, by passing a `nextHash` function. * * @example * ```ts @@ -236,10 +276,10 @@ type MerkleListIteratorBase = { }; /** - * MerkleListIterator is similar to a MerkleList, but it maintains the entire array througout a computation, - * instead of mutating itself / throwing away context while stepping through it. + * MerkleListIterator helps iterating through a Merkle list. + * This works similar to calling `list.pop()` repreatedly, but maintaining the entire list instead of removing elements. * - * The core method that support iteration is {@link next()}. + * The core method that supports iteration is {@link next()}. * * ```ts * let iterator = MerkleListIterator.startIterating(list); From c8d6bfcfa1300a20e96ace38c8638489d0516603 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 15:22:11 +0100 Subject: [PATCH 52/57] improve clarity of forest iteration logic --- src/lib/mina/token/forest-iterator.ts | 57 ++++++++++++--------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/src/lib/mina/token/forest-iterator.ts b/src/lib/mina/token/forest-iterator.ts index 5d83b9e077..90ee4f411a 100644 --- a/src/lib/mina/token/forest-iterator.ts +++ b/src/lib/mina/token/forest-iterator.ts @@ -86,19 +86,17 @@ class TokenAccountUpdateIterator { */ next() { // get next account update from the current forest (might be a dummy) - // and step down into the layer of its children let { accountUpdate, calls } = this.currentLayer.forest.next(); - let forest = AccountUpdateIterator.startIterating(calls); - let parentForest = this.currentLayer.forest; + let childForest = AccountUpdateIterator.startIterating(calls); + let childLayer = { + forest: childForest, + mayUseToken: MayUseToken.InheritFromParent, + }; - this.unfinishedParentLayers.pushIf(parentForest.isAtEnd().not(), { - forest: parentForest, - mayUseToken: this.currentLayer.mayUseToken, - }); - - // check if this account update / it's children can use the token let update = accountUpdate.unhash(); + let usesThisToken = update.tokenId.equals(this.selfToken); + // check if this account update / it's children can use the token let canAccessThisToken = Provable.equal( MayUseToken.type, update.body.mayUseToken, @@ -108,33 +106,30 @@ class TokenAccountUpdateIterator { this.selfToken ); - let usesThisToken = update.tokenId.equals(this.selfToken); - // if we don't have to check the children, ignore the forest by jumping to its end let skipSubtree = canAccessThisToken.not().or(isSelf); - forest.jumpToEndIf(skipSubtree); - - // if we're at the end of the current layer, step up to the next unfinished parent layer - // invariant: the new current layer will _never_ be finished _except_ at the point where we stepped - // through the entire forest and there are no remaining parent layers to finish - let currentLayer = { forest, mayUseToken: MayUseToken.InheritFromParent }; - let currentIsFinished = forest.isAtEnd(); - - let parentLayers = this.unfinishedParentLayers.clone(); - let nextParentLayer = this.unfinishedParentLayers.pop(); - let parentLayersIfSteppingUp = this.unfinishedParentLayers; + childForest.jumpToEndIf(skipSubtree); + + // there are three cases for how to proceed: + // 1. if we have to process children, we step down and add the current layer to the stack of unfinished parent layers + // 2. we don't have to process children, but we're not finished with the current layer yet, so we stay in the current layer + // 3. both of the above are false, so we step up to the next unfinished parent layer + let currentForest = this.currentLayer.forest; + let currentLayerFinished = currentForest.isAtEnd(); + let childLayerFinished = childForest.isAtEnd(); + + this.unfinishedParentLayers.pushIf( + currentLayerFinished.not(), + this.currentLayer + ); + let currentOrParentLayer = + this.unfinishedParentLayers.popIf(childLayerFinished); this.currentLayer = Provable.if( - currentIsFinished, + childLayerFinished, Layer, - nextParentLayer, - currentLayer - ); - this.unfinishedParentLayers = Provable.if( - currentIsFinished, - ParentLayers.provable, - parentLayersIfSteppingUp, - parentLayers + currentOrParentLayer, + childLayer ); return { accountUpdate: update, usesThisToken }; From 0f383542185594800fd0b01030234bb304dc5add Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 1 Feb 2024 15:44:42 +0100 Subject: [PATCH 53/57] normalize comments --- src/lib/mina/token/forest-iterator.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/mina/token/forest-iterator.ts b/src/lib/mina/token/forest-iterator.ts index 90ee4f411a..549a18d68c 100644 --- a/src/lib/mina/token/forest-iterator.ts +++ b/src/lib/mina/token/forest-iterator.ts @@ -112,8 +112,9 @@ class TokenAccountUpdateIterator { // there are three cases for how to proceed: // 1. if we have to process children, we step down and add the current layer to the stack of unfinished parent layers - // 2. we don't have to process children, but we're not finished with the current layer yet, so we stay in the current layer - // 3. both of the above are false, so we step up to the next unfinished parent layer + // 2. if we don't have to process children, but are not finished with the current layer, we stay in the current layer + // (below, this is the case where the current layer is first pushed to and then popped from the stack of unfinished parent layers) + // 3. if both of the above are false, we step up to the next unfinished parent layer let currentForest = this.currentLayer.forest; let currentLayerFinished = currentForest.isAtEnd(); let childLayerFinished = childForest.isAtEnd(); From fbeb4ff62661928a7974412a101f45d393f1a827 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 5 Feb 2024 21:35:44 +0100 Subject: [PATCH 54/57] submodules --- src/bindings | 2 +- src/mina | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings b/src/bindings index 83f8520624..3503101051 160000 --- a/src/bindings +++ b/src/bindings @@ -1 +1 @@ -Subproject commit 83f85206241c2fabd2be360acc5347bc104da452 +Subproject commit 35031010512993426bf226dbd6f1048fa1da09cc diff --git a/src/mina b/src/mina index b1b443ffdc..a5c7f667a5 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit b1b443ffdc15ffd8569f2c244ecdeb5029c35097 +Subproject commit a5c7f667a5008c15243f28921505c3930a4fdf35 From 4703db02e09b14de3275bc6b0ab3e1277b02c6a9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 5 Feb 2024 21:37:21 +0100 Subject: [PATCH 55/57] fix changelog --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efcb589c57..16f8222a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased](https://github.com/o1-labs/o1js/compare/834a44002...HEAD) +### Added + +- `MerkleList` to enable provable operations on a dynamically-sized list https://github.com/o1-labs/o1js/pull/1398 + - including `MerkleListIterator` to iterate over a merkle list +- `TokenAccountUpdateIterator`, a primitive for token contracts to iterate over all token account updates in a transaction. https://github.com/o1-labs/o1js/pull/1398 + ## [0.16.0](https://github.com/o1-labs/o1js/compare/e5d1e0f...834a44002) ### Breaking changes @@ -27,12 +33,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- `MerkleList` to enable provable operations on a dynamically-sized list https://github.com/o1-labs/o1js/pull/1398 - - including `MerkleListIterator` to iterate over a merkle list - Provable type `Packed` to pack small field elements into fewer field elements https://github.com/o1-labs/o1js/pull/1376 - Provable type `Hashed` to represent provable types by their hash https://github.com/o1-labs/o1js/pull/1377 - This also exposes `Poseidon.hashPacked()` to efficiently hash an arbitrary type -- `TokenAccountUpdateIterator`, a primitive for token contracts to iterate over all token account updates in a transaction. https://github.com/o1-labs/o1js/pull/1398 ### Changed From a7b70aff3751efba74d536753bb51e1b0d225834 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 12 Feb 2024 15:22:53 +0100 Subject: [PATCH 56/57] support custom empty hash in merkle list --- src/lib/provable-types/merkle-list.ts | 49 ++++++++++++++++++++------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/lib/provable-types/merkle-list.ts b/src/lib/provable-types/merkle-list.ts index a06e19c563..6e949c050e 100644 --- a/src/lib/provable-types/merkle-list.ts +++ b/src/lib/provable-types/merkle-list.ts @@ -79,7 +79,7 @@ class MerkleList implements MerkleListBase { } isEmpty() { - return this.hash.equals(emptyHash); + return this.hash.equals(this.Constructor.emptyHash); } /** @@ -110,7 +110,7 @@ class MerkleList implements MerkleListBase { return Provable.witness(WithHash(this.innerProvable), () => { let [value, ...data] = this.data.get(); let head = value ?? { - previousHash: emptyHash, + previousHash: this.Constructor.emptyHash, element: this.innerProvable.empty(), }; this.data.set(data); @@ -141,6 +141,7 @@ class MerkleList implements MerkleListBase { pop(): T { let { previousHash, element } = this.popWitness(); let isEmpty = this.isEmpty(); + let emptyHash = this.Constructor.emptyHash; let currentHash = this.nextHash(previousHash, element); currentHash = Provable.if(isEmpty, emptyHash, currentHash); @@ -195,7 +196,8 @@ class MerkleList implements MerkleListBase { */ static create( type: ProvableHashable, - nextHash: (hash: Field, value: T) => Field = merkleListHash(type) + nextHash: (hash: Field, value: T) => Field = merkleListHash(type), + emptyHash_ = emptyHash ): typeof MerkleList & { // override static methods with strict types empty: () => MerkleList; @@ -211,13 +213,14 @@ class MerkleList implements MerkleListBase { }) as ProvableHashable>; static _nextHash = nextHash; + static _emptyHash = emptyHash_; static empty(): MerkleList { - return new this({ hash: emptyHash, data: Unconstrained.from([]) }); + return new this({ hash: emptyHash_, data: Unconstrained.from([]) }); } static from(array: T[]): MerkleList { - let { hash, data } = withHashes(array, nextHash); + let { hash, data } = withHashes(array, nextHash, emptyHash_); return new this({ data: Unconstrained.from(data), hash }); } @@ -230,6 +233,7 @@ class MerkleList implements MerkleListBase { // dynamic subclassing infra static _nextHash: ((hash: Field, t: any) => Field) | undefined; + static _emptyHash: Field | undefined; static _provable: ProvableHashable> | undefined; static _innerProvable: ProvableHashable | undefined; @@ -246,6 +250,11 @@ class MerkleList implements MerkleListBase { return this.Constructor._nextHash(hash, value); } + static get emptyHash() { + assert(this._emptyHash !== undefined, 'MerkleList not initialized'); + return this._emptyHash; + } + get innerProvable(): ProvableHashable { assert( this.Constructor._innerProvable !== undefined, @@ -277,7 +286,7 @@ type MerkleListIteratorBase = { /** * MerkleListIterator helps iterating through a Merkle list. - * This works similar to calling `list.pop()` repreatedly, but maintaining the entire list instead of removing elements. + * This works similar to calling `list.pop()` repeatedly, but maintaining the entire list instead of removing elements. * * The core method that supports iteration is {@link next()}. * @@ -309,13 +318,13 @@ class MerkleListIterator implements MerkleListIteratorBase { } isAtEnd() { - return this.currentHash.equals(emptyHash); + return this.currentHash.equals(this.Constructor.emptyHash); } jumpToEnd() { this.currentIndex.setTo( Unconstrained.witness(() => this.data.get().length) ); - this.currentHash = emptyHash; + this.currentHash = this.Constructor.emptyHash; } jumpToEndIf(condition: Bool) { Provable.asProver(() => { @@ -323,7 +332,11 @@ class MerkleListIterator implements MerkleListIteratorBase { this.currentIndex.set(this.data.get().length); } }); - this.currentHash = Provable.if(condition, emptyHash, this.currentHash); + this.currentHash = Provable.if( + condition, + this.Constructor.emptyHash, + this.currentHash + ); } next() { @@ -333,12 +346,13 @@ class MerkleListIterator implements MerkleListIteratorBase { WithHash(this.innerProvable), () => this.data.get()[this.currentIndex.get()] ?? { - previousHash: emptyHash, + previousHash: this.Constructor.emptyHash, element: this.innerProvable.empty(), } ); let isDummy = this.isAtEnd(); + let emptyHash = this.Constructor.emptyHash; let correctHash = this.nextHash(previousHash, element); let requiredHash = Provable.if(isDummy, emptyHash, correctHash); this.currentHash.assertEquals(requiredHash); @@ -372,7 +386,8 @@ class MerkleListIterator implements MerkleListIteratorBase { */ static create( type: ProvableHashable, - nextHash: (hash: Field, value: T) => Field = merkleListHash(type) + nextHash: (hash: Field, value: T) => Field = merkleListHash(type), + emptyHash_ = emptyHash ): typeof MerkleListIterator & { from: (array: T[]) => MerkleListIterator; startIterating: (list: MerkleListBase) => MerkleListIterator; @@ -392,9 +407,10 @@ class MerkleListIterator implements MerkleListIteratorBase { >; static _nextHash = nextHash; + static _emptyHash = emptyHash_; static from(array: T[]): MerkleListIterator { - let { hash, data } = withHashes(array, nextHash); + let { hash, data } = withHashes(array, nextHash, emptyHash_); return this.startIterating({ data: Unconstrained.from(data), hash }); } @@ -433,6 +449,7 @@ class MerkleListIterator implements MerkleListIteratorBase { // dynamic subclassing infra static _nextHash: ((hash: Field, value: any) => Field) | undefined; + static _emptyHash: Field | undefined; static _provable: ProvableHashable> | undefined; static _innerProvable: ProvableHashable | undefined; @@ -449,6 +466,11 @@ class MerkleListIterator implements MerkleListIteratorBase { return this.Constructor._nextHash(hash, value); } + static get emptyHash() { + assert(this._emptyHash !== undefined, 'MerkleList not initialized'); + return this._emptyHash; + } + get innerProvable(): ProvableHashable { assert( this.Constructor._innerProvable !== undefined, @@ -480,7 +502,8 @@ function merkleListHash(provable: ProvableHashable, prefix = '') { function withHashes( data: T[], - nextHash: (hash: Field, value: T) => Field + nextHash: (hash: Field, value: T) => Field, + emptyHash: Field ): { data: WithHash[]; hash: Field } { let n = data.length; let arrayWithHashes = Array>(n); From f3c1762d553ab031ba2382a227354ed22867a8fc Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 12 Feb 2024 15:23:53 +0100 Subject: [PATCH 57/57] merkle list example that treats actions as a merkle list --- .../zkapps/reducer/actions-as-merkle-list.ts | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 src/examples/zkapps/reducer/actions-as-merkle-list.ts diff --git a/src/examples/zkapps/reducer/actions-as-merkle-list.ts b/src/examples/zkapps/reducer/actions-as-merkle-list.ts new file mode 100644 index 0000000000..86c38af844 --- /dev/null +++ b/src/examples/zkapps/reducer/actions-as-merkle-list.ts @@ -0,0 +1,148 @@ +/** + * This example shows how to iterate through incoming actions, not using `Reducer.reduce` but by + * treating the actions as a merkle list. + * + * This is mainly intended as an example for using `MerkleList`, but it might also be useful as + * a blueprint for processing actions in a custom and more explicit way. + */ +import { + AccountUpdate, + Bool, + Field, + MerkleList, + Mina, + Provable, + PublicKey, + Reducer, + SmartContract, + method, + assert, +} from 'o1js'; + +const { Actions } = AccountUpdate; + +// in this example, an action is just a public key +type Action = PublicKey; +const Action = PublicKey; + +// the actions within one account update are a Merkle list with a custom hash +const emptyHash = Actions.empty().hash; +const nextHash = (hash: Field, action: Action) => + Actions.pushEvent({ hash, data: [] }, action.toFields()).hash; + +class MerkleActions extends MerkleList.create(Action, nextHash, emptyHash) {} + +// the "action state" / actions from many account updates is a Merkle list +// of the above Merkle list, with another custom hash +let emptyActionsHash = Actions.emptyActionState(); +const nextActionsHash = (hash: Field, actions: MerkleActions) => + Actions.updateSequenceState(hash, actions.hash); + +class MerkleActionss extends MerkleList.create( + MerkleActions.provable, + nextActionsHash, + emptyActionsHash +) {} + +// constants for our static-sized provable code +const MAX_UPDATES_WITH_ACTIONS = 100; +const MAX_ACTIONS_PER_UPDATE = 2; + +/** + * This contract allows you to push either 1 or 2 public keys as actions, + * and has a reducer-like method which checks whether a given public key is contained in those actions. + */ +class ActionsContract extends SmartContract { + reducer = Reducer({ actionType: Action }); + + @method + postAddress(address: PublicKey) { + this.reducer.dispatch(address); + } + + // to exhibit the generality of reducer: can dispatch more than 1 action per account update + @method postTwoAddresses(a1: PublicKey, a2: PublicKey) { + this.reducer.dispatch(a1); + this.reducer.dispatch(a2); + } + + @method + assertContainsAddress(address: PublicKey) { + // get actions and, in a witness block, wrap them in a Merkle list of lists + + // note: need to reverse here because `getActions()` returns the last pushed action last, + // but MerkleList.from() wants it to be first to match the natural iteration order + let actionss = this.reducer.getActions().reverse(); + + let merkleActionss = Provable.witness(MerkleActionss.provable, () => + MerkleActionss.from(actionss.map((as) => MerkleActions.from(as))) + ); + + // prove that we know the correct action state + this.account.actionState.requireEquals(merkleActionss.hash); + + // now our provable code to process the actions is very straight-forward + // (note: if we're past the actual sizes, `.pop()` returns a dummy Action -- in this case, the "empty" public key which is not equal to any real address) + let hasAddress = Bool(false); + + for (let i = 0; i < MAX_UPDATES_WITH_ACTIONS; i++) { + let merkleActions = merkleActionss.pop(); + + for (let j = 0; j < MAX_ACTIONS_PER_UPDATE; j++) { + let action = merkleActions.pop(); + hasAddress = hasAddress.or(action.equals(address)); + } + } + + assert(hasAddress); + } +} + +// TESTS + +// set up a local blockchain + +let Local = Mina.LocalBlockchain({ proofsEnabled: false }); +Mina.setActiveInstance(Local); + +let [ + { publicKey: sender, privateKey: senderKey }, + { publicKey: zkappAddress, privateKey: zkappKey }, + { publicKey: otherAddress }, + { publicKey: anotherAddress }, +] = Local.testAccounts; + +let zkapp = new ActionsContract(zkappAddress); + +// deploy the contract + +await ActionsContract.compile(); +console.log( + `rows for ${MAX_UPDATES_WITH_ACTIONS} updates with actions`, + ActionsContract.analyzeMethods().assertContainsAddress.rows +); +let deployTx = await Mina.transaction(sender, () => zkapp.deploy()); +await deployTx.sign([senderKey, zkappKey]).send(); + +// push some actions + +let dispatchTx = await Mina.transaction(sender, () => { + zkapp.postAddress(otherAddress); + zkapp.postAddress(zkappAddress); + zkapp.postTwoAddresses(anotherAddress, sender); + zkapp.postAddress(anotherAddress); + zkapp.postTwoAddresses(zkappAddress, otherAddress); +}); +await dispatchTx.prove(); +await dispatchTx.sign([senderKey]).send(); + +assert(zkapp.reducer.getActions().length === 5); + +// check if the actions contain the `sender` address + +Local.setProofsEnabled(true); +let containsTx = await Mina.transaction(sender, () => + zkapp.assertContainsAddress(sender) +); +await containsTx.prove(); +await containsTx.sign([senderKey]).send();