diff --git a/src/core/cli/comm_data.rs b/src/core/cli/comm_data.rs index c8b03753..402b9b92 100644 --- a/src/core/cli/comm_data.rs +++ b/src/core/cli/comm_data.rs @@ -1,13 +1,17 @@ -use p3_field::Field; +use anyhow::Result; +use p3_field::{Field, PrimeField32}; use serde::{Deserialize, Serialize}; use std::hash::Hash; use crate::{ - core::zstore::{ZPtr, ZStore, DIGEST_SIZE, HASH3_SIZE}, + core::{ + big_num::field_elts_to_biguint, + zstore::{ZPtr, ZStore, DIGEST_SIZE, HASH3_SIZE}, + }, lair::chipset::Chipset, }; -use super::zdag::ZDag; +use super::{paths::commits_dir, zdag::ZDag}; #[derive(Serialize, Deserialize)] pub(crate) struct CommData { @@ -25,7 +29,7 @@ impl CommData { let mut preimg = [F::default(); HASH3_SIZE]; preimg[..DIGEST_SIZE].copy_from_slice(secret); preimg[DIGEST_SIZE..].copy_from_slice(&payload.flatten()); - zstore.hash3(preimg) + zstore.commit(preimg) } } @@ -59,16 +63,6 @@ impl CommData { { ZPtr::comm(self.compute_digest(zstore)) } - - #[inline] - pub(crate) fn populate_zstore>(self, zstore: &mut ZStore) - where - F: Field, - { - let digest = self.compute_digest(zstore); - zstore.intern_comm(digest); - self.zdag.populate_zstore(zstore); - } } impl CommData { @@ -77,3 +71,13 @@ impl CommData { self.zdag.is_flawed(&self.payload, zstore) } } + +impl CommData { + #[inline] + pub(crate) fn dump>(&self, zstore: &mut ZStore) -> Result> { + let comm = self.commit(zstore); + let hash = format!("{:x}", field_elts_to_biguint(&comm.digest)); + std::fs::write(commits_dir()?.join(hash), bincode::serialize(self)?)?; + Ok(comm) + } +} diff --git a/src/core/cli/meta.rs b/src/core/cli/meta.rs index 795fbab7..10909483 100644 --- a/src/core/cli/meta.rs +++ b/src/core/cli/meta.rs @@ -2,10 +2,10 @@ use anyhow::{bail, Result}; use camino::Utf8Path; use itertools::Itertools; use p3_baby_bear::BabyBear; -use p3_field::{AbstractField, PrimeField32}; +use p3_field::PrimeField32; use rustc_hash::FxHashMap; use sphinx_core::stark::StarkGenericConfig; -use std::net::TcpStream; +use std::{io::Write, net::TcpStream}; use crate::{ core::{ @@ -15,9 +15,9 @@ use crate::{ state::{builtin_sym, meta_sym, META_SYMBOLS}, symbol::Symbol, tag::Tag, - zstore::{ZPtr, DIGEST_SIZE}, + zstore::{ZPtr, DIGEST_SIZE, HASH3_SIZE}, }, - lair::{chipset::Chipset, lair_chip::LairMachineProgram}, + lair::{chipset::Chipset, lair_chip::LairMachineProgram, List}, ocaml::compile::compile_and_transform_single_file, }; @@ -29,7 +29,11 @@ use super::{ paths::{commits_dir, proofs_dir}, proofs::{get_verifier_version, CachedProof, ChainProof, OpaqueChainProof, ProtocolProof}, rdg::rand_digest, - repl::Repl, + repl::{OuterScope, Repl}, + scope::{ + dump_scope_microchain, load_scope_bindings, load_scope_comms, load_scope_microchain, + update_scope_bindings, update_scope_comms, ScopeBindings, + }, }; #[allow(clippy::type_complexity)] @@ -274,11 +278,11 @@ impl, C2: Chipset> MetaCmd { let expr = repl .zstore .intern_list([letrec, bindings, current_env_call]); - let (output, _) = repl.reduce_aux(&expr)?; - if output.tag != Tag::Env { - bail!("Reduction resulted in {}", repl.fmt(&output)); + let (env, _) = repl.reduce_aux(&expr)?; + if env.tag != Tag::Env { + bail!("Reduction resulted in {}", repl.fmt(&env)); } - repl.env = output; + repl.env = env; Ok(sym) }, }; @@ -374,20 +378,6 @@ impl, C2: Chipset> MetaCmd { }, }; - /// Persists commitment data and returns the corresponding commitment - fn persist_comm_data( - secret: [F; DIGEST_SIZE], - payload: ZPtr, - repl: &mut Repl, - ) -> Result> { - repl.memoize_dag(&payload); - let comm_data = CommData::new(secret, payload, &repl.zstore); - let comm = comm_data.commit(&mut repl.zstore); - let hash = format!("{:x}", field_elts_to_biguint(&comm.digest)); - std::fs::write(commits_dir()?.join(&hash), bincode::serialize(&comm_data)?)?; - Ok(comm) - } - fn hide( secret: [F; DIGEST_SIZE], payload_expr: &ZPtr, @@ -397,7 +387,14 @@ impl, C2: Chipset> MetaCmd { if payload.tag == Tag::Err { bail!("Payload reduction error: {}", repl.fmt(&payload)); } - Self::persist_comm_data(secret, payload, repl) + let comm = repl.persist_comm_data(secret, payload)?; + if let Some(scope) = repl.current_scope() { + update_scope_comms(&scope.digest, |mut scope_comms| { + scope_comms.insert(comm.digest.into()); + Ok(scope_comms) + })?; + } + Ok(comm) } const HIDE: Self = Self { @@ -439,9 +436,8 @@ impl, C2: Chipset> MetaCmd { name: "commit", summary: "Persists a commitment.", info: &[ - "The secret is an opaque commitment whose digest amounts to zeros", - "and the payload is the reduction of . Equivalent to", - "!(hide #0x0 ).", + "The secret is zero big num and the payload is the reduction of", + ". Equivalent to !(hide #0x0 ).", ], format: "!(commit )", example: &["!(commit 42)"], @@ -452,18 +448,36 @@ impl, C2: Chipset> MetaCmd { }, }; - fn fetch_comm_data(repl: &mut Repl, digest: &[F]) -> Result> { - let hash = format!("{:x}", field_elts_to_biguint(digest)); + fn fetch_persisted_comm_data(repl: &mut Repl, digest: List) -> Result> { + let hash = format!("{:x}", field_elts_to_biguint(&digest)); let comm_data_bytes = std::fs::read(commits_dir()?.join(&hash))?; - let comm_data: CommData = bincode::deserialize(&comm_data_bytes)?; - let payload = comm_data.payload; - comm_data.populate_zstore(&mut repl.zstore); + let CommData { + secret, + payload, + zdag, + } = bincode::deserialize(&comm_data_bytes)?; + let mut preimg = Vec::with_capacity(HASH3_SIZE); + preimg.extend(secret); + preimg.extend(payload.flatten()); + repl.queries + .inject_inv_queries_owned("commit", &repl.toplevel, [(preimg, digest)]); + zdag.populate_zstore(&mut repl.zstore); Ok(payload) } + /// Tries to fetch persisted commitment data if it's not already available + /// in the REPL's `QueryRecord` + fn fetch_comm_data(repl: &mut Repl, digest: List) -> Result<()> { + let inv_comms = repl.queries.get_inv_queries("commit", &repl.toplevel); + if !inv_comms.contains_key(&digest) { + Self::fetch_persisted_comm_data(repl, digest)?; + } + Ok(()) + } + const OPEN: Self = Self { name: "open", - summary: "Fetches a persisted commitment and prints the payload.", + summary: "Fetches a persisted commitment.", info: &[], format: "!(open )", example: &[ @@ -473,9 +487,9 @@ impl, C2: Chipset> MetaCmd { returns: "The commitment payload", run: |repl, args, _dir| { let [&expr] = repl.take(args)?; - let (result, _) = repl.reduce_aux(&expr)?; - match result.tag { - Tag::BigNum | Tag::Comm => Self::fetch_comm_data(repl, &result.digest), + let (ZPtr { tag, digest }, _) = repl.reduce_aux(&expr)?; + match tag { + Tag::BigNum | Tag::Comm => Self::fetch_persisted_comm_data(repl, digest.into()), _ => bail!("Expected a commitment or a BigNum"), } }, @@ -511,15 +525,8 @@ impl, C2: Chipset> MetaCmd { } let (&callable, &call_args) = repl.zstore.fetch_tuple11(call_expr); let (callable, _) = repl.reduce_aux(&callable)?; - match callable.tag { - Tag::BigNum | Tag::Comm => { - let inv_hashes3 = repl.queries.get_inv_queries("hash3", &repl.toplevel); - if !inv_hashes3.contains_key(callable.digest.as_slice()) { - // Try to fetch a persisted commitment. - Self::fetch_comm_data(repl, &callable.digest)?; - } - } - _ => (), + if matches!(callable.tag, Tag::BigNum | Tag::Comm) { + Self::fetch_comm_data(repl, callable.digest.into())?; } let call_args = Self::eval_then_quote(repl, &call_args)?; let call_expr = repl.zstore.intern_cons(callable, call_args); @@ -537,26 +544,38 @@ impl, C2: Chipset> MetaCmd { ], returns: "The call result", run: |repl, args, _dir| { - let env = repl.env; - let (res, _) = Self::call(repl, args, &env)?; + let (res, _) = Self::call(repl, args, &repl.env.clone())?; Ok(res) }, }; - fn persist_chain_comm(repl: &mut Repl, cons: &ZPtr) -> Result<()> { + /// Splits a commitment preimage into the corresponding secret digest and + /// payload `ZPtr`. + fn split_comm_data_preimg(preimg: &[F]) -> ([F; DIGEST_SIZE], ZPtr) { + let mut secret = [F::default(); DIGEST_SIZE]; + secret.copy_from_slice(&preimg[..DIGEST_SIZE]); + let payload = ZPtr::from_flat_data(&preimg[DIGEST_SIZE..]); + (secret, payload) + } + + /// If the callable of a chain result is a commitment, persist its corresponding + /// `CommData`. + fn persist_chain_callable_if_comm(repl: &mut Repl, cons: &ZPtr) -> Result<()> { if cons.tag != Tag::Cons { bail!("Chain result must be a pair"); } + if repl.current_scope().is_some() { + // When in a scope, all commitments are persisted automatically. + return Ok(()); + } let (_, next_callable) = repl.zstore.fetch_tuple11(cons); if matches!(next_callable.tag, Tag::Comm | Tag::BigNum) { - let inv_hashes3 = repl.queries.get_inv_queries("hash3", &repl.toplevel); - let preimg = inv_hashes3 + let inv_comms = repl.queries.get_inv_queries("commit", &repl.toplevel); + let preimg = inv_comms .get(next_callable.digest.as_slice()) .expect("Preimage must be known"); - let mut secret = [F::zero(); DIGEST_SIZE]; - secret.copy_from_slice(&preimg[..DIGEST_SIZE]); - let payload = ZPtr::from_flat_data(&preimg[DIGEST_SIZE..]); - Self::persist_comm_data(secret, payload, repl)?; + let (secret, payload) = Self::split_comm_data_preimg(preimg); + repl.persist_comm_data(secret, payload)?; } Ok(()) } @@ -580,7 +599,7 @@ impl, C2: Chipset> MetaCmd { run: |repl, args, _dir| { let env = repl.zstore.intern_empty_env(); let (cons, _) = Self::call(repl, args, &env)?; - Self::persist_chain_comm(repl, &cons)?; + Self::persist_chain_callable_if_comm(repl, &cons)?; Ok(cons) }, }; @@ -615,7 +634,7 @@ impl, C2: Chipset> MetaCmd { run: |repl, args, _dir| { let (¤t_state_expr, &call_args) = repl.car_cdr(args); let (cons, _) = Self::transition_call(repl, ¤t_state_expr, call_args)?; - Self::persist_chain_comm(repl, &cons)?; + Self::persist_chain_callable_if_comm(repl, &cons)?; Ok(cons) }, }; @@ -866,6 +885,143 @@ impl, C2: Chipset> MetaCmd { Ok(*repl.zstore.t()) }, }; + + fn scope_stack(repl: &mut Repl) -> ZPtr { + let scope_stack = repl.scopes.iter().rev().map(|(s, _)| *s); + repl.zstore.intern_list(scope_stack) + } + + const SCOPE: Self = Self { + name: "scope", + summary: "Enters a new or existing scope", + info: &[ + "Scopes are symbol-named contexts in which:", + "* Commitments created are persisted and become automatically available", + " when entering the same scope again;", + "* It's possible to store (see `scope-store`) variables, which become", + " automatically available when entering the same scope again;", + "* Setting a microchain (see `microchain-set-info`) persists the microchain", + " information within that scope so that it's not necessary to set it", + " again.", + "When entering a scope, commitments and definitions from the outer scope", + "are available for read purposes. The microchain info, however, is not", + "inherited as a security measure and must be set again if intended.", + ], + format: "!(scope )", + example: &["!(scope my-scope)"], + returns: "The resulting stack of scopes", + run: |repl, args, _dir| { + let [&scope] = repl.take(args)?; + Self::validate_binding_symbol(repl, &scope)?; + if repl.scopes.contains_key(&scope) { + bail!("Scope already stacked") + } + + // Save the data for the current environment. + let outer_env = repl.env; + let outer_comms = repl + .queries + .get_inv_queries("commit", &repl.toplevel) + .keys() + .map(Box::clone) + .collect(); + // Note: `Option::take` makes new scopes start with no microchain set. + let outer_microchain = repl.microchain.take(); + + // Attempt to load potentially persisted bindings. + if let Ok(scope_bindings) = load_scope_bindings(&scope.digest) { + let ScopeBindings { binds, zdag } = scope_bindings; + zdag.populate_zstore(&mut repl.zstore); + binds.into_iter().for_each(|(sym, val)| repl.bind(sym, val)); + } + + // Attempt to load potentially persisted commitments. + if let Ok(comms) = load_scope_comms(&scope.digest) { + for digest in comms { + Self::fetch_comm_data(repl, digest)?; + } + } + + // Attempt to load potentially persisted microchain info. + if let Ok(microchain) = load_scope_microchain(&scope.digest) { + repl.microchain = microchain; + } + + let outer_scope = OuterScope { + env: outer_env, + comms: outer_comms, + microchain: outer_microchain, + }; + repl.scopes.insert(scope, outer_scope); + Ok(Self::scope_stack(repl)) + }, + }; + + const SCOPE_POP: Self = Self { + name: "scope-pop", + summary: "Go to the outer scope", + info: &[ + "When a scope is popped, the context from the outer scope is recovered.", + "That is, commitments available, definitions and microchain information.", + ], + format: "!(scope-pop)", + example: &["!(scope-pop)"], + returns: "A pair with the resulting scope stack and the popped scope", + run: |repl, args, _dir| { + if args != repl.zstore.nil() { + bail!("Arguments aren't supported") + } + let Some((sym, outer_scope)) = repl.scopes.pop() else { + bail!("Not in a scope") + }; + let OuterScope { + env, + comms, + microchain, + } = outer_scope; + repl.env = env; + let inv_comms_ref = &mut repl.queries.inv_func_queries[repl.func_indices.commit]; + let inv_comms = std::mem::take(inv_comms_ref).expect("commit coroutine is invertible"); + let filtered = inv_comms + .into_iter() + .filter(|(k, _)| comms.contains(k)) + .collect(); + repl.queries.inv_func_queries[repl.func_indices.commit] = Some(filtered); + repl.microchain = microchain; + let scope_stack = Self::scope_stack(repl); + Ok(repl.zstore.intern_cons(scope_stack, sym)) + }, + }; + + const SCOPE_STORE: Self = Self { + name: "scope-store", + summary: "Stores a symbol binding for availability when re-entering the scope", + info: &[], + format: "!(scope-store ...)", + example: &["!(scope-store var1 var2 var3)"], + returns: "t", + run: |repl, args, _dir| { + let Some(&scope) = repl.current_scope() else { + bail!("Not in a scope"); + }; + repl.memoize_env_dag(); + let (syms, _) = repl.zstore.fetch_list(args); + let env_vec = repl.zstore.fetch_env(&repl.env); + // `env_vec` needs to be consumed in reverse order so bindings can + // be shadowed properly. + let env_map = env_vec.into_iter().rev().collect::>(); + update_scope_bindings(&scope.digest, |mut scope_bindings| { + for sym in &syms { + let Some(val) = env_map.get(sym) else { + bail!("Unbound symbol: {}", repl.fmt(sym)); + }; + scope_bindings.bind(**sym, **val, &repl.zstore); + } + Ok(scope_bindings) + })?; + Ok(*repl.zstore.t()) + }, + }; } type F = BabyBear; @@ -1181,26 +1337,105 @@ impl, C2: Chipset> MetaCmd { }, }; + fn set_microchain( + repl: &mut Repl, + addr: String, + id: [F; DIGEST_SIZE], + ) -> Result<()> { + if repl.microchain.is_some() { + print!("A microchain is already set. Overwrite it? (y/*) "); + std::io::stdout().flush()?; + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + if input != "y\n" { + bail!("Microchain set operation was canceled") + } + } + repl.microchain = Some((addr, id)); + if let Some(scope) = repl.current_scope() { + dump_scope_microchain(&scope.digest, &repl.microchain)?; + } + Ok(()) + } + + const MICROCHAIN_SET_INFO: Self = Self { + name: "microchain-set-info", + summary: "Sets the REPL to point to a particular microchain address and ID", + info: &["When in a scope, this information is persisted across REPL instantiations"], + format: "!(microchain-set-info )", + example: &["!(microchain-set-info \"127.0.0.1:1234\" #c0x123)"], + returns: "t", + run: |repl, args, _dir| { + let [&addr_expr, &id_expr] = repl.take(args)?; + let (addr, _) = repl.reduce_aux(&addr_expr)?; + if addr.tag != Tag::Str { + bail!("Address must be a string"); + } + let (id, _) = repl.reduce_aux(&id_expr)?; + if !matches!(id.tag, Tag::Comm | Tag::BigNum) { + bail!("ID must be a commitment or a big num"); + } + let addr_str = repl.zstore.fetch_string(&addr); + Self::set_microchain(repl, addr_str, id.digest)?; + Ok(*repl.zstore.t()) + }, + }; + + const MICROCHAIN_GET_INFO: Self = Self { + name: "microchain-get-info", + summary: "Retrieves the REPL microchain address and ID", + info: &[], + format: "!(microchain-get-info)", + example: &["!(microchain-get-info)"], + returns: "The address/ID pair for the REPL microchain", + run: |repl, args, _dir| { + if args != repl.zstore.nil() { + bail!("Arguments aren't supported"); + } + let Some((addr, id)) = &repl.microchain else { + bail!("No microchain info set"); + }; + let addr = repl.zstore.intern_string(addr); + let id = repl.zstore.intern_comm(*id); + Ok(repl.zstore.intern_cons(addr, id)) + }, + }; + fn build_comm_data(repl: &mut Repl, digest: &[F]) -> CommData { - let inv_hashes3 = repl.queries.get_inv_queries("hash3", &repl.toplevel); - let callable_preimg = inv_hashes3 - .get(digest) - .expect("Missing commitment preimage"); - let mut secret = [F::zero(); DIGEST_SIZE]; - secret.copy_from_slice(&callable_preimg[..DIGEST_SIZE]); - let payload = ZPtr::from_flat_data(&callable_preimg[DIGEST_SIZE..]); + let inv_comms = repl.queries.get_inv_queries("commit", &repl.toplevel); + let callable_preimg = inv_comms.get(digest).expect("Missing commitment preimage"); + let (secret, payload) = Self::split_comm_data_preimg(callable_preimg); repl.memoize_dag(&payload); CommData::new(secret, payload, &repl.zstore) } + /// Computes a microchain ID by delegating the hashing to the REPL's reduction + /// so the commitment is persisted automatically when in a scope. + fn compute_microchain_id( + repl: &mut Repl, + id_secret: [F; DIGEST_SIZE], + genesis: ZPtr, + ) -> Result> { + let hide = repl + .zstore + .intern_symbol(&builtin_sym("hide"), &repl.lang_symbols); + let secret = repl.zstore.intern_big_num(id_secret); + let genesis_quoted = repl.zstore.intern_quoted(genesis); + let expr = repl.zstore.intern_list([hide, secret, genesis_quoted]); + let (id, _) = repl.reduce_aux(&expr)?; + Ok(id) + } + const MICROCHAIN_START: Self = Self { name: "microchain-start", summary: "Starts a new microchain and returns the resulting ID", info: &[ "A microchain ID is a hiding commitment to the genesis state, using", - "a timestamp-based secret generated in the server.", + "a secret generated in the server.", "Upon success, it becomes possible to open the ID and retrieve genesis", "state associated with the microchain.", + "Starting a microchain attempts to set the REPL microchain with the", + "corresponding info.", ], format: "!(microchain-start )", example: &[ @@ -1236,58 +1471,56 @@ impl, C2: Chipset> MetaCmd { }; let addr_str = repl.zstore.fetch_string(&addr); - let stream = &mut TcpStream::connect(addr_str)?; + let stream = &mut TcpStream::connect(&addr_str)?; write_data(stream, Request::Start(genesis))?; let Response::IdSecret(id_secret) = read_data(stream)? else { bail!("Could not read ID secret from server"); }; - let id_digest = CommData::hash(&id_secret, &state, &mut repl.zstore); + let id = Self::compute_microchain_id(repl, id_secret, state)?; + + // Failing to set the microchain should not bail. + let _ = Self::set_microchain(repl, addr_str, id.digest); - let id = repl.zstore.intern_comm(id_digest); Ok(id) }, }; + /// Sends a generic "get state" request to the REPL microchain. fn send_get_state_request( repl: &mut Repl, - args: &ZPtr, mk_request: fn([F; DIGEST_SIZE]) -> Request, ) -> Result { - let [&addr_expr, &id_expr] = repl.take(args)?; - let (addr, _) = repl.reduce_aux(&addr_expr)?; - if addr.tag != Tag::Str { - bail!("Address must be a string"); - } - let (id, _) = repl.reduce_aux(&id_expr)?; - let addr_str = repl.zstore.fetch_string(&addr); - let mut stream = TcpStream::connect(addr_str)?; - write_data(&mut stream, mk_request(id.digest))?; + let Some((addr, id)) = &repl.microchain else { + bail!("No microchain info set"); + }; + let mut stream = TcpStream::connect(addr)?; + write_data(&mut stream, mk_request(*id))?; Ok(stream) } const MICROCHAIN_GET_GENESIS: Self = Self { name: "microchain-get-genesis", - summary: "Returns the genesis state of a microchain", + summary: "Returns the genesis state of the REPL microchain", info: &[ "Similarly to `microchain-start`, the preimage of the ID becomes", "available so opening the ID returns the genesis state.", ], - format: "!(microchain-get-genesis )", - example: &[ - "!(defq state0 !(microchain-get-genesis \"127.0.0.1:1234\" #c0x123))", - "!(assert-eq state0 (open #c0x123))", - ], + format: "!(microchain-get-genesis)", + example: &["!(defq state0 !(microchain-get-genesis))"], returns: "The microchain's genesis state", run: |repl, args, _dir| { - let mut stream = Self::send_get_state_request(repl, args, Request::GetGenesis)?; + if args != repl.zstore.nil() { + bail!("Arguments aren't supported"); + } + let mut stream = Self::send_get_state_request(repl, Request::GetGenesis)?; let Response::Genesis(id_secret, chain_state) = read_data(&mut stream)? else { bail!("Could not read state from server"); }; let state = chain_state.into_zptr(&mut repl.zstore); - // memoize preimg so it's possible to open the ID - CommData::hash(&id_secret, &state, &mut repl.zstore); + // Memoize preimg so it's possible to open the ID. + Self::compute_microchain_id(repl, id_secret, state)?; Ok(state) }, @@ -1295,13 +1528,16 @@ impl, C2: Chipset> MetaCmd { const MICROCHAIN_GET_STATE: Self = Self { name: "microchain-get-state", - summary: "Returns the current state of a microchain", + summary: "Returns the current state of the REPL microchain", info: &[], - format: "!(microchain-get-state )", - example: &["!(microchain-get-state \"127.0.0.1:1234\" #c0x123)"], + format: "!(microchain-get-state)", + example: &["!(microchain-get-state)"], returns: "The microchain's latest state", run: |repl, args, _dir| { - let mut stream = Self::send_get_state_request(repl, args, Request::GetState)?; + if args != repl.zstore.nil() { + bail!("Arguments aren't supported"); + } + let mut stream = Self::send_get_state_request(repl, Request::GetState)?; let Response::State(chain_state) = read_data(&mut stream)? else { bail!("Could not read state from server"); }; @@ -1315,18 +1551,14 @@ impl, C2: Chipset> MetaCmd { summary: "Proves a state transition via chaining and sends the proof to a microchain server", info: &["The transition is successful iff the proof is accepted by the server."], - format: "!(microchain-transition ...)", - example: &["!(microchain-transition \"127.0.0.1:1234\" #c0x123 state arg0 arg1)"], + format: "!(microchain-transition ...)", + example: &["!(microchain-transition state arg0 arg1)"], returns: "The new state", run: |repl, args, _dir| { - let (&addr_expr, rest) = repl.car_cdr(args); - let (&id_expr, &rest) = repl.car_cdr(rest); - let (addr, _) = repl.reduce_aux(&addr_expr)?; - if addr.tag != Tag::Str { - bail!("Address must be a string"); - } - let (id, _) = repl.reduce_aux(&id_expr)?; - let (¤t_state_expr, &call_args) = repl.car_cdr(&rest); + let Some((addr, id)) = repl.microchain.clone() else { + bail!("No microchain info set"); + }; + let (¤t_state_expr, &call_args) = repl.car_cdr(args); let (state, call_args) = Self::transition_call(repl, ¤t_state_expr, call_args)?; if state.tag != Tag::Cons { bail!("New state is not a pair"); @@ -1345,15 +1577,14 @@ impl, C2: Chipset> MetaCmd { CallableData::Fun(LurkData::new(state_callable, &repl.zstore)) }; - let chain_proof = ChainProof { + let proof = ChainProof { crypto_proof, call_args, next_chain_result, next_callable, }; - let addr_str = repl.zstore.fetch_string(&addr); - let stream = &mut TcpStream::connect(addr_str)?; - write_data(stream, Request::Transition(id.digest, chain_proof))?; + let stream = &mut TcpStream::connect(addr)?; + write_data(stream, Request::Transition { id, proof })?; match read_data::(stream)? { Response::ProofAccepted => { println!("Proof accepted by the server"); @@ -1377,16 +1608,14 @@ impl, C2: Chipset> MetaCmd { name: "microchain-verify", summary: "Checks if a series of microchain transition proofs takes state A to B", info: &["The state arguments are meant to be the genesis and the current state."], - format: "!(microchain-verify )", - example: &["!(microchain-verify \"127.0.0.1:1234\" #c0x123 genesis current)"], + format: "!(microchain-verify )", + example: &["!(microchain-verify genesis current)"], returns: "t", run: |repl, args, _dir| { - let [&addr_expr, &id_expr, &initial_state_expr, &final_state_expr] = repl.take(args)?; - let (addr, _) = repl.reduce_aux(&addr_expr)?; - if addr.tag != Tag::Str { - bail!("Address must be a string"); - } - let (id, _) = repl.reduce_aux(&id_expr)?; + let Some((addr, id)) = repl.microchain.clone() else { + bail!("No microchain info set"); + }; + let [&initial_state_expr, &final_state_expr] = repl.take(args)?; let (initial_state, _) = repl.reduce_aux(&initial_state_expr)?; if initial_state.tag != Tag::Cons { bail!("Initial state must be a pair"); @@ -1395,11 +1624,14 @@ impl, C2: Chipset> MetaCmd { if final_state.tag != Tag::Cons { bail!("Final state must be a pair"); } - let addr_str = repl.zstore.fetch_string(&addr); - let stream = &mut TcpStream::connect(addr_str)?; + let stream = &mut TcpStream::connect(addr)?; write_data( stream, - Request::GetProofs(id.digest, initial_state.digest, final_state.digest), + Request::GetProofs { + id, + initial_state_digest: initial_state.digest, + final_state_digest: final_state.digest, + }, )?; let Response::Proofs(proofs) = read_data(stream)? else { bail!("Could not read proofs from server"); @@ -1530,6 +1762,11 @@ pub(crate) fn meta_cmds, C2: Chipset>() -> MetaCmdsMap { let zptr = comm_data.commit(zstore); - comm_data.populate_zstore(zstore); + comm_data.zdag.populate_zstore(zstore); zptr } CallableData::Fun(lurk_data) => lurk_data.populate_zstore(zstore), @@ -91,11 +90,24 @@ impl ChainState { #[derive(Serialize, Deserialize)] pub(crate) enum Request { + /// Spawn a new microchain with a given genesis state Start(ChainState), + /// Request the genesis state of a microchain by ID GetGenesis([F; DIGEST_SIZE]), + /// Request the current state of a microchain by ID GetState([F; DIGEST_SIZE]), - Transition([F; DIGEST_SIZE], ChainProof), - GetProofs([F; DIGEST_SIZE], [F; DIGEST_SIZE], [F; DIGEST_SIZE]), + /// Provide a proof of state transition for a microchain + Transition { + id: [F; DIGEST_SIZE], + proof: ChainProof, + }, + /// Request the sequence of proofs from a microchain that can prove the transition + /// from an initial state to a final state. States are referenced by their digests + GetProofs { + id: [F; DIGEST_SIZE], + initial_state_digest: [F; DIGEST_SIZE], + final_state_digest: [F; DIGEST_SIZE], + }, } #[derive(Serialize, Deserialize)] @@ -171,7 +183,7 @@ impl MicrochainArgs { }; return_msg!(Response::State(state)); } - Request::Transition(id, chain_proof) => { + Request::Transition { id, proof } => { let (Ok(mut proofs), Ok(state)) = (load_proofs(&id), load_state(&id)) else { return_msg!(Response::NoDataForId); @@ -182,7 +194,7 @@ impl MicrochainArgs { call_args, next_chain_result, next_callable, - } = chain_proof; + } = proof; let next_chain_result_zptr = { if next_chain_result.is_flawed(&mut zstore) { @@ -262,78 +274,22 @@ impl MicrochainArgs { return_msg!(Response::ProofAccepted); } - Request::GetProofs(id, initial_digest, final_digest) => { + Request::GetProofs { + id, + initial_state_digest: initial_digest, + final_state_digest: final_digest, + } => { let Ok(mut proofs) = load_proofs(&id) else { return_msg!(Response::NoDataForId); }; - // let proof_index = load_proof_index(&id)?; - // let Some(initial_index) = proof_index.index_by_prev(&initial_digest) else { - // return_msg!(Response::NoProofForInitialState); - // }; - // let Some(final_index) = proof_index.index_by_next(&final_digest) else { - // return_msg!(Response::NoProofForFinalState); - // }; - - // the following code snippet is only meant to support version transitioning - // and should be eliminated (in favor of the code above) once legacy microchains - // are dropped - let proof_index = load_proof_index(&id).unwrap_or_default(); - let initial_index = - if let Some(index) = proof_index.index_by_prev(&initial_digest) { - index - } else { - let (_, genesis_state) = load_genesis(&id)?; - let genesis_result_zptr = genesis_state.chain_result.zptr; - let genesis_callable_zptr = - genesis_state.callable_data.zptr(&mut zstore); - let genesis_zptr = zstore - .intern_cons(genesis_result_zptr, genesis_callable_zptr); - if genesis_zptr.digest == initial_digest { - 0 - } else { - let mut index = None; - for (i, proof) in proofs.iter().enumerate() { - let OpaqueChainProof { - next_chain_result, - next_callable, - .. - } = proof; - let next_state = zstore - .intern_cons(*next_chain_result, *next_callable); - if next_state.digest == initial_digest { - index = Some(i + 1); - break; - } - } - let Some(index) = index else { - return_msg!(Response::NoProofForInitialState); - }; - index - } - }; - let final_index = - if let Some(index) = proof_index.index_by_next(&final_digest) { - index - } else { - let mut index = None; - for (i, proof) in proofs.iter().enumerate() { - let OpaqueChainProof { - next_chain_result, - next_callable, - .. - } = proof; - let next_state = - zstore.intern_cons(*next_chain_result, *next_callable); - if next_state.digest == final_digest { - index = Some(i); - break; - } - } - let Some(index) = index else { - return_msg!(Response::NoProofForFinalState); - }; - index - }; + let proof_index = load_proof_index(&id)?; + let Some(initial_index) = proof_index.index_by_prev(&initial_digest) + else { + return_msg!(Response::NoProofForInitialState); + }; + let Some(final_index) = proof_index.index_by_next(&final_digest) else { + return_msg!(Response::NoProofForFinalState); + }; proofs.truncate(final_index + 1); proofs.drain(..initial_index); @@ -349,6 +305,24 @@ impl MicrochainArgs { } } +pub(crate) fn read_data Deserialize<'a>>(stream: &mut TcpStream) -> Result { + let mut size_bytes = [0; 8]; + stream.read_exact(&mut size_bytes)?; + let size = usize::from_le_bytes(size_bytes); + let mut data_buffer = vec![0; size]; + stream.read_exact(&mut data_buffer)?; + let data = bincode::deserialize(&data_buffer)?; + Ok(data) +} + +pub(crate) fn write_data(stream: &mut TcpStream, data: T) -> Result<()> { + let data_bytes = bincode::serialize(&data)?; + stream.write_all(&data_bytes.len().to_le_bytes())?; + stream.write_all(&data_bytes)?; + stream.flush()?; + Ok(()) +} + /// Holds indices of proofs in a sequence of state transitions. The index of a /// proof can be looked up by the digest of the previous state or by the digest /// of the next state. @@ -378,67 +352,39 @@ impl ProofIndex { } } -fn dump_microchain_data(id: &[F], name: &str, data: &T) -> Result<()> { - let hash = format!("{:x}", field_elts_to_biguint(id)); - let dir = microchains_dir()?.join(hash); - std::fs::create_dir_all(&dir)?; - std::fs::write(dir.join(name), bincode::serialize(data)?)?; - Ok(()) -} +const GENESIS_FILE: &str = "genesis"; +const PROOFS_FILE: &str = "proofs"; +const STATE_FILE: &str = "state"; +const PROOF_INDEX_FILE: &str = "proof_index"; fn dump_genesis(id: &[F], genesis: &Genesis) -> Result<()> { - dump_microchain_data(id, "genesis", genesis) + dump_to_hash_dir(µchains_dir(), id, GENESIS_FILE, genesis) } fn dump_proofs(id: &[F], proofs: &[OpaqueChainProof]) -> Result<()> { - dump_microchain_data(id, "proofs", proofs) + dump_to_hash_dir(µchains_dir(), id, PROOFS_FILE, proofs) } fn dump_state(id: &[F], state: &ChainState) -> Result<()> { - dump_microchain_data(id, "state", state) + dump_to_hash_dir(µchains_dir(), id, STATE_FILE, state) } fn dump_proof_index(id: &[F], proof_index: &ProofIndex) -> Result<()> { - dump_microchain_data(id, "proof_index", proof_index) -} - -fn load_microchain_data Deserialize<'a>>(id: &[F], name: &str) -> Result { - let hash = format!("{:x}", field_elts_to_biguint(id)); - let bytes = std::fs::read(microchains_dir()?.join(hash).join(name))?; - let data = bincode::deserialize(&bytes)?; - Ok(data) + dump_to_hash_dir(µchains_dir(), id, PROOF_INDEX_FILE, proof_index) } fn load_genesis(id: &[F]) -> Result { - load_microchain_data(id, "genesis") + load_from_hash_dir(µchains_dir(), id, GENESIS_FILE) } fn load_proofs(id: &[F]) -> Result> { - load_microchain_data(id, "proofs") + load_from_hash_dir(µchains_dir(), id, PROOFS_FILE) } fn load_state(id: &[F]) -> Result { - load_microchain_data(id, "state") + load_from_hash_dir(µchains_dir(), id, STATE_FILE) } fn load_proof_index(id: &[F]) -> Result> { - load_microchain_data(id, "proof_index") -} - -pub(crate) fn read_data Deserialize<'a>>(stream: &mut TcpStream) -> Result { - let mut size_bytes = [0; 8]; - stream.read_exact(&mut size_bytes)?; - let size = usize::from_le_bytes(size_bytes); - let mut data_buffer = vec![0; size]; - stream.read_exact(&mut data_buffer)?; - let data = bincode::deserialize(&data_buffer)?; - Ok(data) -} - -pub(crate) fn write_data(stream: &mut TcpStream, data: T) -> Result<()> { - let data_bytes = bincode::serialize(&data)?; - stream.write_all(&data_bytes.len().to_le_bytes())?; - stream.write_all(&data_bytes)?; - stream.flush()?; - Ok(()) + load_from_hash_dir(µchains_dir(), id, PROOF_INDEX_FILE) } diff --git a/src/core/cli/mod.rs b/src/core/cli/mod.rs index 5cd07602..2c36b25f 100644 --- a/src/core/cli/mod.rs +++ b/src/core/cli/mod.rs @@ -8,6 +8,7 @@ mod paths; mod proofs; mod rdg; pub mod repl; +mod scope; #[cfg(test)] mod tests; mod zdag; diff --git a/src/core/cli/paths.rs b/src/core/cli/paths.rs index 7828af29..3cb58814 100644 --- a/src/core/cli/paths.rs +++ b/src/core/cli/paths.rs @@ -1,7 +1,11 @@ use anyhow::Result; -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; +use p3_field::PrimeField; +use serde::{Deserialize, Serialize}; use std::path::Path; +use crate::core::big_num::field_elts_to_biguint; + use super::config::get_config; #[inline] @@ -21,6 +25,11 @@ pub(crate) fn lurk_dir() -> Result<&'static Utf8PathBuf> { create_dir_all_and_return(&get_config().lurk_dir) } +#[inline] +pub(crate) fn repl_history() -> Result { + Ok(lurk_dir()?.join("repl-history")) +} + #[inline] pub(crate) fn proofs_dir() -> Result { create_dir_all_and_return(get_config().lurk_dir.join("proofs")) @@ -32,11 +41,48 @@ pub(crate) fn commits_dir() -> Result { } #[inline] -pub(crate) fn microchains_dir() -> Result { - create_dir_all_and_return(get_config().lurk_dir.join("microchains")) +pub(crate) fn microchains_dir() -> Utf8PathBuf { + get_config().lurk_dir.join("microchains") } #[inline] -pub(crate) fn repl_history() -> Result { - Ok(lurk_dir()?.join("repl-history")) +pub(crate) fn scopes_dir() -> Utf8PathBuf { + get_config().lurk_dir.join("scopes") +} + +pub(crate) fn dump_to_hash_dir( + outer_dir: &Utf8PathBuf, + hash: &[F], + name: impl AsRef, + data: &T, +) -> Result<()> { + let hash = format!("{:x}", field_elts_to_biguint(hash)); + let dir = outer_dir.join(hash); + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(name), bincode::serialize(data)?)?; + Ok(()) +} + +pub(crate) fn load_from_hash_dir Deserialize<'a>>( + outer_dir: &Utf8PathBuf, + hash: &[F], + name: impl AsRef, +) -> Result { + let hash = format!("{:x}", field_elts_to_biguint(hash)); + let bytes = std::fs::read(outer_dir.join(hash).join(name))?; + let data = bincode::deserialize(&bytes)?; + Ok(data) +} + +pub(crate) fn remove_from_hash_dir( + outer_dir: &Utf8PathBuf, + hash: &[F], + name: impl AsRef, +) -> Result<()> { + let hash = format!("{:x}", field_elts_to_biguint(hash)); + let path = outer_dir.join(hash).join(name); + if path.exists() { + std::fs::remove_file(path)?; + } + Ok(()) } diff --git a/src/core/cli/repl.rs b/src/core/cli/repl.rs index 02d96ffe..66f76367 100644 --- a/src/core/cli/repl.rs +++ b/src/core/cli/repl.rs @@ -23,6 +23,7 @@ use crate::{ big_num::field_elts_to_biguint, chipset::LurkChip, cli::{ + comm_data::CommData, debug::{FormattedDebugData, FormattedDebugEntry}, meta::{meta_cmds, MetaCmdsMap}, paths::{current_dir, proofs_dir, repl_history}, @@ -43,12 +44,15 @@ use crate::{ }, lair::{ chipset::{Chipset, NoChip}, - execute::{DebugEntry, DebugEntryKind, QueryRecord, QueryResult, Shard}, + execute::{DebugEntry, DebugEntryKind, QueryMap, QueryRecord, QueryResult, Shard}, lair_chip::LairMachineProgram, toplevel::Toplevel, + FxIndexMap, List, }, }; +use super::scope::update_scope_comms; + #[derive(Helper, Highlighter, Hinter, Completer)] struct InputValidator { state: StateRcCell, @@ -105,9 +109,10 @@ struct ProcessedDebugEntry { } /// Holds the indices of some functions in the Lurk toplevel -struct FuncIndices { +pub(crate) struct FuncIndices { lurk_main: usize, eval: usize, + pub(crate) commit: usize, egress: usize, } @@ -116,20 +121,36 @@ impl FuncIndices { Self { lurk_main: toplevel.func_by_name("lurk_main").index, eval: toplevel.func_by_name("eval").index, + commit: toplevel.func_by_name("commit").index, egress: toplevel.func_by_name("egress").index, } } } +/// Address and ID of a microchain +pub(crate) type MicrochainInfo = (String, [F; DIGEST_SIZE]); + +/// Holds data for the outer scope, needed when a scope is popped. +pub(crate) struct OuterScope { + pub(crate) env: ZPtr, + pub(crate) comms: FxHashSet>, + pub(crate) microchain: Option>, +} + pub(crate) struct Repl, C2: Chipset> { pub(crate) zstore: ZStore, pub(crate) queries: QueryRecord, pub(crate) toplevel: Toplevel, - func_indices: FuncIndices, + pub(crate) func_indices: FuncIndices, pub(crate) env: ZPtr, pub(crate) state: StateRcCell, pub(crate) meta_cmds: MetaCmdsMap, pub(crate) lang_symbols: FxHashSet, + pub(crate) microchain: Option>, + /// A sequenced map from scope names (symbols) to their immediate outer scopes. + /// This could arguably be a `Vec`, but an `IndexMap` provides a fast way to + /// detect potential scoping cycles. + pub(crate) scopes: FxIndexMap, OuterScope>, } impl> Repl { @@ -146,6 +167,8 @@ impl> Repl { state: State::init_lurk_state().rccell(), meta_cmds: meta_cmds(), lang_symbols, + microchain: None, + scopes: FxIndexMap::default(), } } } @@ -167,10 +190,8 @@ impl, C2: Chipset> Repl { let Some(public_values) = self.queries.public_values.as_ref() else { bail!("No data found for latest computation"); }; - let proof_key_img: &[BabyBear; DIGEST_SIZE] = &self - .zstore - .hash3(public_values[..INPUT_SIZE].try_into().unwrap()); - let proof_key = format!("{:x}", field_elts_to_biguint(proof_key_img)); + let proof_key_img = self.zstore.hasher.hash(&public_values[..INPUT_SIZE]); + let proof_key = format!("{:x}", field_elts_to_biguint(&proof_key_img)); let proof_path = proofs_dir()?.join(&proof_key); let machine = new_machine(&self.toplevel); let (pk, vk) = machine.setup(&LairMachineProgram); @@ -263,13 +284,12 @@ impl, C2: Chipset> Repl { self.zstore.fmt_with_state(&self.state, zptr) } + /// Feeds the `QueryRecord` with new preimages that might be necessary for + /// the ingress phase of an incoming Lair execution. fn prepare_queries(&mut self) { self.queries.clean(); - let hashes3 = std::mem::take(&mut self.zstore.hashes3_diff); let hashes4 = std::mem::take(&mut self.zstore.hashes4_diff); let hashes5 = std::mem::take(&mut self.zstore.hashes5_diff); - self.queries - .inject_inv_queries_owned("hash3", &self.toplevel, hashes3); self.queries .inject_inv_queries_owned("hash4", &self.toplevel, hashes4); self.queries @@ -310,11 +330,7 @@ impl, C2: Chipset> Repl { fn tmp_queries_for_egression(&self) -> QueryRecord { let mut inv_func_queries = Vec::with_capacity(self.queries.inv_func_queries.len()); for inv_query_map in &self.queries.inv_func_queries { - if inv_query_map.is_some() { - inv_func_queries.push(Some(Default::default())); - } else { - inv_func_queries.push(None); - } + inv_func_queries.push(inv_query_map.as_ref().map(|_| Default::default())); } QueryRecord { public_values: None, @@ -429,6 +445,47 @@ impl, C2: Chipset> Repl { } } + /// Persists commitment data and returns the corresponding commitment + pub(crate) fn persist_comm_data( + &mut self, + secret: [F; DIGEST_SIZE], + payload: ZPtr, + ) -> Result> { + self.memoize_dag(&payload); + let comm_data = CommData::new(secret, payload, &self.zstore); + comm_data.dump(&mut self.zstore) + } + + #[inline] + pub(crate) fn current_scope(&self) -> Option<&ZPtr> { + self.scopes.last().map(|(scope, _)| scope) + } + + /// Retrieve commitment data from calls to `commit` or `hide`. As a side-effect, + /// persist the updated set of commitments for the scope at hand. + fn collect_scopped_commitments( + scope_digest: &[F], + comms_queries: &QueryMap, + ) -> Result)>> { + let mut collected = Vec::with_capacity(comms_queries.len()); + update_scope_comms(scope_digest, |mut scope_comms| { + for (preimg, result) in comms_queries { + if !result.direct_call { + // Skip queries that weren't computed via `commit` or `hide`. + // That is, `open` and `secret` don't count. + continue; + } + scope_comms.insert(result.expect_output().into()); + let mut secret = [F::zero(); DIGEST_SIZE]; + secret.copy_from_slice(&preimg[..DIGEST_SIZE]); + let payload = ZPtr::from_flat_data(&preimg[DIGEST_SIZE..]); + collected.push((secret, payload)); + } + Ok(scope_comms) + })?; + Ok(collected) + } + /// Reduces a Lurk expression with a clone of the REPL's queries so the latest /// provable computation isn't affected. After the reduction is over, retrieve /// the (potentially enriched) inverse query maps so commitments aren't lost. @@ -445,6 +502,15 @@ impl, C2: Chipset> Repl { &mut queries_tmp, None, ); + + if let Some(&scope) = self.current_scope() { + let comms_queries = queries_tmp.get_queries("commit", &self.toplevel); + let commitments = Self::collect_scopped_commitments(&scope.digest, comms_queries)?; + for (secret, payload) in commitments { + self.persist_comm_data(secret, payload)?; + } + } + let emitted_raw_vec = std::mem::take(&mut queries_tmp.emitted); let mut emitted = Vec::with_capacity(emitted_raw_vec.len()); for emitted_raw in &emitted_raw_vec { @@ -472,6 +538,15 @@ impl, C2: Chipset> Repl { &mut self.queries, Some(self.func_indices.eval), ); + + if let Some(&scope) = self.current_scope() { + let comms_queries = self.queries.get_queries("commit", &self.toplevel); + let commitments = Self::collect_scopped_commitments(&scope.digest, comms_queries)?; + for (secret, payload) in commitments { + self.persist_comm_data(secret, payload)?; + } + } + if !self.queries.emitted.is_empty() { let mut queries_tmp = self.tmp_queries_for_egression(); let mut emitted = Vec::with_capacity(self.queries.emitted.len()); @@ -639,10 +714,9 @@ impl, C2: Chipset> Repl { } } - pub(crate) fn init_config() -> Config { + fn init_config() -> Config { let var = std::env::var("EDITOR"); - let is_vi = |var: String| VI_EDITORS.iter().any(|&x| x == var.as_str()); - if let Ok(true) = var.map(is_vi) { + if let Ok(true) = var.map(|val| VI_EDITORS.contains(&val.as_str())) { Config::builder().edit_mode(EditMode::Vi).build() } else { Config::default() diff --git a/src/core/cli/scope.rs b/src/core/cli/scope.rs new file mode 100644 index 00000000..0e3977be --- /dev/null +++ b/src/core/cli/scope.rs @@ -0,0 +1,120 @@ +//! This module implements the persistency layer to support scope-related meta commands. +//! +//! One remarkable note is that, even though scopes are named after symbols, their +//! data have to be persisted on directories named after symbol digests because +//! symbols paths may contain characters that aren't allowed for folder names. + +use anyhow::Result; +use p3_field::PrimeField; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::{Deserialize, Serialize}; +use std::hash::Hash; + +use crate::{ + core::{ + cli::{repl::MicrochainInfo, zdag::ZDag}, + zstore::{ZPtr, ZStore}, + }, + lair::{chipset::Chipset, List}, +}; + +use super::paths::{dump_to_hash_dir, load_from_hash_dir, remove_from_hash_dir, scopes_dir}; + +#[derive(Serialize, Deserialize, Default)] +pub(crate) struct ScopeBindings { + pub(crate) binds: FxHashMap, ZPtr>, + pub(crate) zdag: ZDag, +} + +impl ScopeBindings { + // FIXME: `zdag` may contain dead unreachable data if a binding is shadowed. + pub(crate) fn bind>( + &mut self, + sym: ZPtr, + val: ZPtr, + zstore: &ZStore, + ) { + self.zdag.populate_with_many([&sym, &val], zstore); + self.binds.insert(sym, val); + } +} + +const BINDINGS_FILE: &str = "bindings"; +const COMMS_FILE: &str = "comms"; +const MICROCHAIN_FILE: &str = "microchain"; + +// Scope commitments + +pub(crate) fn update_scope_comms(scope_digest: &[F], mut f: T) -> Result<()> +where + T: FnMut(FxHashSet>) -> Result>>, +{ + let comms = load_scope_comms(scope_digest).unwrap_or_default(); + let comms = f(comms)?; + if comms.is_empty() { + remove_scope_comms(scope_digest) + } else { + dump_scope_comms(scope_digest, &comms) + } +} + +#[inline] +pub(crate) fn load_scope_comms(scope_digest: &[F]) -> Result>> { + load_from_hash_dir(&scopes_dir(), scope_digest, COMMS_FILE) +} + +fn dump_scope_comms(scope_digest: &[F], comms: &FxHashSet>) -> Result<()> { + dump_to_hash_dir(&scopes_dir(), scope_digest, COMMS_FILE, comms) +} + +fn remove_scope_comms(scope_digest: &[F]) -> Result<()> { + remove_from_hash_dir(&scopes_dir(), scope_digest, COMMS_FILE) +} + +// Scope bindings + +pub(crate) fn update_scope_bindings(scope_digest: &[F], mut f: T) -> Result<()> +where + T: FnMut(ScopeBindings) -> Result>, +{ + let bindings = load_scope_bindings(scope_digest).unwrap_or_default(); + let bindings = f(bindings)?; + if bindings.binds.is_empty() { + remove_scope_bindings(scope_digest) + } else { + dump_scope_bindings(scope_digest, &bindings) + } +} + +#[inline] +pub(crate) fn load_scope_bindings(scope_digest: &[F]) -> Result> { + load_from_hash_dir(&scopes_dir(), scope_digest, BINDINGS_FILE) +} + +fn dump_scope_bindings( + scope_digest: &[F], + scope_bindings: &ScopeBindings, +) -> Result<()> { + dump_to_hash_dir(&scopes_dir(), scope_digest, BINDINGS_FILE, scope_bindings) +} + +fn remove_scope_bindings(scope_digest: &[F]) -> Result<()> { + remove_from_hash_dir(&scopes_dir(), scope_digest, BINDINGS_FILE) +} + +// Scope microchain + +#[inline] +pub(crate) fn load_scope_microchain( + scope_digest: &[F], +) -> Result>> { + load_from_hash_dir(&scopes_dir(), scope_digest, MICROCHAIN_FILE) +} + +#[inline] +pub(crate) fn dump_scope_microchain( + scope_digest: &[F], + microchain: &Option>, +) -> Result<()> { + dump_to_hash_dir(&scopes_dir(), scope_digest, MICROCHAIN_FILE, microchain) +} diff --git a/src/core/eval_compiled.rs b/src/core/eval_compiled.rs index 258441b0..f6962cb6 100644 --- a/src/core/eval_compiled.rs +++ b/src/core/eval_compiled.rs @@ -27,7 +27,7 @@ use super::{ ingress::{egress, ingress, preallocate_symbols, SymbolsDigests}, lang::{Coroutine, Lang}, misc::{ - big_num_lessthan, digest_equal, hash3, hash4, hash5, u64_add, u64_divrem, u64_iszero, + big_num_lessthan, commit, digest_equal, hash4, hash5, u64_add, u64_divrem, u64_iszero, u64_lessthan, u64_mul, u64_sub, }, symbol::Symbol, @@ -45,7 +45,7 @@ fn native_lurk_funcs( // Builtins preallocate_symbols(digests), // External chip wrappers - hash3(), + commit(), hash4(), hash5(), u64_add(), @@ -376,7 +376,7 @@ pub fn eval_unop(digests: &SymbolsDigests) -> FuncE { Tag::Comm, Tag::BigNum => { let comm_hash: [8] = load(arg); let (secret: [8], tag, padding: [7], arg_digest: [8]) = preimg( - hash3, + commit, comm_hash, |fs| format!("Preimage not found for #{:#x}", field_elts_to_biguint(fs)) ); @@ -502,7 +502,7 @@ pub fn eval_binop(digests: &SymbolsDigests) -> FuncE { let secret: [8] = load(val1); let (val2_tag, val2_digest: [8]) = call(egress, val2_tag, val2); let padding = [0; 7]; - let comm_hash: [8] = call(hash3, secret, val2_tag, padding, val2_digest); + let comm_hash: [8] = call(commit, secret, val2_tag, padding, val2_digest); let comm_ptr = store(comm_hash); let comm_tag = Tag::Comm; return (comm_tag, comm_ptr) diff --git a/src/core/eval_direct.rs b/src/core/eval_direct.rs index 2fc0f4a0..f46e6168 100644 --- a/src/core/eval_direct.rs +++ b/src/core/eval_direct.rs @@ -18,7 +18,7 @@ use super::{ ingress::{egress, ingress, preallocate_symbols, InternalTag, SymbolsDigests}, lang::{Coroutine, Lang}, misc::{ - big_num_lessthan, digest_equal, hash3, hash4, hash5, u64_add, u64_divrem, u64_iszero, + big_num_lessthan, commit, digest_equal, hash4, hash5, u64_add, u64_divrem, u64_iszero, u64_lessthan, u64_mul, u64_sub, }, symbol::Symbol, @@ -59,7 +59,7 @@ fn native_lurk_funcs( env_lookup(), ingress(digests), egress(digests), - hash3(), + commit(), hash4(), hash5(), u64_add(), @@ -919,7 +919,7 @@ pub fn open_comm() -> FuncE { fn open_comm(hash_ptr): [2] { let comm_hash: [8] = load(hash_ptr); let (_secret: [8], payload_tag, padding: [7], val_digest: [8]) = preimg( - hash3, + commit, comm_hash, |fs| format!("Preimage not found for #{:#x}", field_elts_to_biguint(fs)) ); @@ -1527,7 +1527,7 @@ pub fn eval_opening_unop(digests: &SymbolsDigests) -> FuncE< let (val_tag, val_digest: [8]) = call(egress, val_tag, val); let padding = [0; 7]; let zero = 0; - let comm_hash: [8] = call(hash3, zero, padding, val_tag, padding, val_digest); + let comm_hash: [8] = call(commit, zero, padding, val_tag, padding, val_digest); let comm_tag = Tag::Comm; let comm_ptr = store(comm_hash); return (comm_tag, comm_ptr) @@ -1538,7 +1538,7 @@ pub fn eval_opening_unop(digests: &SymbolsDigests) -> FuncE< Tag::Comm, Tag::BigNum => { let comm_hash: [8] = load(val); let (secret: [8], tag, padding: [7], val_digest: [8]) = preimg( - hash3, + commit, comm_hash, |fs| format!("Preimage not found for #{:#x}", field_elts_to_biguint(fs)) ); @@ -1599,7 +1599,7 @@ pub fn eval_hide() -> FuncE { let secret: [8] = load(val1); let (val2_tag, val2_digest: [8]) = call(egress, val2_tag, val2); let padding = [0; 7]; - let comm_hash: [8] = call(hash3, secret, val2_tag, padding, val2_digest); + let comm_hash: [8] = call(commit, secret, val2_tag, padding, val2_digest); let comm_ptr = store(comm_hash); let comm_tag = Tag::Comm; return (comm_tag, comm_ptr) @@ -2007,7 +2007,7 @@ mod test { let env_lookup = FuncChip::from_name("env_lookup", toplevel); let ingress = FuncChip::from_name("ingress", toplevel); let egress = FuncChip::from_name("egress", toplevel); - let hash3 = FuncChip::from_name("hash3", toplevel); + let commit = FuncChip::from_name("commit", toplevel); let hash4 = FuncChip::from_name("hash4", toplevel); let hash5 = FuncChip::from_name("hash5", toplevel); let u64_add = FuncChip::from_name("u64_add", toplevel); @@ -2050,7 +2050,7 @@ mod test { expect_eq(env_lookup.width(), expect!["52"]); expect_eq(ingress.width(), expect!["104"]); expect_eq(egress.width(), expect!["81"]); - expect_eq(hash3.width(), expect!["493"]); + expect_eq(commit.width(), expect!["493"]); expect_eq(hash4.width(), expect!["655"]); expect_eq(hash5.width(), expect!["815"]); expect_eq(u64_add.width(), expect!["53"]); diff --git a/src/core/misc.rs b/src/core/misc.rs index a47656d5..3828a3bb 100644 --- a/src/core/misc.rs +++ b/src/core/misc.rs @@ -2,9 +2,14 @@ use p3_field::AbstractField; use crate::{func, lair::expr::FuncE}; -pub fn hash3() -> FuncE { +// This coroutine is used specifically to create commitments. If we ever need +// the `hasher3` gadget to ingress/egress data, we should define a new `hash3` +// coroutine so the data doesn't get mixed up. We need this separation so API +// consumers (such as the REPL) can have precise information about what's been +// committed to so far. +pub fn commit() -> FuncE { func!( - invertible fn hash3(preimg: [24]): [8] { + invertible fn commit(preimg: [24]): [8] { let img: [8] = extern_call(hasher3, preimg); return img } diff --git a/src/core/state.rs b/src/core/state.rs index dcd97093..6dd58450 100644 --- a/src/core/state.rs +++ b/src/core/state.rs @@ -315,7 +315,7 @@ pub(crate) const BUILTIN_SYMBOLS: [&str; 43] = [ "fail", ]; -pub(crate) const META_SYMBOLS: [&str; 39] = [ +pub(crate) const META_SYMBOLS: [&str; 44] = [ "def", "defq", "defrec", @@ -348,11 +348,16 @@ pub(crate) const META_SYMBOLS: [&str; 39] = [ "defprotocol", "prove-protocol", "verify-protocol", + "microchain-set-info", + "microchain-get-info", "microchain-start", "microchain-get-genesis", "microchain-get-state", "microchain-transition", "microchain-verify", + "scope", + "scope-pop", + "scope-store", "load-ocaml", "load-ocaml-expr", ]; diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index 6b6ead71..48423828 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -34,10 +34,8 @@ fn run_tests>( config: BabyBearPoseidon2, ) { let mut record = QueryRecord::new(toplevel); - let hashes3 = std::mem::take(&mut zstore.hashes3_diff); let hashes4 = std::mem::take(&mut zstore.hashes4_diff); let hashes5 = std::mem::take(&mut zstore.hashes5_diff); - record.inject_inv_queries_owned("hash3", toplevel, hashes3); record.inject_inv_queries_owned("hash4", toplevel, hashes4); record.inject_inv_queries_owned("hash5", toplevel, hashes5); diff --git a/src/core/zstore.rs b/src/core/zstore.rs index 0fb751c5..4a5f15d5 100644 --- a/src/core/zstore.rs +++ b/src/core/zstore.rs @@ -250,12 +250,17 @@ impl> Hasher { #[derive(Clone)] pub struct ZStore> { - hasher: Hasher, + pub(crate) hasher: Hasher, pub(crate) dag: FxHashMap, ZPtrType>, - pub hashes3: FxHashMap<[F; HASH3_SIZE], [F; DIGEST_SIZE]>, + + // This map is used specifically to store commitments. Analogously to the + // `commit` coroutine, if we ever need the `hash3` hasher to intern data, we + // should define new `hashes3` and `hashes3_diff` maps so the data doesn't + // get mixed up. + pub comms: FxHashMap<[F; HASH3_SIZE], [F; DIGEST_SIZE]>, + pub hashes4: FxHashMap<[F; HASH4_SIZE], [F; DIGEST_SIZE]>, pub hashes5: FxHashMap<[F; HASH5_SIZE], [F; DIGEST_SIZE]>, - pub hashes3_diff: FxHashMap<[F; HASH3_SIZE], [F; DIGEST_SIZE]>, pub hashes4_diff: FxHashMap<[F; HASH4_SIZE], [F; DIGEST_SIZE]>, pub hashes5_diff: FxHashMap<[F; HASH5_SIZE], [F; DIGEST_SIZE]>, str_cache: FxHashMap>, @@ -271,10 +276,9 @@ impl Default for ZStore { let mut zstore = Self { hasher: lurk_hasher(), dag: Default::default(), - hashes3: Default::default(), + comms: Default::default(), hashes4: Default::default(), hashes5: Default::default(), - hashes3_diff: Default::default(), hashes4_diff: Default::default(), hashes5_diff: Default::default(), str_cache: Default::default(), @@ -302,13 +306,12 @@ pub(crate) fn builtin_set() -> &'static IndexSet { } impl> ZStore { - pub(crate) fn hash3(&mut self, preimg: [F; HASH3_SIZE]) -> [F; DIGEST_SIZE] { - if let Some(img) = self.hashes3.get(&preimg) { + pub(crate) fn commit(&mut self, preimg: [F; HASH3_SIZE]) -> [F; DIGEST_SIZE] { + if let Some(img) = self.comms.get(&preimg) { return *img; } let digest = into_sized(&self.hasher.hash3.execute_simple(&preimg)); - self.hashes3.insert(preimg, digest); - self.hashes3_diff.insert(preimg, digest); + self.comms.insert(preimg, digest); digest } diff --git a/src/lair/execute.rs b/src/lair/execute.rs index 6674f003..6387a0b5 100644 --- a/src/lair/execute.rs +++ b/src/lair/execute.rs @@ -19,7 +19,7 @@ use super::{ FxIndexMap, List, }; -type QueryMap = FxIndexMap, QueryResult>; +pub(crate) type QueryMap = FxIndexMap, QueryResult>; type InvQueryMap = FxHashMap, List>; pub(crate) type MemMap = FxIndexMap, QueryResult>; @@ -30,6 +30,9 @@ pub struct QueryResult { pub(crate) requires: Vec, pub(crate) depth: u32, pub(crate) depth_requires: Vec, + /// Whether this result was computed directly (rather than via an inverse + /// query) at least once + pub(crate) direct_call: bool, } impl QueryResult { @@ -41,6 +44,16 @@ impl QueryResult { pub(crate) fn new_lookup(&mut self, nonce: usize, caller_requires: &mut Vec) { caller_requires.push(self.provide.new_lookup(nonce as u32)); } + + fn set_as_direct_call(&mut self) { + self.direct_call = true; + } + + fn dummy_direct_call() -> Self { + let mut res = Self::default(); + res.set_as_direct_call(); + res + } } #[derive(Debug, Clone, Eq, PartialEq)] @@ -352,6 +365,15 @@ impl QueryRecord { .expect("Inverse query map not found") } + pub fn get_queries, C2: Chipset>( + &self, + name: &'static str, + toplevel: &Toplevel, + ) -> &QueryMap { + let func = toplevel.func_by_name(name); + &self.func_queries[func.index] + } + /// Erases the records of func, memory and bytes queries, but leaves the history /// of invertible queries untouched pub fn clean(&mut self) { @@ -441,7 +463,7 @@ impl Func { dbg_func_idx: Option, ) -> Result<(List, u32)> { let mut func_index = self.index; - let mut query_result = QueryResult::default(); + let mut query_result = QueryResult::dummy_direct_call(); query_result.provide.count = 1; let (mut nonce, _) = queries.func_queries[func_index].insert_full(args.into(), query_result); @@ -502,6 +524,7 @@ impl Func { if let Some((query_idx, _, result)) = queries.func_queries[*callee_index].get_full_mut(inp.as_slice()) { + result.set_as_direct_call(); let Some(out) = result.output.as_ref() else { bail!("Loop detected"); }; @@ -520,7 +543,7 @@ impl Func { } else { // insert dummy entry let (callee_nonce, _) = queries.func_queries[*callee_index] - .insert_full(inp.clone().into(), QueryResult::default()); + .insert_full(inp.clone().into(), QueryResult::dummy_direct_call()); // `map_buffer` will become the map for the called function let mut map_buffer = inp; let mut requires_buffer = Vec::new(); diff --git a/src/loam/allocation.rs b/src/loam/allocation.rs index 41bf3f9c..602e94eb 100644 --- a/src/loam/allocation.rs +++ b/src/loam/allocation.rs @@ -102,7 +102,7 @@ impl Allocator { } pub fn import_zstore(&mut self, zstore: &ZStore) { - self.import_hashes3(&zstore.hashes3); + self.import_hashes3(&zstore.comms); self.import_hashes4(&zstore.hashes4); self.import_hashes5(&zstore.hashes5); }