diff --git a/hydroflow/examples/modules/main.rs b/hydroflow/examples/modules/main.rs new file mode 100644 index 00000000000..eea522e245e --- /dev/null +++ b/hydroflow/examples/modules/main.rs @@ -0,0 +1,46 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use hydroflow::hydroflow_syntax; +use hydroflow::scheduled::graph::Hydroflow; +use hydroflow::util::multiset::HashMultiSet; + +pub fn main() { + let output = Rc::new(RefCell::new( + HashMultiSet::<(usize, usize, usize)>::default(), + )); + + let mut df: Hydroflow = { + let output = output.clone(); + hydroflow_syntax! { + source_iter(0..2) -> [0]cj; + source_iter(0..2) -> [1]cj; + source_iter(0..2) -> [2]cj; + + cj = import!("triple_cross_join.hf") + -> for_each(|x| output.borrow_mut().insert(x)); + } + }; + + df.run_available(); + + #[rustfmt::skip] + assert_eq!( + *output.borrow(), + HashMultiSet::from_iter([ + (0, 0, 0), + (0, 0, 1), + (0, 1, 0), + (0, 1, 1), + (1, 0, 0), + (1, 0, 1), + (1, 1, 0), + (1, 1, 1), + ]) + ); +} + +#[test] +fn test() { + main(); +} diff --git a/hydroflow/examples/modules/triple_cross_join.hf b/hydroflow/examples/modules/triple_cross_join.hf new file mode 100644 index 00000000000..bf1083257d2 --- /dev/null +++ b/hydroflow/examples/modules/triple_cross_join.hf @@ -0,0 +1,15 @@ +input[0] + -> [0]cj1; + +input[1] + -> [1]cj1; + +cj1 = cross_join() + -> [0]cj2; + +input[2] + -> [1]cj2; + +cj2 = cross_join() + -> map(|((a, b), c)| (a, b, c)) + -> output; diff --git a/hydroflow_datalog_core/src/lib.rs b/hydroflow_datalog_core/src/lib.rs index e755403f23c..e62a45607c8 100644 --- a/hydroflow_datalog_core/src/lib.rs +++ b/hydroflow_datalog_core/src/lib.rs @@ -273,6 +273,7 @@ pub fn gen_hydroflow_graph( if !diagnostics.is_empty() { Err(diagnostics) } else { + flat_graph.merge_modules(); eliminate_extra_unions_tees(&mut flat_graph); Ok(flat_graph) } diff --git a/hydroflow_lang/src/graph/di_mul_graph.rs b/hydroflow_lang/src/graph/di_mul_graph.rs index 71d61cb723e..b12b593acf2 100644 --- a/hydroflow_lang/src/graph/di_mul_graph.rs +++ b/hydroflow_lang/src/graph/di_mul_graph.rs @@ -180,6 +180,26 @@ where Some((new_edge, (pred_edge, succ_edge))) } + /// Remove an edge from the graph. If the edgeId is found then the edge is removed from the graph and returned. + /// If the edgeId was not found in the graph then nothing is returned and nothing is done. + pub fn remove_edge(&mut self, e: E) -> Option<(V, V)> { + let Some((src, dst)) = self.edges.remove(e) else { + return None; + }; + + self.succs[src].retain(|x| *x != e); + self.preds[dst].retain(|x| *x != e); + + Some((src, dst)) + } + + /// Remove a vertex from the graph, it must have no edges to or from it when doing this. + pub fn remove_vertex(&mut self, v: V) { + assert!(!self.edges.values().any(|(v1, v2)| *v1 == v || *v2 == v),); + self.preds.remove(v); + self.succs.remove(v); + } + /// Get the source and destination vertex IDs for the given edge ID. pub fn edge(&self, e: E) -> Option<(V, V)> { self.edges.get(e).copied() diff --git a/hydroflow_lang/src/graph/flat_graph_builder.rs b/hydroflow_lang/src/graph/flat_graph_builder.rs index 4f011bc1728..6bd6d411515 100644 --- a/hydroflow_lang/src/graph/flat_graph_builder.rs +++ b/hydroflow_lang/src/graph/flat_graph_builder.rs @@ -3,11 +3,12 @@ use std::borrow::Cow; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use proc_macro2::Span; use quote::ToTokens; use syn::spanned::Spanned; -use syn::{Ident, ItemUse}; +use syn::{Error, Ident, ItemUse}; use super::ops::find_op_op_constraints; use super::{GraphNodeId, HydroflowGraph, Node, PortIndexValue}; @@ -44,6 +45,16 @@ pub struct FlatGraphBuilder { /// Use statements. uses: Vec, + + /// In order to make import!() statements relative to the current file, we need to know where the file is that is building the flat graph. + macro_invocation_path: PathBuf, + + /// If the flat graph is being loaded as a module, then there are some additional things that happen + /// 1. the varname 'input' and 'output' is reserved and used for the input and output of the module. + /// 2. two initial module boundary nodes are inserted into the graph before statements are processed, one input and one output. + is_module: bool, + input_node: GraphNodeId, + output_node: GraphNodeId, } impl FlatGraphBuilder { @@ -53,8 +64,42 @@ impl FlatGraphBuilder { } /// Convert the Hydroflow code AST into a graph builder. - pub fn from_hfcode(input: HfCode) -> Self { - input.into() + pub fn from_hfcode(input: HfCode, macro_invocation_path: PathBuf) -> Self { + let mut builder = Self { + macro_invocation_path, + ..Default::default() + }; + builder.process_statements(input.statements); + + builder + } + + /// Convert the Hydroflow code AST into a graph builder. + pub fn from_hfmodule(input: HfCode) -> Self { + let mut builder = Self::default(); + builder.is_module = true; + builder.input_node = builder.flat_graph.insert_node( + Node::ModuleBoundary { + input: true, + import_expr: Span::call_site(), + }, + Some(Ident::new("input", Span::call_site())), + ); + builder.output_node = builder.flat_graph.insert_node( + Node::ModuleBoundary { + input: false, + import_expr: Span::call_site(), + }, + Some(Ident::new("output", Span::call_site())), + ); + builder.process_statements(input.statements); + builder + } + + fn process_statements(&mut self, statements: impl IntoIterator) { + for stmt in statements { + self.add_statement(stmt); + } } /// Build into an unpartitioned [`HydroflowGraph`], returning a tuple of a `HydroflowGraph` and @@ -77,7 +122,13 @@ impl FlatGraphBuilder { } HfStatement::Named(named) => { let stmt_span = named.span(); - let ends = self.add_pipeline(named.pipeline, Some(&named.name)); + let ends = match self.add_pipeline(named.pipeline, Some(&named.name)) { + Err(err) => { + self.diagnostics.push(err.into()); + return; + } + Ok(ends) => ends, + }; match self.varname_ends.entry(named.name) { Entry::Vacant(vacant_entry) => { vacant_entry.insert(Ok(ends)); @@ -106,33 +157,57 @@ impl FlatGraphBuilder { } } HfStatement::Pipeline(pipeline_stmt) => { - self.add_pipeline(pipeline_stmt.pipeline, None); + if let Err(err) = self.add_pipeline(pipeline_stmt.pipeline, None) { + self.diagnostics.push(err.into()); + } } } } /// Helper: Add a pipeline, i.e. `a -> b -> c`. Return the input and output ends for it. - fn add_pipeline(&mut self, pipeline: Pipeline, current_varname: Option<&Ident>) -> Ends { + fn add_pipeline( + &mut self, + pipeline: Pipeline, + current_varname: Option<&Ident>, + ) -> syn::Result { match pipeline { Pipeline::Paren(ported_pipeline_paren) => { let (inn_port, pipeline_paren, out_port) = PortIndexValue::from_ported(ported_pipeline_paren); - let og_ends = self.add_pipeline(*pipeline_paren.pipeline, current_varname); - Self::helper_combine_ends(&mut self.diagnostics, og_ends, inn_port, out_port) + let og_ends = self.add_pipeline(*pipeline_paren.pipeline, current_varname)?; + Ok(Self::helper_combine_ends( + &mut self.diagnostics, + og_ends, + inn_port, + out_port, + )) } Pipeline::Name(pipeline_name) => { let (inn_port, ident, out_port) = PortIndexValue::from_ported(pipeline_name); - // We could lookup non-forward references immediately, but easier to just have one - // consistent code path. -mingwei - Ends { - inn: Some((inn_port, GraphDet::Undetermined(ident.clone()))), - out: Some((out_port, GraphDet::Undetermined(ident))), + + if self.is_module && ident == "input" { + Ok(Ends { + inn: Some((inn_port, GraphDet::Determined(self.input_node))), + out: Some((out_port, GraphDet::Determined(self.input_node))), + }) + } else if self.is_module && ident == "output" { + Ok(Ends { + inn: Some((inn_port, GraphDet::Determined(self.output_node))), + out: Some((out_port, GraphDet::Determined(self.output_node))), + }) + } else { + // We could lookup non-forward references immediately, but easier to just have one + // consistent code path. -mingwei + Ok(Ends { + inn: Some((inn_port, GraphDet::Undetermined(ident.clone()))), + out: Some((out_port, GraphDet::Undetermined(ident))), + }) } } Pipeline::Link(pipeline_link) => { // Add the nested LHS and RHS of this link. - let lhs_ends = self.add_pipeline(*pipeline_link.lhs, current_varname); - let rhs_ends = self.add_pipeline(*pipeline_link.rhs, current_varname); + let lhs_ends = self.add_pipeline(*pipeline_link.lhs, current_varname)?; + let rhs_ends = self.add_pipeline(*pipeline_link.rhs, current_varname)?; // Outer (first and last) ends. let outer_ends = Ends { @@ -145,17 +220,103 @@ impl FlatGraphBuilder { inn: rhs_ends.inn, }; self.links.push(link_ends); - outer_ends + Ok(outer_ends) } Pipeline::Operator(operator) => { let op_span = Some(operator.span()); let nid = self .flat_graph .insert_node(Node::Operator(operator), current_varname.cloned()); - Ends { + Ok(Ends { inn: Some((PortIndexValue::Elided(op_span), GraphDet::Determined(nid))), out: Some((PortIndexValue::Elided(op_span), GraphDet::Determined(nid))), + }) + } + Pipeline::Import(import) => { + // TODO: https://github.com/rust-lang/rfcs/pull/3200 + // this would be way better... + let mut dir = self.macro_invocation_path.clone(); + dir.pop(); + + let file_contents = std::fs::read_to_string(dir.join(import.filename.value())) + .map_err(|e| { + Error::new( + import.filename.span(), + format!("filename: {}, err: {e}", import.filename.value()), + ) + })?; + + // TODO: see also above, parse_str sets all the spans in the resulting parsed token stream to the parent macro invocation span. + // This means that any error inside the imported module will manifest as a giant red squiggly line under the parent hydroflow_syntax!{} call. + let statements = match syn::parse_str::(&file_contents) { + Ok(code) => code, + Err(err) => { + self.diagnostics.push(err.clone().into()); + return syn::Result::Err(err); + } + }; + + let flat_graph_builder = crate::graph::FlatGraphBuilder::from_hfmodule(statements); + let (flat_graph, _uses, diagnostics) = flat_graph_builder.build(); + diagnostics + .iter() + .for_each(crate::diagnostic::Diagnostic::emit); + + let mut ends = Ends { + inn: None, + out: None, + }; + + let mut node_mapping = BTreeMap::new(); + + for (nid, node) in flat_graph.nodes() { + match node { + Node::Operator(_) => { + let varname = flat_graph.node_varname(nid); + let new_id = self.flat_graph.insert_node(node.clone(), varname); + node_mapping.insert(nid, new_id); + } + Node::ModuleBoundary { input, .. } => { + let new_id = self.flat_graph.insert_node( + Node::ModuleBoundary { + input: *input, + import_expr: import.span(), + }, + Some(Ident::new( + &format!("module_{}", input.to_string()), + import.span(), + )), + ); + node_mapping.insert(nid, new_id); + + if *input { + ends.inn = Some(( + PortIndexValue::Elided(None), + GraphDet::Determined(new_id), + )); + } else { + ends.out = Some(( + PortIndexValue::Elided(None), + GraphDet::Determined(new_id), + )); + } + } + _ => panic!(), + } + } + + for (eid, (src, dst)) in flat_graph.edges() { + let (src_port, dst_port) = flat_graph.edge_ports(eid); + + self.flat_graph.insert_edge( + *node_mapping.get(&src).unwrap(), + src_port.clone(), + *node_mapping.get(&dst).unwrap(), + dst_port.clone(), + ); } + + Ok(ends) } } } @@ -515,6 +676,9 @@ impl FlatGraphBuilder { ); } Node::Handoff { .. } => todo!("Node::Handoff"), + Node::ModuleBoundary { .. } => { + // Module boundaries don't require any checking. + } } } } @@ -577,13 +741,3 @@ impl FlatGraphBuilder { } } } - -impl From for FlatGraphBuilder { - fn from(input: HfCode) -> Self { - let mut builder = Self::default(); - for stmt in input.statements { - builder.add_statement(stmt); - } - builder - } -} diff --git a/hydroflow_lang/src/graph/hydroflow_graph.rs b/hydroflow_lang/src/graph/hydroflow_graph.rs index f696d0ce9c6..cfcb5186848 100644 --- a/hydroflow_lang/src/graph/hydroflow_graph.rs +++ b/hydroflow_lang/src/graph/hydroflow_graph.rs @@ -19,6 +19,7 @@ use super::{ }; use crate::diagnostic::{Diagnostic, Level}; use crate::graph::ops::null_write_iterator_fn; +use crate::graph::MODULE_BOUNDARY_NODE_STR; use crate::pretty_span::{PrettyRowCol, PrettySpan}; /// A graph representing a Hydroflow dataflow graph (with or without subgraph partitioning, @@ -76,6 +77,11 @@ impl HydroflowGraph { self.operator_instances.get(node_id) } + /// Get the debug variable name attached to a graph node. + pub fn node_varname(&self, node_id: GraphNodeId) -> Option { + self.node_varnames.get(node_id).map(|x| x.0.clone()) + } + /// Get subgraph for node. pub fn node_subgraph(&self, node_id: GraphNodeId) -> Option { self.node_subgraph.get(node_id).copied() @@ -383,6 +389,71 @@ impl HydroflowGraph { let (_, dst_port) = self.ports.remove(succ_edge_id).unwrap(); self.ports.insert(new_edge_id, (src_port, dst_port)); } + + /// When modules are imported into a flat graph, they come with an input and output ModuleBoundary node. + /// The partitioner doesn't understand these nodes and will panic if it encounters them. + /// merge_modules removes them from the graph, stitching the input and ouput sides of the ModuleBondaries based on their ports + /// For example: + /// source_iter([]) -> \[myport\]ModuleBoundary(input)\[my_port\] -> map(|x| x) -> ModuleBoundary(output) -> null(); + /// in the above eaxmple, the \[myport\] port will be used to connect the source_iter with the map that is inside of the module. + /// The output module boundary has elided ports, this is also used to match up the input/output across the module boundary. + pub fn merge_modules(&mut self) { + let mut to_remove = Vec::new(); + + for (nid, node) in self.nodes() { + if matches!(node, Node::ModuleBoundary { .. }) { + to_remove.push(nid); + } + } + + for nid in to_remove { + self.remove_module_boundary(nid); + } + } + + /// see `merge_modules` + /// This function removes a singular module boundary from the graph and performs the necessary stitching to fix the graph aftward. + /// `merge_modules` calls this function for each module boundary in the graph. + pub fn remove_module_boundary(&mut self, nid: GraphNodeId) { + assert!( + self.node_subgraph.is_empty() && self.subgraph_nodes.is_empty(), + "Should not remove intermediate node after subgraph partitioning" + ); + + let mut predecessor_ports = BTreeMap::new(); + let mut successor_ports = BTreeMap::new(); + + for eid in self.node_predecessor_edges(nid) { + let (predecessor_port, successor_port) = self.edge_ports(eid); + predecessor_ports.insert(successor_port.clone(), (eid, predecessor_port.clone())); + } + + for eid in self.node_successor_edges(nid) { + let (predecessor_port, successor_port) = self.edge_ports(eid); + successor_ports.insert(predecessor_port.clone(), (eid, successor_port.clone())); + } + + assert_eq!( + predecessor_ports.keys().collect::>(), + successor_ports.keys().collect::>() + ); + + for (port, (predecessor_edge, predecessor_port)) in predecessor_ports { + let (successor_edge, successor_port) = successor_ports.remove(&port).unwrap(); + + let (src, _) = self.graph.remove_edge(predecessor_edge).unwrap(); + let (_, dst) = self.graph.remove_edge(successor_edge).unwrap(); + + self.ports.remove(predecessor_edge); + self.ports.remove(successor_edge); + + let eid = self.graph.insert_edge(src, dst); + self.ports.insert(eid, (predecessor_port, successor_port)); + } + + self.graph.remove_vertex(nid); + self.nodes.remove(nid); + } } // Edge methods. impl HydroflowGraph { @@ -523,11 +594,13 @@ impl HydroflowGraph { node_id.data(), if is_pred { "recv" } else { "send" } ), + Node::ModuleBoundary { .. } => panic!(), }; let span = match (is_pred, &self.nodes[node_id]) { (_, Node::Operator(operator)) => operator.span(), (true, &Node::Handoff { src_span, .. }) => src_span, (false, &Node::Handoff { dst_span, .. }) => dst_span, + (_, Node::ModuleBoundary { .. }) => panic!(), }; Ident::new(&*name, span) } @@ -584,6 +657,7 @@ impl HydroflowGraph { .filter_map(|(node_id, node)| match node { Node::Operator(_) => None, &Node::Handoff { src_span, dst_span } => Some((node_id, (src_span, dst_span))), + Node::ModuleBoundary { .. } => panic!(), }) .map(|(node_id, (src_span, dst_span))| { let ident_send = Ident::new(&*format!("hoff_{:?}_send", node_id.data()), dst_span); @@ -1186,6 +1260,7 @@ impl HydroflowGraph { writeln!(write, "{:?} = {};", key.data(), op.to_token_stream())?; } Node::Handoff { .. } => unimplemented!("HANDOFF IN FLAT GRAPH."), + Node::ModuleBoundary { .. } => panic!(), } } writeln!(write)?; @@ -1225,6 +1300,14 @@ impl HydroflowGraph { Node::Handoff { .. } => { writeln!(write, r#" {:?}{{"{}"}}"#, key.data(), HANDOFF_NODE_STR) } + Node::ModuleBoundary { .. } => { + writeln!( + write, + r#" {:?}{{"{}"}}"#, + key.data(), + MODULE_BOUNDARY_NODE_STR + ) + } }?; } writeln!(write)?; diff --git a/hydroflow_lang/src/graph/mod.rs b/hydroflow_lang/src/graph/mod.rs index b993868ecf9..922575eaf04 100644 --- a/hydroflow_lang/src/graph/mod.rs +++ b/hydroflow_lang/src/graph/mod.rs @@ -24,6 +24,8 @@ mod flow_props; mod graph_write; mod hydroflow_graph; +use std::path::PathBuf; + pub use di_mul_graph::DiMulGraph; pub use eliminate_extra_unions_tees::eliminate_extra_unions_tees; pub use flat_graph_builder::FlatGraphBuilder; @@ -52,6 +54,7 @@ const CONTEXT: &str = "context"; const HYDROFLOW: &str = "df"; const HANDOFF_NODE_STR: &str = "handoff"; +const MODULE_BOUNDARY_NODE_STR: &str = "module_boundary"; mod serde_syn { use serde::{Deserialize, Deserializer, Serializer}; @@ -91,6 +94,18 @@ pub enum Node { #[serde(skip, default = "Span::call_site")] dst_span: Span, }, + + /// Module Boundary, used for importing modules. Only exists prior to partitioning. + ModuleBoundary { + /// If this module is an input or output boundary. + input: bool, + + /// The span of the import!() expression that imported this module. + /// The value of this span when the ModuleBoundary node is still inside the module is Span::call_site() + /// TODO: This could one day reference into the module file itself? + #[serde(skip, default = "Span::call_site")] + import_expr: Span, + }, } impl Node { /// Return the node as a human-readable string. @@ -98,6 +113,7 @@ impl Node { match self { Node::Operator(op) => op.to_pretty_string().into(), Node::Handoff { .. } => HANDOFF_NODE_STR.into(), + Node::ModuleBoundary { .. } => MODULE_BOUNDARY_NODE_STR.into(), } } @@ -106,6 +122,7 @@ impl Node { match self { Self::Operator(op) => op.span(), &Self::Handoff { src_span, dst_span } => src_span.join(dst_span).unwrap_or(src_span), + Self::ModuleBoundary { import_expr, .. } => *import_expr, } } } @@ -116,6 +133,9 @@ impl std::fmt::Debug for Node { write!(f, "Node::Operator({} span)", PrettySpan(operator.span())) } Self::Handoff { .. } => write!(f, "Node::Handoff"), + Self::ModuleBoundary { input, .. } => { + write!(f, "Node::ModuleBoundary{{input: {}}}", input) + } } } } @@ -350,11 +370,13 @@ impl Ord for PortIndexValue { pub fn build_hfcode( hf_code: HfCode, root: &TokenStream, + macro_invocation_path: PathBuf, ) -> (Option<(HydroflowGraph, TokenStream)>, Vec) { - let flat_graph_builder = FlatGraphBuilder::from_hfcode(hf_code); + let flat_graph_builder = FlatGraphBuilder::from_hfcode(hf_code, macro_invocation_path); let (mut flat_graph, uses, mut diagnostics) = flat_graph_builder.build(); - eliminate_extra_unions_tees(&mut flat_graph); if !diagnostics.iter().any(Diagnostic::is_error) { + flat_graph.merge_modules(); + eliminate_extra_unions_tees(&mut flat_graph); match partition_graph(flat_graph) { Ok(mut partitioned_graph) => { // Propgeate flow properties throughout the graph. diff --git a/hydroflow_lang/src/graph/propegate_flow_props.rs b/hydroflow_lang/src/graph/propegate_flow_props.rs index 87b7824fe31..048b4e1caea 100644 --- a/hydroflow_lang/src/graph/propegate_flow_props.rs +++ b/hydroflow_lang/src/graph/propegate_flow_props.rs @@ -73,6 +73,10 @@ pub fn propegate_flow_props( graph.set_edge_flow_props(out_edge, flow_props); } } + _ => { + // If a module boundary is encountered then something has gone wrong. + panic!(); + } } } Ok(()) diff --git a/hydroflow_lang/src/parse.rs b/hydroflow_lang/src/parse.rs index f830e62a70a..440cdcc9a5f 100644 --- a/hydroflow_lang/src/parse.rs +++ b/hydroflow_lang/src/parse.rs @@ -9,8 +9,8 @@ use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; use syn::token::{Bracket, Paren}; use syn::{ - bracketed, parenthesized, AngleBracketedGenericArguments, Expr, ExprPath, GenericArgument, - Ident, ItemUse, LitInt, Path, PathArguments, PathSegment, Token, + bracketed, parenthesized, AngleBracketedGenericArguments, Error, Expr, ExprPath, + GenericArgument, Ident, ItemUse, LitInt, LitStr, Path, PathArguments, PathSegment, Token, }; pub struct HfCode { @@ -109,11 +109,13 @@ impl ToTokens for PipelineStatement { } } +#[derive(Debug)] pub enum Pipeline { Paren(Ported), Name(Ported), Link(PipelineLink), Operator(Operator), + Import(Import), } impl Pipeline { fn parse_one(input: ParseStream) -> syn::Result { @@ -135,15 +137,23 @@ impl Pipeline { else { Err(lookahead2.error()) } - } // Ident - else if lookahead1.peek(Ident) { + } else if lookahead1.peek(Ident) { + let speculative = input.fork(); + let ident: Ident = speculative.parse()?; + let lookahead2 = speculative.lookahead1(); + // If has paren or generic next, it's an operator - if input.peek2(Paren) || input.peek2(Token![<]) || input.peek2(Token![::]) { + if lookahead2.peek(Paren) || lookahead2.peek(Token![<]) || lookahead2.peek(Token![::]) { Ok(Self::Operator(input.parse()?)) - } + // macro-style expression "x!.." + } else if lookahead2.peek(Token![!]) { + match ident.to_string().as_str() { + "import" => Ok(Self::Import(input.parse()?)), + _ => Err(Error::new(ident.span(), r#"Expected "import""#)), + } // Otherwise it's a name - else { + } else { Ok(Self::Name(input.parse()?)) } } @@ -177,10 +187,44 @@ impl ToTokens for Pipeline { Pipeline::Link(x) => x.to_tokens(tokens), Pipeline::Name(x) => x.to_tokens(tokens), Pipeline::Operator(x) => x.to_tokens(tokens), + Pipeline::Import(x) => x.to_tokens(tokens), } } } +#[derive(Debug)] +pub struct Import { + pub import: Ident, + pub bang: Token![!], + pub paren_token: Paren, + pub filename: LitStr, +} +impl Parse for Import { + fn parse(input: ParseStream) -> syn::Result { + let import = input.parse()?; + let bang = input.parse()?; + let content; + let paren_token = parenthesized!(content in input); + let filename: LitStr = content.parse()?; + + Ok(Self { + import, + bang, + paren_token, + filename, + }) + } +} +impl ToTokens for Import { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.import.to_tokens(tokens); + self.bang.to_tokens(tokens); + self.paren_token + .surround(tokens, |tokens| self.filename.to_tokens(tokens)); + } +} + +#[derive(Debug)] pub struct Ported { pub inn: Option, pub inner: Inner, @@ -218,6 +262,7 @@ where } } +#[derive(Debug)] pub struct PipelineParen { pub paren_token: Paren, pub pipeline: Box, @@ -241,6 +286,7 @@ impl ToTokens for PipelineParen { } } +#[derive(Debug)] pub struct PipelineLink { pub lhs: Box, pub arrow: Token![->], @@ -263,6 +309,7 @@ impl ToTokens for PipelineLink { } } +#[derive(Debug)] pub struct Indexing { pub bracket_token: Bracket, pub index: PortIndex, @@ -316,7 +363,7 @@ impl ToTokens for PortIndex { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Operator { pub path: Path, pub paren_token: Paren, diff --git a/hydroflow_macro/src/lib.rs b/hydroflow_macro/src/lib.rs index 54909541d72..3c67aef38d5 100644 --- a/hydroflow_macro/src/lib.rs +++ b/hydroflow_macro/src/lib.rs @@ -59,9 +59,11 @@ fn hydroflow_syntax_internal( input: proc_macro::TokenStream, min_diagnostic_level: Option, ) -> proc_macro::TokenStream { + let macro_invocation_path = proc_macro::Span::call_site().source_file().path(); + let input = parse_macro_input!(input as HfCode); let root = root(); - let (graph_code_opt, diagnostics) = build_hfcode(input, &root); + let (graph_code_opt, diagnostics) = build_hfcode(input, &root, macro_invocation_path); let tokens = graph_code_opt .map(|(_graph, code)| code) .unwrap_or_else(|| quote! { #root::scheduled::graph::Hydroflow::new() }); @@ -96,20 +98,28 @@ fn hydroflow_syntax_internal( /// Used for testing, users will want to use [`hydroflow_syntax!`] instead. #[proc_macro] pub fn hydroflow_parser(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let macro_invocation_path = proc_macro::Span::call_site().source_file().path(); + let input = parse_macro_input!(input as HfCode); - let flat_graph_builder = FlatGraphBuilder::from_hfcode(input); - let (flat_graph, _uses, diagnostics) = flat_graph_builder.build(); - diagnostics.iter().for_each(Diagnostic::emit); - let flat_mermaid = flat_graph.mermaid_string_flat(); + let flat_graph_builder = FlatGraphBuilder::from_hfcode(input, macro_invocation_path); + let (mut flat_graph, _uses, diagnostics) = flat_graph_builder.build(); + if !diagnostics.iter().any(Diagnostic::is_error) { + flat_graph.merge_modules(); - let part_graph = partition_graph(flat_graph).unwrap(); - let part_mermaid = part_graph.to_mermaid(); + let flat_mermaid = flat_graph.mermaid_string_flat(); - let lit0 = Literal::string(&*flat_mermaid); - let lit1 = Literal::string(&*part_mermaid); + let part_graph = partition_graph(flat_graph).unwrap(); + let part_mermaid = part_graph.to_mermaid(); - quote! { println!("{}\n\n{}\n", #lit0, #lit1); }.into() + let lit0 = Literal::string(&*flat_mermaid); + let lit1 = Literal::string(&*part_mermaid); + + quote! { println!("{}\n\n{}\n", #lit0, #lit1); }.into() + } else { + diagnostics.iter().for_each(Diagnostic::emit); + quote! {}.into() + } } #[doc(hidden)] diff --git a/website_playground/src/lib.rs b/website_playground/src/lib.rs index 7864dae3939..bbf61f43e9d 100644 --- a/website_playground/src/lib.rs +++ b/website_playground/src/lib.rs @@ -1,6 +1,7 @@ mod utils; use std::cell::RefCell; use std::collections::HashMap; +use std::path::PathBuf; use std::task::{Context, Poll}; use std::thread_local; @@ -107,7 +108,8 @@ pub struct HydroflowOutput { pub fn compile_hydroflow(program: String) -> JsValue { let out = match syn::parse_str(&program) { Ok(input) => { - let (graph_code_opt, diagnostics) = build_hfcode(input, "e!(hydroflow)); + let (graph_code_opt, diagnostics) = + build_hfcode(input, "e!(hydroflow), PathBuf::default()); let output = graph_code_opt.map(|(graph, code)| { let mermaid = graph.to_mermaid(); let file = syn::parse_quote! {