From 518195afc8e0f81bcceaf35fcc55316f8f822349 Mon Sep 17 00:00:00 2001 From: b1ek Date: Fri, 15 Nov 2024 16:45:57 +1000 Subject: [PATCH 01/10] add Serialize and Deserialize traits to all types --- src/modules/block.rs | 3 ++- src/modules/builtin/cd.rs | 3 ++- src/modules/builtin/echo.rs | 3 ++- src/modules/builtin/exit.rs | 3 ++- src/modules/builtin/len.rs | 3 ++- src/modules/builtin/mv.rs | 3 ++- src/modules/builtin/nameof.rs | 3 ++- src/modules/command/cmd.rs | 3 ++- src/modules/command/modifier.rs | 3 ++- src/modules/condition/failed.rs | 3 ++- src/modules/condition/ifchain.rs | 3 ++- src/modules/condition/ifcond.rs | 3 ++- src/modules/expression/binop/add.rs | 3 ++- src/modules/expression/binop/and.rs | 3 ++- src/modules/expression/binop/div.rs | 3 ++- src/modules/expression/binop/eq.rs | 3 ++- src/modules/expression/binop/ge.rs | 3 ++- src/modules/expression/binop/gt.rs | 3 ++- src/modules/expression/binop/le.rs | 3 ++- src/modules/expression/binop/lt.rs | 3 ++- src/modules/expression/binop/modulo.rs | 3 ++- src/modules/expression/binop/mul.rs | 3 ++- src/modules/expression/binop/neq.rs | 3 ++- src/modules/expression/binop/or.rs | 3 ++- src/modules/expression/binop/range.rs | 3 ++- src/modules/expression/binop/sub.rs | 3 ++- src/modules/expression/expr.rs | 5 +++-- src/modules/expression/literal/array.rs | 3 ++- src/modules/expression/literal/bool.rs | 3 ++- src/modules/expression/literal/null.rs | 3 ++- src/modules/expression/literal/number.rs | 3 ++- src/modules/expression/literal/status.rs | 3 ++- src/modules/expression/literal/text.rs | 3 ++- src/modules/expression/parentheses.rs | 3 ++- src/modules/expression/ternop/ternary.rs | 3 ++- src/modules/expression/typeop/cast.rs | 3 ++- src/modules/expression/typeop/is.rs | 3 ++- src/modules/expression/unop/neg.rs | 3 ++- src/modules/expression/unop/not.rs | 3 ++- src/modules/function/declaration.rs | 3 ++- src/modules/function/fail.rs | 3 ++- src/modules/function/invocation.rs | 3 ++- src/modules/function/ret.rs | 3 ++- src/modules/imports/import.rs | 3 ++- src/modules/imports/import_string.rs | 3 ++- src/modules/loops/break_stmt.rs | 3 ++- src/modules/loops/continue_stmt.rs | 3 ++- src/modules/loops/infinite_loop.rs | 3 ++- src/modules/loops/iter_loop.rs | 3 ++- src/modules/main.rs | 3 ++- src/modules/shorthand/add.rs | 3 ++- src/modules/shorthand/div.rs | 3 ++- src/modules/shorthand/modulo.rs | 3 ++- src/modules/shorthand/mul.rs | 3 ++- src/modules/shorthand/sub.rs | 3 ++- src/modules/statement/comment.rs | 3 ++- src/modules/statement/comment_doc.rs | 3 ++- src/modules/statement/stmt.rs | 5 +++-- src/modules/types.rs | 3 ++- src/modules/variable/get.rs | 3 ++- src/modules/variable/init.rs | 3 ++- src/modules/variable/set.rs | 3 ++- src/utils/cc_flags.rs | 4 +++- src/utils/context.rs | 9 +++++---- src/utils/function_cache.rs | 8 +++++--- src/utils/import_cache.rs | 8 +++++--- src/utils/metadata/parser.rs | 3 ++- 67 files changed, 146 insertions(+), 76 deletions(-) diff --git a/src/modules/block.rs b/src/modules/block.rs index 64df4c3a..3df31299 100644 --- a/src/modules/block.rs +++ b/src/modules/block.rs @@ -3,12 +3,13 @@ use std::ops::Index; use heraclitus_compiler::prelude::*; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::utils::{metadata::ParserMetadata, TranslateMetadata}; use crate::translate::module::TranslateModule; use super::statement::stmt::Statement; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Block { pub statements: Vec } diff --git a/src/modules/builtin/cd.rs b/src/modules/builtin/cd.rs index d4a6ea4d..904fd285 100644 --- a/src/modules/builtin/cd.rs +++ b/src/modules/builtin/cd.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::modules::expression::expr::Expr; use crate::docs::module::DocumentationModule; use crate::modules::types::{Type, Typed}; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Cd { value: Expr } diff --git a/src/modules/builtin/echo.rs b/src/modules/builtin/echo.rs index 559d33f2..1900baa1 100644 --- a/src/modules/builtin/echo.rs +++ b/src/modules/builtin/echo.rs @@ -1,10 +1,11 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Echo { value: Box } diff --git a/src/modules/builtin/exit.rs b/src/modules/builtin/exit.rs index 533a5e6d..caabc4c1 100644 --- a/src/modules/builtin/exit.rs +++ b/src/modules/builtin/exit.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::modules::expression::expr::Expr; use crate::docs::module::DocumentationModule; use crate::modules::types::{Type, Typed}; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Exit { code: Option } diff --git a/src/modules/builtin/len.rs b/src/modules/builtin/len.rs index a48d283c..a69874e4 100644 --- a/src/modules/builtin/len.rs +++ b/src/modules/builtin/len.rs @@ -5,8 +5,9 @@ use crate::modules::types::{Type, Typed}; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Len { value: Box, } diff --git a/src/modules/builtin/mv.rs b/src/modules/builtin/mv.rs index 35cec231..9ce3e54c 100644 --- a/src/modules/builtin/mv.rs +++ b/src/modules/builtin/mv.rs @@ -1,6 +1,7 @@ use std::mem::swap; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::modules::expression::expr::Expr; use crate::modules::condition::failed::Failed; use crate::translate::module::TranslateModule; @@ -9,7 +10,7 @@ use crate::modules::types::{Type, Typed}; use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::modules::command::modifier::CommandModifier; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Mv { source: Expr, destination: Expr, diff --git a/src/modules/builtin/nameof.rs b/src/modules/builtin/nameof.rs index 019b79b8..befccb57 100644 --- a/src/modules/builtin/nameof.rs +++ b/src/modules/builtin/nameof.rs @@ -4,8 +4,9 @@ use crate::modules::variable::variable_name_extensions; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Nameof { name: String, } diff --git a/src/modules/command/cmd.rs b/src/modules/command/cmd.rs index 850c629e..af5ea24e 100644 --- a/src/modules/command/cmd.rs +++ b/src/modules/command/cmd.rs @@ -1,13 +1,14 @@ use std::mem::swap; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::{condition::failed::Failed, expression::literal::bool, types::{Type, Typed}}, utils::{ParserMetadata, TranslateMetadata}}; use crate::modules::expression::expr::Expr; use crate::translate::module::TranslateModule; use crate::modules::expression::literal::{parse_interpolated_region, translate_interpolated_region}; use super::modifier::CommandModifier; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Command { strings: Vec, interps: Vec, diff --git a/src/modules/command/modifier.rs b/src/modules/command/modifier.rs index e04f9770..2e0f0e92 100644 --- a/src/modules/command/modifier.rs +++ b/src/modules/command/modifier.rs @@ -1,12 +1,13 @@ use std::mem::swap; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::block::Block; use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CommandModifier { pub block: Box, pub is_block: bool, diff --git a/src/modules/condition/failed.rs b/src/modules/condition/failed.rs index 915a72ea..8124bd11 100644 --- a/src/modules/condition/failed.rs +++ b/src/modules/condition/failed.rs @@ -1,10 +1,11 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::modules::block::Block; use crate::modules::statement::stmt::Statement; use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Failed { pub is_parsed: bool, is_question_mark: bool, diff --git a/src/modules/condition/ifchain.rs b/src/modules/condition/ifchain.rs index 849de8db..d2e83dad 100644 --- a/src/modules/condition/ifchain.rs +++ b/src/modules/condition/ifchain.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::translate::module::TranslateModule; @@ -6,7 +7,7 @@ use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::modules::block::Block; use crate::modules::statement::stmt::Statement; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct IfChain { cond_blocks: Vec<(Expr, Block)>, false_block: Option> diff --git a/src/modules/condition/ifcond.rs b/src/modules/condition/ifcond.rs index 79bf81b0..af0200f2 100644 --- a/src/modules/condition/ifcond.rs +++ b/src/modules/condition/ifcond.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::translate::module::TranslateModule; @@ -7,7 +8,7 @@ use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::modules::block::Block; use crate::modules::statement::stmt::{Statement, StatementType}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct IfCondition { expr: Box, true_block: Box, diff --git a/src/modules/expression/binop/add.rs b/src/modules/expression/binop/add.rs index d23caf64..b2348425 100644 --- a/src/modules/expression/binop/add.rs +++ b/src/modules/expression/binop/add.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -9,7 +10,7 @@ use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Add { left: Box, right: Box, diff --git a/src/modules/expression/binop/and.rs b/src/modules/expression/binop/and.rs index fa488153..60c0ef16 100644 --- a/src/modules/expression/binop/and.rs +++ b/src/modules/expression/binop/and.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -10,7 +11,7 @@ use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct And { left: Box, right: Box diff --git a/src/modules/expression/binop/div.rs b/src/modules/expression/binop/div.rs index 357d814a..c761a4e3 100644 --- a/src/modules/expression/binop/div.rs +++ b/src/modules/expression/binop/div.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, utils::TranslateMetadata}; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -10,7 +11,7 @@ use crate::translate::module::TranslateModule; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Div { left: Box, right: Box diff --git a/src/modules/expression/binop/eq.rs b/src/modules/expression/binop/eq.rs index 96455f87..d2b10d78 100644 --- a/src/modules/expression/binop/eq.rs +++ b/src/modules/expression/binop/eq.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use super::{strip_text_quotes, BinOp}; use crate::modules::types::{Typed, Type}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Eq { left: Box, right: Box diff --git a/src/modules/expression/binop/ge.rs b/src/modules/expression/binop/ge.rs index 93564016..267890a9 100644 --- a/src/modules/expression/binop/ge.rs +++ b/src/modules/expression/binop/ge.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Ge { left: Box, right: Box diff --git a/src/modules/expression/binop/gt.rs b/src/modules/expression/binop/gt.rs index a29b56c7..3bc78bde 100644 --- a/src/modules/expression/binop/gt.rs +++ b/src/modules/expression/binop/gt.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Gt { left: Box, right: Box diff --git a/src/modules/expression/binop/le.rs b/src/modules/expression/binop/le.rs index 479454e4..c927448e 100644 --- a/src/modules/expression/binop/le.rs +++ b/src/modules/expression/binop/le.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Le { left: Box, right: Box diff --git a/src/modules/expression/binop/lt.rs b/src/modules/expression/binop/lt.rs index 2ca6cf2f..4f54cd71 100644 --- a/src/modules/expression/binop/lt.rs +++ b/src/modules/expression/binop/lt.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Lt { left: Box, right: Box diff --git a/src/modules/expression/binop/modulo.rs b/src/modules/expression/binop/modulo.rs index 5349f4ba..2edb2a50 100644 --- a/src/modules/expression/binop/modulo.rs +++ b/src/modules/expression/binop/modulo.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::translate::compute::{ArithOp, translate_computation}; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Modulo { left: Box, right: Box diff --git a/src/modules/expression/binop/mul.rs b/src/modules/expression/binop/mul.rs index 7855161a..8aa86400 100644 --- a/src/modules/expression/binop/mul.rs +++ b/src/modules/expression/binop/mul.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::translate::compute::{translate_computation, ArithOp}; @@ -9,7 +10,7 @@ use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Mul { left: Box, right: Box diff --git a/src/modules/expression/binop/neq.rs b/src/modules/expression/binop/neq.rs index c8cc203c..6d676005 100644 --- a/src/modules/expression/binop/neq.rs +++ b/src/modules/expression/binop/neq.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::translate::compute::{translate_computation, ArithOp}; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use super::{strip_text_quotes, BinOp}; use crate::modules::types::{Typed, Type}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Neq { left: Box, right: Box diff --git a/src/modules/expression/binop/or.rs b/src/modules/expression/binop/or.rs index 786fad8d..d9eeac91 100644 --- a/src/modules/expression/binop/or.rs +++ b/src/modules/expression/binop/or.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::translate::compute::{translate_computation, ArithOp}; @@ -9,7 +10,7 @@ use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Or { left: Box, right: Box diff --git a/src/modules/expression/binop/range.rs b/src/modules/expression/binop/range.rs index 68481e05..bb654095 100644 --- a/src/modules/expression/binop/range.rs +++ b/src/modules/expression/binop/range.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::modules::{expression::expr::Expr, types::{Type, Typed}}; @@ -8,7 +9,7 @@ use crate::translate::module::TranslateModule; use crate::utils::TranslateMetadata; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Range { from: Box, to: Box, diff --git a/src/modules/expression/binop/sub.rs b/src/modules/expression/binop/sub.rs index 6bd0d70e..a2c94b7c 100644 --- a/src/modules/expression/binop/sub.rs +++ b/src/modules/expression/binop/sub.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{handle_binop, error_type_match}; use crate::translate::compute::{ArithOp, translate_computation}; @@ -9,7 +10,7 @@ use crate::modules::types::{Typed, Type}; use super::BinOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Sub { left: Box, right: Box diff --git a/src/modules/expression/expr.rs b/src/modules/expression/expr.rs index 0c964d91..8d811c10 100644 --- a/src/modules/expression/expr.rs +++ b/src/modules/expression/expr.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::builtin::len::Len; use crate::modules::command::cmd::Command; @@ -49,7 +50,7 @@ use crate::modules::function::invocation::FunctionInvocation; use crate::modules::builtin::nameof::Nameof; use crate::{document_expression, parse_expr, parse_expr_group, translate_expression}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum ExprType { Bool(Bool), Number(Number), @@ -84,7 +85,7 @@ pub enum ExprType { Is(Is), } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct Expr { pub value: Option, pub kind: Type, diff --git a/src/modules/expression/literal/array.rs b/src/modules/expression/literal/array.rs index 857af8f9..c27d74a4 100644 --- a/src/modules/expression/literal/array.rs +++ b/src/modules/expression/literal/array.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::{expression::expr::Expr, types::{try_parse_type, Type, Typed}}, utils::metadata::ParserMetadata}; use crate::translate::module::TranslateModule; use crate::utils::TranslateMetadata; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Array { exprs: Vec, kind: Type diff --git a/src/modules/expression/literal/bool.rs b/src/modules/expression/literal/bool.rs index 97fb48a4..1f471f6d 100644 --- a/src/modules/expression/literal/bool.rs +++ b/src/modules/expression/literal/bool.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::metadata::ParserMetadata}; use crate::translate::module::TranslateModule; use crate::utils::TranslateMetadata; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bool { value: bool } diff --git a/src/modules/expression/literal/null.rs b/src/modules/expression/literal/null.rs index 69e4cecb..ef7fff06 100644 --- a/src/modules/expression/literal/null.rs +++ b/src/modules/expression/literal/null.rs @@ -1,8 +1,9 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::{ParserMetadata, TranslateMetadata}}; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Null {} impl Typed for Null { diff --git a/src/modules/expression/literal/number.rs b/src/modules/expression/literal/number.rs index d20e108c..6c0211c6 100644 --- a/src/modules/expression/literal/number.rs +++ b/src/modules/expression/literal/number.rs @@ -1,8 +1,9 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::metadata::{ParserMetadata, TranslateMetadata}}; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Number { value: String } diff --git a/src/modules/expression/literal/status.rs b/src/modules/expression/literal/status.rs index 5d09ea9d..7f74ef2e 100644 --- a/src/modules/expression/literal/status.rs +++ b/src/modules/expression/literal/status.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::metadata::ParserMetadata}; use crate::translate::module::TranslateModule; use crate::utils::TranslateMetadata; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Status; impl Typed for Status { diff --git a/src/modules/expression/literal/text.rs b/src/modules/expression/literal/text.rs index 54537383..f47c5481 100644 --- a/src/modules/expression/literal/text.rs +++ b/src/modules/expression/literal/text.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::{ParserMetadata, TranslateMetadata}}; use crate::translate::module::TranslateModule; use crate::modules::expression::expr::Expr; use super::{parse_interpolated_region, translate_interpolated_region}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Text { strings: Vec, interps: Vec, diff --git a/src/modules/expression/parentheses.rs b/src/modules/expression/parentheses.rs index 332566b8..8cae9542 100644 --- a/src/modules/expression/parentheses.rs +++ b/src/modules/expression/parentheses.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::types::{Type, Typed}, utils::metadata::ParserMetadata}; use crate::translate::module::TranslateModule; use super::expr::Expr; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Parentheses { value: Box, kind: Type diff --git a/src/modules/expression/ternop/ternary.rs b/src/modules/expression/ternop/ternary.rs index b8b2eecf..a9db0107 100644 --- a/src/modules/expression/ternop/ternary.rs +++ b/src/modules/expression/ternop/ternary.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::binop::get_binop_position_info; use crate::modules::types::{Type, Typed}; @@ -7,7 +8,7 @@ use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use super::TernOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Ternary { cond: Box, true_expr: Box, diff --git a/src/modules/expression/typeop/cast.rs b/src/modules/expression/typeop/cast.rs index 70c19f38..9a2d6ba6 100644 --- a/src/modules/expression/typeop/cast.rs +++ b/src/modules/expression/typeop/cast.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::modules::expression::expr::Expr; use crate::{docs::module::DocumentationModule, translate::module::TranslateModule}; use crate::utils::cc_flags::{get_ccflag_name, CCFlags}; @@ -8,7 +9,7 @@ use crate::modules::types::{Type, Typed}; use super::TypeOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Cast { expr: Box, kind: Type diff --git a/src/modules/expression/typeop/is.rs b/src/modules/expression/typeop/is.rs index d9927dbf..69e3476d 100644 --- a/src/modules/expression/typeop/is.rs +++ b/src/modules/expression/typeop/is.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::utils::{ParserMetadata, TranslateMetadata}; @@ -7,7 +8,7 @@ use crate::modules::types::{Typed, Type}; use super::TypeOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Is { expr: Box, kind: Type diff --git a/src/modules/expression/unop/neg.rs b/src/modules/expression/unop/neg.rs index 2c7faec5..a03890d7 100644 --- a/src/modules/expression/unop/neg.rs +++ b/src/modules/expression/unop/neg.rs @@ -1,10 +1,11 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{utils::{metadata::ParserMetadata, TranslateMetadata}, modules::types::{Type, Typed}, translate::{module::TranslateModule, compute::{translate_computation, ArithOp}}}; use super::{super::expr::Expr, UnOp}; use crate::docs::module::DocumentationModule; use crate::error_type_match; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Neg { expr: Box } diff --git a/src/modules/expression/unop/not.rs b/src/modules/expression/unop/not.rs index b0944af8..1abe082a 100644 --- a/src/modules/expression/unop/not.rs +++ b/src/modules/expression/unop/not.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::utils::{metadata::ParserMetadata, TranslateMetadata}; use crate::translate::{compute::{translate_computation, ArithOp}, module::TranslateModule}; use crate::modules::types::{Type, Typed}; @@ -7,7 +8,7 @@ use super::super::expr::Expr; use crate::error_type_match; use super::UnOp; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Not { expr: Box } diff --git a/src/modules/function/declaration.rs b/src/modules/function/declaration.rs index e25f83fd..b37366d7 100644 --- a/src/modules/function/declaration.rs +++ b/src/modules/function/declaration.rs @@ -8,6 +8,7 @@ use std::path::Path; use heraclitus_compiler::prelude::*; use itertools::izip; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::statement::comment_doc::CommentDoc; use crate::modules::expression::expr::Expr; @@ -23,7 +24,7 @@ use crate::modules::types::parse_type; use crate::utils::function_metadata::FunctionMetadata; use super::declaration_utils::*; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FunctionDeclaration { pub name: String, pub arg_refs: Vec, diff --git a/src/modules/function/fail.rs b/src/modules/function/fail.rs index 90cb0290..194933fa 100644 --- a/src/modules/function/fail.rs +++ b/src/modules/function/fail.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::modules::types::{Type, Typed}; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Fail { pub expr: Expr, pub code: String, diff --git a/src/modules/function/invocation.rs b/src/modules/function/invocation.rs index a98f81eb..0406a1ef 100644 --- a/src/modules/function/invocation.rs +++ b/src/modules/function/invocation.rs @@ -2,6 +2,7 @@ use std::mem::swap; use heraclitus_compiler::prelude::*; use itertools::izip; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::command::modifier::CommandModifier; use crate::modules::condition::failed::Failed; @@ -12,7 +13,7 @@ use crate::translate::module::TranslateModule; use crate::modules::expression::expr::Expr; use super::invocation_utils::*; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FunctionInvocation { name: String, args: Vec, diff --git a/src/modules/function/ret.rs b/src/modules/function/ret.rs index 5f9317b2..e365a74f 100644 --- a/src/modules/function/ret.rs +++ b/src/modules/function/ret.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::modules::types::{Type, Typed}; @@ -6,7 +7,7 @@ use crate::utils::function_metadata::FunctionMetadata; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Return { pub expr: Expr } diff --git a/src/modules/imports/import.rs b/src/modules/imports/import.rs index e318f325..1a79ce4b 100644 --- a/src/modules/imports/import.rs +++ b/src/modules/imports/import.rs @@ -1,5 +1,6 @@ use std::fs; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::compiler::AmberCompiler; use crate::docs::module::DocumentationModule; use crate::modules::block::Block; @@ -11,7 +12,7 @@ use crate::translate::module::TranslateModule; use crate::Cli; use super::import_string::ImportString; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Import { path: ImportString, token_import: Option, diff --git a/src/modules/imports/import_string.rs b/src/modules/imports/import_string.rs index 35b9104a..894453a0 100644 --- a/src/modules/imports/import_string.rs +++ b/src/modules/imports/import_string.rs @@ -1,8 +1,9 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use std::path::Path; use crate::utils::ParserMetadata; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImportString { pub value: String } diff --git a/src/modules/loops/break_stmt.rs b/src/modules/loops/break_stmt.rs index c26cd7a7..8794480e 100644 --- a/src/modules/loops/break_stmt.rs +++ b/src/modules/loops/break_stmt.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Break; impl SyntaxModule for Break { diff --git a/src/modules/loops/continue_stmt.rs b/src/modules/loops/continue_stmt.rs index 9e1defb2..667ecbbd 100644 --- a/src/modules/loops/continue_stmt.rs +++ b/src/modules/loops/continue_stmt.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Continue; impl SyntaxModule for Continue { diff --git a/src/modules/loops/infinite_loop.rs b/src/modules/loops/infinite_loop.rs index 9f040651..08cfec4d 100644 --- a/src/modules/loops/infinite_loop.rs +++ b/src/modules/loops/infinite_loop.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::translate::module::TranslateModule; use crate::utils::context::Context; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::modules::block::Block; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct InfiniteLoop { block: Block, } diff --git a/src/modules/loops/iter_loop.rs b/src/modules/loops/iter_loop.rs index e962bb90..169e844a 100644 --- a/src/modules/loops/iter_loop.rs +++ b/src/modules/loops/iter_loop.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::modules::types::{Typed, Type}; @@ -8,7 +9,7 @@ use crate::utils::context::Context; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::modules::block::Block; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct IterLoop { block: Block, iter_expr: Expr, diff --git a/src/modules/main.rs b/src/modules/main.rs index 2f95d7cc..ccb269fd 100644 --- a/src/modules/main.rs +++ b/src/modules/main.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; @@ -7,7 +8,7 @@ use crate::modules::block::Block; use super::variable::variable_name_extensions; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Main { pub args: Option, pub block: Block, diff --git a/src/modules/shorthand/add.rs b/src/modules/shorthand/add.rs index 0afeb2ea..fa1c8889 100644 --- a/src/modules/shorthand/add.rs +++ b/src/modules/shorthand/add.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::error_type_match; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::translate::{module::TranslateModule, compute::{ArithOp, translate_computation}}; use crate::modules::types::{Type, Typed}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShorthandAdd { var: String, expr: Box, diff --git a/src/modules/shorthand/div.rs b/src/modules/shorthand/div.rs index 79a68d9b..5de51e0a 100644 --- a/src/modules/shorthand/div.rs +++ b/src/modules/shorthand/div.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::error_type_match; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::translate::{module::TranslateModule, compute::{ArithOp, translate_computation}}; use crate::modules::types::{Type, Typed}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShorthandDiv { var: String, expr: Box, diff --git a/src/modules/shorthand/modulo.rs b/src/modules/shorthand/modulo.rs index f9f92ff9..bba53cae 100644 --- a/src/modules/shorthand/modulo.rs +++ b/src/modules/shorthand/modulo.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::error_type_match; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::translate::{module::TranslateModule, compute::{ArithOp, translate_computation}}; use crate::modules::types::{Type, Typed}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShorthandModulo { var: String, expr: Box, diff --git a/src/modules/shorthand/mul.rs b/src/modules/shorthand/mul.rs index 7d06c87e..5e365ae9 100644 --- a/src/modules/shorthand/mul.rs +++ b/src/modules/shorthand/mul.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::error_type_match; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::translate::{module::TranslateModule, compute::{ArithOp, translate_computation}}; use crate::modules::types::{Type, Typed}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShorthandMul { var: String, expr: Box, diff --git a/src/modules/shorthand/sub.rs b/src/modules/shorthand/sub.rs index 0aed82cc..bbf3340e 100644 --- a/src/modules/shorthand/sub.rs +++ b/src/modules/shorthand/sub.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::error_type_match; use crate::modules::expression::expr::Expr; @@ -8,7 +9,7 @@ use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::translate::{module::TranslateModule, compute::{ArithOp, translate_computation}}; use crate::modules::types::{Type, Typed}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShorthandSub { var: String, expr: Box, diff --git a/src/modules/statement/comment.rs b/src/modules/statement/comment.rs index 4fb8bbc1..a6f445d8 100644 --- a/src/modules/statement/comment.rs +++ b/src/modules/statement/comment.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::utils::metadata::ParserMetadata; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Comment { pub value: String } diff --git a/src/modules/statement/comment_doc.rs b/src/modules/statement/comment_doc.rs index df801372..a3d8e029 100644 --- a/src/modules/statement/comment_doc.rs +++ b/src/modules/statement/comment_doc.rs @@ -1,10 +1,11 @@ use heraclitus_compiler::prelude::*; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::utils::metadata::ParserMetadata; use crate::translate::module::TranslateModule; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CommentDoc { pub value: String } diff --git a/src/modules/statement/stmt.rs b/src/modules/statement/stmt.rs index 8459561d..e9ac9f0d 100644 --- a/src/modules/statement/stmt.rs +++ b/src/modules/statement/stmt.rs @@ -1,5 +1,6 @@ use heraclitus_compiler::prelude::*; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use crate::modules::expression::expr::{Expr, ExprType}; @@ -43,7 +44,7 @@ use crate::modules::builtin::{ use super::comment_doc::CommentDoc; use super::comment::Comment; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum StatementType { Expr(Expr), VariableInit(VariableInit), @@ -73,7 +74,7 @@ pub enum StatementType { CommentDoc(CommentDoc), } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Statement { pub value: Option } diff --git a/src/modules/types.rs b/src/modules/types.rs index 41b2bebb..e35c7ee1 100644 --- a/src/modules/types.rs +++ b/src/modules/types.rs @@ -1,9 +1,10 @@ use std::fmt::Display; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::utils::ParserMetadata; -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub enum Type { #[default] Null, Text, diff --git a/src/modules/variable/get.rs b/src/modules/variable/get.rs index 133357be..50cbcf16 100644 --- a/src/modules/variable/get.rs +++ b/src/modules/variable/get.rs @@ -1,9 +1,10 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::{docs::module::DocumentationModule, modules::{expression::expr::Expr, types::{Type, Typed}}, utils::{ParserMetadata, TranslateMetadata}}; use crate::translate::module::TranslateModule; use super::{variable_name_extensions, handle_variable_reference, handle_index_accessor}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct VariableGet { pub name: String, kind: Type, diff --git a/src/modules/variable/init.rs b/src/modules/variable/init.rs index ad4171c8..231319a5 100644 --- a/src/modules/variable/init.rs +++ b/src/modules/variable/init.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::types::{Typed, Type}; use crate::modules::expression::expr::Expr; @@ -6,7 +7,7 @@ use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use super::{variable_name_extensions, handle_identifier_name}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct VariableInit { name: String, expr: Box, diff --git a/src/modules/variable/set.rs b/src/modules/variable/set.rs index 94fe4031..a6837f85 100644 --- a/src/modules/variable/set.rs +++ b/src/modules/variable/set.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::{modules::expression::expr::Expr, translate::module::TranslateModule}; use crate::utils::{ParserMetadata, TranslateMetadata}; use super::{variable_name_extensions, handle_variable_reference, handle_index_accessor}; use crate::modules::types::{Typed, Type}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct VariableSet { name: String, expr: Box, diff --git a/src/utils/cc_flags.rs b/src/utils/cc_flags.rs index cdd31c1c..c1d1326c 100644 --- a/src/utils/cc_flags.rs +++ b/src/utils/cc_flags.rs @@ -1,4 +1,6 @@ -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] pub enum CCFlags { AllowNestedIfElse, AllowGenericReturn, diff --git a/src/utils/context.rs b/src/utils/context.rs index 0988ab14..2415a85f 100644 --- a/src/utils/context.rs +++ b/src/utils/context.rs @@ -3,9 +3,10 @@ use crate::modules::expression::expr::Expr; use crate::modules::types::Type; use amber_meta::ContextHelper; use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct FunctionDecl { pub name: String, pub arg_names: Vec, @@ -35,7 +36,7 @@ impl FunctionDecl { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct VariableDecl { pub name: String, pub kind: Type, @@ -43,7 +44,7 @@ pub struct VariableDecl { pub is_ref: bool, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct ScopeUnit { pub vars: HashMap, pub funs: HashMap, @@ -94,7 +95,7 @@ impl ScopeUnit { } } -#[derive(Clone, Debug, ContextHelper)] +#[derive(Clone, Debug, ContextHelper, Serialize, Deserialize)] pub struct Context { /// The current index in the expression pub index: usize, diff --git a/src/utils/function_cache.rs b/src/utils/function_cache.rs index 37cd91df..1e9fd4bf 100644 --- a/src/utils/function_cache.rs +++ b/src/utils/function_cache.rs @@ -1,8 +1,10 @@ use std::collections::HashMap; +use serde::{Deserialize, Serialize}; + use crate::modules::{types::Type, block::Block}; use super::context::Context; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] /// This is a compiled function instance pub struct FunctionInstance { pub variant_id: usize, @@ -11,14 +13,14 @@ pub struct FunctionInstance { pub block: Block } -#[derive(Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] /// This is a cached data representing a function pub struct FunctionCacheEntry { pub instances: Vec, pub context: Context } -#[derive(Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] // This is a map of all generated functions based on their invocations pub struct FunctionCache { pub funs: HashMap, diff --git a/src/utils/import_cache.rs b/src/utils/import_cache.rs index 0839ca58..feebb027 100644 --- a/src/utils/import_cache.rs +++ b/src/utils/import_cache.rs @@ -1,19 +1,21 @@ +use serde::{Deserialize, Serialize}; + use crate::modules::block::Block; use super::context::FunctionDecl; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FileMetadata { pub block: Block, pub pub_funs: Vec } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FileCache { pub path: String, pub metadata: Option } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImportCache { /// The paths of the imports (used to be able to resolve imports with topological sort) pub import_graph: Vec>, diff --git a/src/utils/metadata/parser.rs b/src/utils/metadata/parser.rs index 9a41f4a0..516e17e3 100644 --- a/src/utils/metadata/parser.rs +++ b/src/utils/metadata/parser.rs @@ -2,6 +2,7 @@ use std::collections::BTreeSet; use heraclitus_compiler::prelude::*; use amber_meta::ContextManager; +use serde::{Deserialize, Serialize}; use crate::modules::block::Block; use crate::modules::types::Type; use crate::utils::context::{Context, ScopeUnit, VariableDecl, FunctionDecl}; @@ -9,7 +10,7 @@ use crate::utils::function_interface::FunctionInterface; use crate::utils::import_cache::ImportCache; use crate::utils::function_cache::FunctionCache; -#[derive(Debug, ContextManager)] +#[derive(Debug, Clone, ContextManager, Serialize, Deserialize)] pub struct ParserMetadata { /// Code if the parser is in eval mode pub eval_code: Option, From d0a6b21a7bf7ecdb85e1d4c46b86c8b5b135efa9 Mon Sep 17 00:00:00 2001 From: b1ek Date: Fri, 15 Nov 2024 16:46:33 +1000 Subject: [PATCH 02/10] add compiler cache --- Cargo.lock | 330 ++++++++++++++++++++++++------------------ Cargo.toml | 4 +- build.rs | 7 + src/compiler.rs | 45 ++++++ src/compiler/cache.rs | 85 +++++++++++ 5 files changed, 333 insertions(+), 138 deletions(-) create mode 100644 src/compiler/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 11c75646..d623b2b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,8 @@ dependencies = [ "include_dir", "itertools", "predicates", + "rmp-serde", + "serde", "similar-string", "tempfile", "test-generator", @@ -37,9 +39,9 @@ dependencies = [ name = "amber-meta" version = "0.1.0" dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.36", - "syn 2.0.68", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.87", ] [[package]] @@ -59,9 +61,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -74,36 +76,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -114,13 +116,14 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "assert_cmd" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" +checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" dependencies = [ "anstyle", "bstr", "doc-comment", + "libc", "predicates", "predicates-core", "predicates-tree", @@ -129,9 +132,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bitflags" @@ -141,9 +144,9 @@ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bstr" -version = "1.9.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" dependencies = [ "memchr", "regex-automata", @@ -165,6 +168,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "capitalize" version = "0.3.4" @@ -173,9 +182,12 @@ checksum = "6b5271031022835ee8c7582fe67403bd6cb3d962095787af7921027234bab5bf" [[package]] name = "cc" -version = "1.0.104" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -194,7 +206,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -205,9 +217,9 @@ checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" [[package]] name = "clap" -version = "4.5.8" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -215,9 +227,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.8" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -227,27 +239,27 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.8" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", - "proc-macro2 1.0.86", - "quote 1.0.36", - "syn 2.0.68", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.87", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" @@ -261,9 +273,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "difflib" @@ -295,9 +307,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "float-cmp" @@ -322,13 +334,12 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heraclitus-compiler" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c689c82b381fc881791741e1ab4a4d03cf3cf0f453b30a9f4ed88a54cd8798e" +version = "1.8.0" dependencies = [ "capitalize", "colored", "pad", + "serde", ] [[package]] @@ -339,9 +350,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -375,15 +386,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.36", + "proc-macro2 1.0.89", + "quote 1.0.37", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -396,9 +407,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -411,9 +422,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "linux-raw-sys" @@ -450,9 +461,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "pad" @@ -463,11 +474,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "predicates" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" dependencies = [ "anstyle", "difflib", @@ -479,15 +496,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" dependencies = [ "predicates-core", "termtree", @@ -504,9 +521,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -522,18 +539,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ - "proc-macro2 1.0.86", + "proc-macro2 1.0.89", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -543,9 +560,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -554,15 +571,37 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" +dependencies = [ + "byteorder", + "rmp", + "serde", +] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ "bitflags", "errno", @@ -588,24 +627,30 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.36", - "syn 2.0.68", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.87", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "similar-string" version = "1.4.3" @@ -631,25 +676,26 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.36", + "proc-macro2 1.0.89", + "quote 1.0.37", "unicode-ident", ] [[package]] name = "tempfile" -version = "3.10.1" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -684,15 +730,15 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" @@ -717,57 +763,58 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.86", - "quote 1.0.36", - "syn 2.0.68", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ - "quote 1.0.36", + "quote 1.0.37", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.36", - "syn 2.0.68", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wildmatch" @@ -781,7 +828,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -799,7 +846,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -819,18 +875,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -841,9 +897,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -853,9 +909,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -865,15 +921,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -883,9 +939,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -895,9 +951,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -907,9 +963,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -919,6 +975,6 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/Cargo.toml b/Cargo.toml index 3c501685..7518a7ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,9 +20,11 @@ chrono = "0.4.38" clap = { version = "4.4.18", features = ["derive"] } colored = "2.0.0" glob = "0.3" -heraclitus-compiler = "1.8.1" +heraclitus-compiler = { path = "../heraclitus", features = ["serde"] } include_dir = "0.7.4" itertools = "0.13.0" +rmp-serde = "1.3.0" +serde = { version = "1.0.215", features = ["derive"] } similar-string = "1.4.2" test-generator = "0.3.1" wildmatch = "2.4.0" diff --git a/build.rs b/build.rs index 8e208bc5..fd8b49b5 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,11 @@ +use std::process::Command; + fn main() { // See [https://doc.rust-lang.org/cargo/reference/build-scripts.html]. build_helper::rerun_if_changed("src/tests"); + + let git_hash = Command::new("git").arg("rev-parse").arg("HEAD").output().unwrap(); + let git_hash = String::from_utf8(git_hash.stdout).unwrap(); + + println!("cargo:rustc-env=GIT_HASH={git_hash}"); } diff --git a/src/compiler.rs b/src/compiler.rs index 44dddb68..e864b010 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -5,6 +5,7 @@ use crate::translate::check_all_blocks; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::{rules, Cli}; +use cache::PreparsedFile; use postprocessor::PostProcessor; use chrono::prelude::*; use colored::Colorize; @@ -19,6 +20,7 @@ use std::process::{Command, ExitStatus}; use std::time::Instant; pub mod postprocessor; +pub mod cache; const NO_CODE_PROVIDED: &str = "No code has been provided to the compiler"; const AMBER_DEBUG_PARSER: &str = "AMBER_DEBUG_PARSER"; @@ -96,6 +98,10 @@ impl AmberCompiler { tokens: Vec, is_docs_gen: bool, ) -> Result<(Block, ParserMetadata), Message> { + if let Some(cached) = self.load_cache()? { + return Ok(( cached.block, cached.meta )) + } + let code = self.cc.code.as_ref().expect(NO_CODE_PROVIDED).clone(); let mut meta = ParserMetadata::new(tokens, self.path.clone(), Some(code)); meta.is_docs_gen = is_docs_gen; @@ -118,6 +124,20 @@ impl AmberCompiler { time.elapsed().as_millis() ); } + + let time = Instant::now(); + if let Some(path) = self.path.clone() { + if let Ok(_) = PreparsedFile::save(&path, block.clone(), &meta) { + if Self::env_flag_set(AMBER_DEBUG_TIME) { + println!( + "[{}]\tin\t{}ms\t{path}", + "Saved cache".blue(), + time.elapsed().as_millis() + ); + } + } + } + // Return result match result { Ok(()) => Ok((block, meta)), @@ -253,7 +273,32 @@ impl AmberCompiler { .show(); } + pub fn load_cache(&self) -> Result, Message> { + if self.cli_opts.no_cache { + return Ok(None) + } + + if let Some(path) = self.path.clone() { + let time = Instant::now(); + if let Some(cached) = PreparsedFile::load_for(&path).map_err(|err| Message::new_err_msg(err.to_string()))? { + if Self::env_flag_set(AMBER_DEBUG_TIME) { + let pathname = self.path.clone().unwrap_or(String::from("unknown")); + println!( + "[{}]\tin\t{}ms\t{pathname}", + "Loaded cache".blue(), + time.elapsed().as_millis() + ) + } + return Ok(Some(cached)) + } + } + Ok(None) + } + pub fn compile(&self) -> Result<(Vec, String), Message> { + if let Some(cached) = self.load_cache()? { + return Ok(( vec![], self.translate(cached.block, cached.meta)? )) + } let tokens = self.tokenize()?; let (block, meta) = self.parse(tokens, false)?; let messages = meta.messages.clone(); diff --git a/src/compiler/cache.rs b/src/compiler/cache.rs new file mode 100644 index 00000000..88f2355f --- /dev/null +++ b/src/compiler/cache.rs @@ -0,0 +1,85 @@ +use std::{error::Error, fs::{self, File, Metadata}, path::PathBuf, time::SystemTime}; +use serde::{Serialize, Deserialize}; + +use crate::{modules::block::Block, utils::ParserMetadata}; + +const GIT_HASH: &'static str = env!("GIT_HASH"); + +#[derive(Debug, Serialize, Deserialize)] +pub struct PreparsedFile { + pub block: Block, + pub meta: ParserMetadata, + pub mtime: SystemTime, + pub len: u64, + pub amber_git_hash: String +} + +impl PreparsedFile { + pub fn load_for>(filename: T) -> Result, Box> { + let mut filename: PathBuf = filename.into(); + if ! filename.is_file() { + return Ok(None); + } + + let meta = File::open(&filename)?; + let meta = meta.metadata()?; + + filename.set_extension("abc"); // amber compiled + if ! filename.is_file() { + return Ok(None); + } + + let preparsed = fs::read(&filename)?; + if let Ok(preparsed) = rmp_serde::from_slice::(&preparsed) { + + if ! preparsed.validate(&meta) { + fs::remove_file(&filename)?; + return Ok(None); + } + + Ok(Some(preparsed)) + } else { + Ok(None) + } + } + + pub fn save>(filename: T, block: Block, pmeta: &ParserMetadata) -> Result> { + let filename: PathBuf = filename.into(); + + let meta = File::open(&filename)?; + let meta = meta.metadata()?; + + let preparsed = PreparsedFile { + block, + meta: pmeta.clone(), + mtime: meta.modified()?, + len: meta.len(), + amber_git_hash: GIT_HASH.to_string() + }; + + preparsed.save_to_file(filename)?; + + Ok(preparsed) + } + + fn save_to_file>(&self, filename: T) -> Result<(), Box> { + let mut filename: PathBuf = filename.into(); + filename.set_extension("abc"); + + let serialized = rmp_serde::to_vec(self)?; + fs::write(filename, serialized)?; + + Ok(()) + } + + fn validate(self: &Self, meta: &Metadata) -> bool { + match meta.modified() { + Ok(v) => { + v == self.mtime + && meta.len() == self.len + && self.amber_git_hash == GIT_HASH + }, + Err(_) => false + } + } +} From 8b613f5e9a1d5e69c1ffce6323e1a3317f5d48b0 Mon Sep 17 00:00:00 2001 From: b1ek Date: Fri, 15 Nov 2024 16:46:41 +1000 Subject: [PATCH 03/10] no cache flag --- src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.rs b/src/main.rs index 5b10fd2f..37836df8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,9 @@ pub struct Cli { /// Minify the resulting code #[arg(long)] minify: bool, + + #[arg(long, short)] + no_cache: bool } fn main() -> Result<(), Box> { From 8b1dfecf2cd6a15f19d23aa94ce319f09cebbf8b Mon Sep 17 00:00:00 2001 From: b1ek Date: Fri, 15 Nov 2024 17:22:09 +1000 Subject: [PATCH 04/10] move preparsed cache to cache::parse and add file meta to compiler --- src/compiler.rs | 21 ++++++--- src/compiler/cache.rs | 84 +--------------------------------- src/compiler/cache/parse.rs | 85 +++++++++++++++++++++++++++++++++++ src/compiler/file_source.rs | 23 ++++++++++ src/main.rs | 37 +++++++++++++-- src/modules/imports/import.rs | 29 +++++++++--- 6 files changed, 182 insertions(+), 97 deletions(-) create mode 100644 src/compiler/cache/parse.rs create mode 100644 src/compiler/file_source.rs diff --git a/src/compiler.rs b/src/compiler.rs index e864b010..b18fadce 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -5,7 +5,8 @@ use crate::translate::check_all_blocks; use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::{rules, Cli}; -use cache::PreparsedFile; +use cache::parse::PreparsedFile; +use file_source::FileMeta; use postprocessor::PostProcessor; use chrono::prelude::*; use colored::Colorize; @@ -20,6 +21,7 @@ use std::process::{Command, ExitStatus}; use std::time::Instant; pub mod postprocessor; +pub mod file_source; pub mod cache; const NO_CODE_PROVIDED: &str = "No code has been provided to the compiler"; @@ -30,14 +32,16 @@ pub struct AmberCompiler { pub cc: Compiler, pub path: Option, pub cli_opts: Cli, + pub file_meta: FileMeta } impl AmberCompiler { - pub fn new(code: String, path: Option, cli_opts: Cli) -> AmberCompiler { + pub fn new(code: String, path: Option, cli_opts: Cli, file_meta: FileMeta) -> AmberCompiler { AmberCompiler { cc: Compiler::new("Amber", rules::get_rules()), path, cli_opts, + file_meta } .load_code(AmberCompiler::comment_shebang(code)) } @@ -98,7 +102,7 @@ impl AmberCompiler { tokens: Vec, is_docs_gen: bool, ) -> Result<(Block, ParserMetadata), Message> { - if let Some(cached) = self.load_cache()? { + if let Some(cached) = self.load_parse_cache()? { return Ok(( cached.block, cached.meta )) } @@ -273,8 +277,13 @@ impl AmberCompiler { .show(); } - pub fn load_cache(&self) -> Result, Message> { - if self.cli_opts.no_cache { + pub fn cache_disabled(&self) -> bool { + self.cli_opts.no_cache + || self.file_meta.source.cache_disabled() + } + + pub fn load_parse_cache(&self) -> Result, Message> { + if self.cache_disabled() { return Ok(None) } @@ -296,7 +305,7 @@ impl AmberCompiler { } pub fn compile(&self) -> Result<(Vec, String), Message> { - if let Some(cached) = self.load_cache()? { + if let Some(cached) = self.load_parse_cache()? { return Ok(( vec![], self.translate(cached.block, cached.meta)? )) } let tokens = self.tokenize()?; diff --git a/src/compiler/cache.rs b/src/compiler/cache.rs index 88f2355f..38c0eba2 100644 --- a/src/compiler/cache.rs +++ b/src/compiler/cache.rs @@ -1,85 +1,3 @@ -use std::{error::Error, fs::{self, File, Metadata}, path::PathBuf, time::SystemTime}; -use serde::{Serialize, Deserialize}; - -use crate::{modules::block::Block, utils::ParserMetadata}; - const GIT_HASH: &'static str = env!("GIT_HASH"); -#[derive(Debug, Serialize, Deserialize)] -pub struct PreparsedFile { - pub block: Block, - pub meta: ParserMetadata, - pub mtime: SystemTime, - pub len: u64, - pub amber_git_hash: String -} - -impl PreparsedFile { - pub fn load_for>(filename: T) -> Result, Box> { - let mut filename: PathBuf = filename.into(); - if ! filename.is_file() { - return Ok(None); - } - - let meta = File::open(&filename)?; - let meta = meta.metadata()?; - - filename.set_extension("abc"); // amber compiled - if ! filename.is_file() { - return Ok(None); - } - - let preparsed = fs::read(&filename)?; - if let Ok(preparsed) = rmp_serde::from_slice::(&preparsed) { - - if ! preparsed.validate(&meta) { - fs::remove_file(&filename)?; - return Ok(None); - } - - Ok(Some(preparsed)) - } else { - Ok(None) - } - } - - pub fn save>(filename: T, block: Block, pmeta: &ParserMetadata) -> Result> { - let filename: PathBuf = filename.into(); - - let meta = File::open(&filename)?; - let meta = meta.metadata()?; - - let preparsed = PreparsedFile { - block, - meta: pmeta.clone(), - mtime: meta.modified()?, - len: meta.len(), - amber_git_hash: GIT_HASH.to_string() - }; - - preparsed.save_to_file(filename)?; - - Ok(preparsed) - } - - fn save_to_file>(&self, filename: T) -> Result<(), Box> { - let mut filename: PathBuf = filename.into(); - filename.set_extension("abc"); - - let serialized = rmp_serde::to_vec(self)?; - fs::write(filename, serialized)?; - - Ok(()) - } - - fn validate(self: &Self, meta: &Metadata) -> bool { - match meta.modified() { - Ok(v) => { - v == self.mtime - && meta.len() == self.len - && self.amber_git_hash == GIT_HASH - }, - Err(_) => false - } - } -} +pub mod parse; \ No newline at end of file diff --git a/src/compiler/cache/parse.rs b/src/compiler/cache/parse.rs new file mode 100644 index 00000000..d30b1fa1 --- /dev/null +++ b/src/compiler/cache/parse.rs @@ -0,0 +1,85 @@ +use std::{error::Error, fs::{self, File, Metadata}, path::PathBuf, time::SystemTime}; +use serde::{Serialize, Deserialize}; + +use crate::{modules::block::Block, utils::ParserMetadata}; + +use super::GIT_HASH; + +#[derive(Debug, Serialize, Deserialize)] +pub struct PreparsedFile { + pub block: Block, + pub meta: ParserMetadata, + pub mtime: SystemTime, + pub len: u64, + pub amber_git_hash: String +} + +impl PreparsedFile { + pub fn load_for>(filename: T) -> Result, Box> { + let mut filename: PathBuf = filename.into(); + if ! filename.is_file() { + return Ok(None); + } + + let meta = File::open(&filename)?; + let meta = meta.metadata()?; + + filename.set_extension("abc"); // amber compiled + if ! filename.is_file() { + return Ok(None); + } + + let preparsed = fs::read(&filename)?; + if let Ok(preparsed) = rmp_serde::from_slice::(&preparsed) { + + if ! preparsed.validate(&meta) { + fs::remove_file(&filename)?; + return Ok(None); + } + + Ok(Some(preparsed)) + } else { + Ok(None) + } + } + + pub fn save>(filename: T, block: Block, pmeta: &ParserMetadata) -> Result> { + let filename: PathBuf = filename.into(); + + let meta = File::open(&filename)?; + let meta = meta.metadata()?; + + let preparsed = PreparsedFile { + block, + meta: pmeta.clone(), + mtime: meta.modified()?, + len: meta.len(), + amber_git_hash: GIT_HASH.to_string() + }; + + preparsed.save_to_file(filename)?; + + Ok(preparsed) + } + + fn save_to_file>(&self, filename: T) -> Result<(), Box> { + let mut filename: PathBuf = filename.into(); + filename.set_extension("abc"); + + let serialized = rmp_serde::to_vec(self)?; + fs::write(filename, serialized)?; + + Ok(()) + } + + fn validate(self: &Self, meta: &Metadata) -> bool { + match meta.modified() { + Ok(v) => { + v == self.mtime + && meta.len() == self.len + && self.amber_git_hash == GIT_HASH + }, + Err(_) => false + } + } +} diff --git a/src/compiler/file_source.rs b/src/compiler/file_source.rs new file mode 100644 index 00000000..c33acf37 --- /dev/null +++ b/src/compiler/file_source.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy)] +pub struct FileMeta { + pub is_import: bool, + pub source: FileSource +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum FileSource { + File, + Stdlib, + Stream +} + +impl FileSource { + pub fn cache_disabled(&self) -> bool { + match &self { + Self::Stream => true, + _ => false + } + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 37836df8..8da1c96a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ pub mod tests; use crate::compiler::AmberCompiler; use clap::Parser; use colored::Colorize; +use compiler::file_source::{FileMeta, FileSource}; use heraclitus_compiler::prelude::*; use std::error::Error; use std::fs; @@ -65,6 +66,8 @@ fn main() -> Result<(), Box> { } fn handle_compile(cli: Cli) -> Result<(), Box> { + let mut file_source = FileSource::File; + let input = if let Some(input) = cli.input.clone() { String::from(input.to_string_lossy().trim()) } else { @@ -72,6 +75,7 @@ fn handle_compile(cli: Cli) -> Result<(), Box> { }; let code = if input == "-" { + file_source = FileSource::Stream; let mut buf = String::new(); match stdin().read_to_string(&mut buf) { Ok(_) => buf, @@ -84,7 +88,16 @@ fn handle_compile(cli: Cli) -> Result<(), Box> { } }; - let (messages, code) = match AmberCompiler::new(code, Some(input), cli.clone()).compile() { + let compiler = AmberCompiler::new( + code, + Some(input), + cli.clone(), + FileMeta { + is_import: false, + source: file_source + } + ); + let (messages, code) = match compiler.compile() { Ok(result) => result, Err(err) => { err.show(); @@ -126,7 +139,16 @@ fn handle_compile(cli: Cli) -> Result<(), Box> { } fn handle_eval(code: String, cli: Cli) -> Result<(), Box> { - match AmberCompiler::new(code, None, cli).compile() { + let compiler = AmberCompiler::new( + code, + None, + cli, + FileMeta { + is_import: false, + source: FileSource::Stream + } + ); + match compiler.compile() { Ok((messages, code)) => { messages.iter().for_each(|m| m.show()); (!messages.is_empty()).then(render_dash); @@ -172,7 +194,16 @@ fn handle_docs(cli: Cli) -> Result<(), Box> { } }; - match AmberCompiler::new(code, Some(input), cli).generate_docs(output) { + let compiler = AmberCompiler::new( + code, + Some(input), + cli, + FileMeta { + is_import: false, + source: FileSource::File + } + ); + match compiler.generate_docs(output) { Ok(_) => Ok(()), Err(err) => { err.show(); diff --git a/src/modules/imports/import.rs b/src/modules/imports/import.rs index 1a79ce4b..1a088e93 100644 --- a/src/modules/imports/import.rs +++ b/src/modules/imports/import.rs @@ -1,6 +1,7 @@ use std::fs; use heraclitus_compiler::prelude::*; use serde::{Deserialize, Serialize}; +use crate::compiler::file_source::{FileMeta, FileSource}; use crate::compiler::AmberCompiler; use crate::docs::module::DocumentationModule; use crate::modules::block::Block; @@ -19,7 +20,8 @@ pub struct Import { token_path: Option, is_all: bool, is_pub: bool, - export_defs: Vec<(String, Option, Option)> + export_defs: Vec<(String, Option, Option)>, + source: Option } impl Import { @@ -74,13 +76,19 @@ impl Import { fn resolve_import(&mut self, meta: &ParserMetadata) -> Result { if self.path.value.starts_with("std/") { match stdlib::resolve(self.path.value.replace("std/", "")) { - Some(v) => Ok(v), + Some(v) => { + self.source = Some(FileSource::Stdlib); + Ok(v) + }, None => error!(meta, self.token_path.clone(), format!("Standard library module '{}' does not exist", self.path.value)) } } else { match fs::read_to_string(self.path.value.clone()) { - Ok(content) => Ok(content), + Ok(content) => { + self.source = Some(FileSource::File); + Ok(content) + }, Err(err) => error!(meta, self.token_path.clone() => { message: format!("Could not read file '{}'", self.path.value), comment: err.to_string() @@ -98,7 +106,17 @@ impl Import { } fn handle_compile_code(&mut self, meta: &mut ParserMetadata, imported_code: String) -> SyntaxResult { - match AmberCompiler::new(imported_code.clone(), Some(self.path.value.clone()), Cli::default()).tokenize() { + let compiler = AmberCompiler::new( + imported_code.clone(), + Some(self.path.value.clone()), + Cli::default(), + FileMeta { + is_import: true, + source: self.source.unwrap() + } + ); + + match compiler.tokenize() { Ok(tokens) => { let mut block = Block::new(); // Save snapshot of current file @@ -130,7 +148,8 @@ impl SyntaxModule for Import { token_path: None, is_all: false, is_pub: false, - export_defs: vec![] + export_defs: vec![], + source: None } } From 5ee632ee277be93e35fce8ea37038f26657aadf8 Mon Sep 17 00:00:00 2001 From: b1ek Date: Fri, 15 Nov 2024 17:22:42 +1000 Subject: [PATCH 05/10] fix tests --- src/tests/mod.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 2e7d26a2..f0efcc05 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,3 +1,4 @@ +use crate::compiler::file_source::{FileMeta, FileSource}; use crate::compiler::AmberCompiler; use crate::Cli; extern crate test_generator; @@ -13,9 +14,21 @@ pub mod postprocessor; mod stdlib; mod validity; +pub fn compiler(code: impl Into) -> AmberCompiler { + AmberCompiler::new( + code.into(), + None, + Cli::default(), + FileMeta { + is_import: false, + source: FileSource::Stream + } + ) +} + /// compare the output of the given code with the expected output pub fn test_amber(code: impl Into, result: impl AsRef) { - match AmberCompiler::new(code.into(), None, Cli::default()).test_eval() { + match compiler(code).test_eval() { Ok(eval_result) => assert_eq!( eval_result.trim_end_matches('\n'), result.as_ref().trim_end_matches('\n') @@ -27,10 +40,11 @@ pub fn test_amber(code: impl Into, result: impl AsRef) { pub fn compile_code>(code: T) -> String { let cli = Cli { no_proc: vec!["*".into()], + no_cache: true, ..Cli::default() }; - AmberCompiler::new(code.into(), None, cli) + compiler(code) .compile() .unwrap() .1 From a9bf59a0207a9c834f337683a7cdbc1254cb98ca Mon Sep 17 00:00:00 2001 From: b1ek Date: Sat, 16 Nov 2024 13:52:33 +1000 Subject: [PATCH 06/10] tokenize cache for stdlib --- Cargo.lock | 10 + Cargo.toml | 1 + src/compiler.rs | 52 +++++ src/compiler/cache.rs | 18 +- src/compiler/cache/parse.rs | 24 ++- src/compiler/cache/tokenize.rs | 199 ++++++++++++++++++ src/compiler/file_source.rs | 4 +- src/tests/validity/range_loop_float.ab | 8 + .../validity/range_loop_float_negative.ab | 8 + src/tests/validity/range_loop_negative.ab | 7 + src/tests/validity/range_loop_noop.ab | 7 + 11 files changed, 331 insertions(+), 7 deletions(-) create mode 100644 src/compiler/cache/tokenize.rs create mode 100644 src/tests/validity/range_loop_float.ab create mode 100644 src/tests/validity/range_loop_float_negative.ab create mode 100644 src/tests/validity/range_loop_negative.ab create mode 100644 src/tests/validity/range_loop_noop.ab diff --git a/Cargo.lock b/Cargo.lock index d623b2b3..6722ad70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,7 @@ dependencies = [ "colored", "glob", "heraclitus-compiler", + "home", "include_dir", "itertools", "predicates", @@ -342,6 +343,15 @@ dependencies = [ "serde", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "httpdate" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 7518a7ba..a05a8229 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ clap = { version = "4.4.18", features = ["derive"] } colored = "2.0.0" glob = "0.3" heraclitus-compiler = { path = "../heraclitus", features = ["serde"] } +home = "0.5.9" include_dir = "0.7.4" itertools = "0.13.0" rmp-serde = "1.3.0" diff --git a/src/compiler.rs b/src/compiler.rs index b18fadce..32d867d5 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -6,6 +6,7 @@ use crate::translate::module::TranslateModule; use crate::utils::{ParserMetadata, TranslateMetadata}; use crate::{rules, Cli}; use cache::parse::PreparsedFile; +use cache::tokenize::PretokenizedFile; use file_source::FileMeta; use postprocessor::PostProcessor; use chrono::prelude::*; @@ -68,6 +69,19 @@ impl AmberCompiler { } pub fn tokenize(&self) -> Result, Message> { + let time = Instant::now(); + if let Some(cache) = self.load_token_cache()? { + if Self::env_flag_set(AMBER_DEBUG_TIME) { + let pathname = self.path.clone().unwrap_or(String::from("unknown")); + println!( + "[{}]\tin\t{}ms\t{pathname}", + "Loaded cache".cyan(), + time.elapsed().as_millis() + ); + } + return Ok(cache.tokens) + } + let time = Instant::now(); match self.cc.tokenize() { Ok(tokens) => { @@ -79,6 +93,21 @@ impl AmberCompiler { time.elapsed().as_millis() ); } + let time = Instant::now(); + if ! self.cache_disabled() { + if let Some(filename) = self.path.clone() { + PretokenizedFile::save(filename, self.file_meta, tokens.clone()).map_err(|x| Message::new_err_msg(format!("Error while saving token cache: {x}")))?; + if Self::env_flag_set(AMBER_DEBUG_TIME) { + let pathname = self.path.clone().unwrap_or(String::from("unknown")); + println!( + "[{}]\tin\t{}ms\t{pathname}", + "Saved cache".cyan(), + time.elapsed().as_millis() + ); + } + } + } + Ok(tokens) } Err((err_type, pos)) => { @@ -304,6 +333,29 @@ impl AmberCompiler { Ok(None) } + pub fn load_token_cache(&self) -> Result, Message> { + if self.cache_disabled() { + return Ok(None) + } + + if let Some(path) = self.path.clone() { + let time = Instant::now(); + if let Some(cached) = PretokenizedFile::load_for(path, self.file_meta).map_err(|err| Message::new_err_msg(err.to_string()))? { + if Self::env_flag_set(AMBER_DEBUG_TIME) { + let pathname = self.path.clone().unwrap_or(String::from("unknown")); + println!( + "[{}]\tin\t{}ms\t{pathname}", + "Loaded cache".cyan(), + time.elapsed().as_millis() + ) + } + return Ok(Some(cached)) + } + } + + Ok(None) + } + pub fn compile(&self) -> Result<(Vec, String), Message> { if let Some(cached) = self.load_parse_cache()? { return Ok(( vec![], self.translate(cached.block, cached.meta)? )) diff --git a/src/compiler/cache.rs b/src/compiler/cache.rs index 38c0eba2..cd28b053 100644 --- a/src/compiler/cache.rs +++ b/src/compiler/cache.rs @@ -1,3 +1,19 @@ +use std::{fs, path::PathBuf}; + const GIT_HASH: &'static str = env!("GIT_HASH"); -pub mod parse; \ No newline at end of file +pub mod parse; +pub mod tokenize; + +pub fn home_cache() -> Option { + if let Some(mut home) = home::home_dir() { + home.push(".cache"); + home.push("amber"); + if ! home.is_dir() { + fs::create_dir_all(&home).expect("Couldn't create ~/.cache/amber"); + } + Some(home) + } else { + None + } +} diff --git a/src/compiler/cache/parse.rs b/src/compiler/cache/parse.rs index d30b1fa1..ac205824 100644 --- a/src/compiler/cache/parse.rs +++ b/src/compiler/cache/parse.rs @@ -5,6 +5,8 @@ use crate::{modules::block::Block, utils::ParserMetadata}; use super::GIT_HASH; +pub const FILE_EXT: &'static str = "ab.pr.c"; + #[derive(Debug, Serialize, Deserialize)] pub struct PreparsedFile { pub block: Block, @@ -24,13 +26,13 @@ impl PreparsedFile { let meta = File::open(&filename)?; let meta = meta.metadata()?; - filename.set_extension("abc"); // amber compiled + filename.set_extension(FILE_EXT); // amber compiled if ! filename.is_file() { return Ok(None); } let preparsed = fs::read(&filename)?; - if let Ok(preparsed) = rmp_serde::from_slice::(&preparsed) { + if let Ok(preparsed) = Self::try_from(preparsed) { if ! preparsed.validate(&meta) { fs::remove_file(&filename)?; @@ -64,9 +66,9 @@ impl PreparsedFile { fn save_to_file>(&self, filename: T) -> Result<(), Box> { let mut filename: PathBuf = filename.into(); - filename.set_extension("abc"); + filename.set_extension(FILE_EXT); - let serialized = rmp_serde::to_vec(self)?; + let serialized: Vec = self.try_into()?; fs::write(filename, serialized)?; Ok(()) @@ -83,3 +85,17 @@ impl PreparsedFile { } } } + +impl TryInto> for &PreparsedFile { + type Error = String; + fn try_into(self) -> Result, Self::Error> { + Ok(rmp_serde::to_vec(&self).map_err(|x| x.to_string())?) + } +} + +impl TryFrom> for PreparsedFile { + type Error = String; + fn try_from(value: Vec) -> Result { + Ok(rmp_serde::from_slice(&value).map_err(|x| x.to_string())?) + } +} diff --git a/src/compiler/cache/tokenize.rs b/src/compiler/cache/tokenize.rs new file mode 100644 index 00000000..18eaa2d4 --- /dev/null +++ b/src/compiler/cache/tokenize.rs @@ -0,0 +1,199 @@ +use std::fs::File; +use std::{fs, path::PathBuf}; +use std::time::SystemTime; + +use heraclitus_compiler::prelude::Token; +use serde::{Deserialize, Serialize}; + +use crate::compiler::cache::GIT_HASH; +use crate::compiler::file_source::{FileMeta, FileSource}; +use crate::stdlib; + +use super::home_cache; + +pub const FILE_EXT: &'static str = "ab.tk.c"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PretokenizedFile { + pub tokens: Vec, + pub mtime: Option, + pub len: u64, + pub amber_git_hash: String, + pub file_meta: FileMeta +} + +impl PretokenizedFile { + pub fn load_for(filename: impl Into, file_meta: FileMeta) -> Result, Box> { + if file_meta.source.cache_disabled() { + return Ok(None) + } + + let filename: PathBuf = filename.into(); + + match file_meta.source { + FileSource::File => { + if ! filename.is_file() { + println!("not a file: {filename:?}"); + return Ok(None) + } + }, + FileSource::Stdlib => { + if Self::get_stdlib(&filename).is_none() { + println!("not a stdlib: {filename:?}"); + return Ok(None) + } + }, + _ => unimplemented!() + } + + let cache_filename = Self::get_path(&filename, file_meta); + + if let Some(cache_filename) = cache_filename { + + if ! cache_filename.is_file() { + println!("not a cachefile: {filename:?}"); + return Ok(None) + } + + let parsed = Self::try_from(fs::read(cache_filename)?)?; + + if parsed.validate(&filename, file_meta) { + Ok(Some(parsed)) + } else { + println!("invalid cachefile for {filename:?}"); + Ok(None) + } + + } else { + Err(String::from("Couldn't get cache filename!").into()) + } + } + + pub fn save(filename: impl Into, file_meta: FileMeta, tokens: Vec) -> Result<(), Box> { + let filename: PathBuf = filename.into(); + + let (mtime, len) = match file_meta.source { + FileSource::File => { + let file = File::open(&filename)?; + let meta = file.metadata()?; + (Some(meta.modified()?), meta.len()) + }, + FileSource::Stdlib => { + if let Some(file) = Self::get_stdlib(&filename) { + + (None, file.len() as u64) + + } else { + return Err(String::from("Couldn't find stdlib file").into()) + } + } + _ => unimplemented!() + }; + + let cache = Self { + tokens, + mtime, + len, + amber_git_hash: GIT_HASH.into(), + file_meta + }; + + cache.save_to_file(&filename, file_meta)?; + + Ok(()) + } + + fn get_stdlib(filename: impl Into) -> Option { + let filename = filename.into(); + stdlib::resolve(filename.components().last().unwrap().as_os_str().to_str().unwrap()) + } + + fn save_to_file(&self, filename: impl Into, file_meta: FileMeta) -> Result<(), Box> { + let filename: PathBuf = filename.into(); + + if let Some(cache_file) = Self::get_path(&filename, file_meta) { + let serialized: Vec = self.try_into()?; + fs::write(&cache_file, serialized).map_err(|x| format!("Cannot write to {cache_file:?}: {x}"))?; + Ok(()) + } else { + Err(String::from("Couldn't get path to saved directory").into()) + } + } + + fn get_path(filename: impl Into, file_meta: FileMeta) -> Option { + let filename: PathBuf = filename.into(); + let filename: PathBuf = filename.to_str().unwrap().replace('/', "_").replace(":", "_").into(); + + match file_meta.source { + FileSource::Stdlib => { + home_cache().map(|mut x| { + x.push(&filename); + x.set_extension(FILE_EXT); + x + }) + }, + FileSource::File => { + let mut filename = filename.clone(); + filename.set_extension(FILE_EXT); + Some(filename) + }, + _ => unimplemented!() + } + } + + fn validate(&self, filename: impl Into, file_meta: FileMeta) -> bool { + assert_eq!(self.file_meta, file_meta); + let filename: PathBuf = filename.into(); + + if file_meta.source.cache_disabled() { + return false; + } + if self.amber_git_hash != GIT_HASH { + return false; + } + + match file_meta.source { + FileSource::File => { + if let Ok(meta) = File::open(&filename) { + if let Ok(meta) = meta.metadata() { + if ! meta.is_file() { return false } + if meta.len() != self.len { return false } + + if let Ok(mtime) = meta.modified() { + if let Some(smtime) = self.mtime { + if mtime != smtime { return false } + } + } + } + } + }, + FileSource::Stdlib => { + if let Some(file) = stdlib::resolve(filename.to_str().unwrap()) { + if file.len() != self.len as usize { + return false; + } + if self.mtime.is_some() { + return false; + } + } + }, + _ => unimplemented!() + } + + true + } +} + +impl TryInto> for &PretokenizedFile { + type Error = String; + fn try_into(self) -> Result, Self::Error> { + Ok(rmp_serde::to_vec(&self).map_err(|x| x.to_string())?) + } +} + +impl TryFrom> for PretokenizedFile { + type Error = String; + fn try_from(value: Vec) -> Result { + Ok(rmp_serde::from_slice(&value).map_err(|x| x.to_string())?) + } +} diff --git a/src/compiler/file_source.rs b/src/compiler/file_source.rs index c33acf37..0f791ffb 100644 --- a/src/compiler/file_source.rs +++ b/src/compiler/file_source.rs @@ -1,12 +1,12 @@ use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize)] pub struct FileMeta { pub is_import: bool, pub source: FileSource } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize)] pub enum FileSource { File, Stdlib, diff --git a/src/tests/validity/range_loop_float.ab b/src/tests/validity/range_loop_float.ab new file mode 100644 index 00000000..1da57c92 --- /dev/null +++ b/src/tests/validity/range_loop_float.ab @@ -0,0 +1,8 @@ +// Output +// 1 +// 2 +// 3 + +loop i in 1..4.5 { + echo i +} diff --git a/src/tests/validity/range_loop_float_negative.ab b/src/tests/validity/range_loop_float_negative.ab new file mode 100644 index 00000000..34b67a8e --- /dev/null +++ b/src/tests/validity/range_loop_float_negative.ab @@ -0,0 +1,8 @@ +// Output +// -4.5 +// -3.5 +// -2.5 + +loop i in -4.5..-1 { + echo i +} diff --git a/src/tests/validity/range_loop_negative.ab b/src/tests/validity/range_loop_negative.ab new file mode 100644 index 00000000..1dc35f90 --- /dev/null +++ b/src/tests/validity/range_loop_negative.ab @@ -0,0 +1,7 @@ +// Output +// -2 +// -1 + +loop i in -2..0 { + echo i +} \ No newline at end of file diff --git a/src/tests/validity/range_loop_noop.ab b/src/tests/validity/range_loop_noop.ab new file mode 100644 index 00000000..cb30be12 --- /dev/null +++ b/src/tests/validity/range_loop_noop.ab @@ -0,0 +1,7 @@ +// Output +// -1 +// 0 + +loop i in -1..=0 { + echo i +} \ No newline at end of file From 81138ce74995c258c28cc5e0ad697554825c7fba Mon Sep 17 00:00:00 2001 From: b1ek Date: Tue, 19 Nov 2024 11:27:12 +1000 Subject: [PATCH 07/10] add option to precompile stdlib --- .gitignore | 3 ++ Cargo.lock | 65 ++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/compiler/cache/tokenize.rs | 6 +--- src/main.rs | 16 +++++++-- src/stdlib.rs | 65 +++++++++++++++++++++++++++++++++- 6 files changed, 146 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index f0d86dc6..e4857e13 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ flamegraph.svg .direnv /result +# Cache files +*.ab.pr.c +*.ab.tk.c diff --git a/Cargo.lock b/Cargo.lock index 6722ad70..eed964ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,7 @@ dependencies = [ "heraclitus-compiler", "home", "include_dir", + "indicatif", "itertools", "predicates", "rmp-serde", @@ -272,6 +273,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width 0.1.14", + "windows-sys 0.52.0", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -296,6 +310,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "errno" version = "0.3.9" @@ -335,7 +355,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heraclitus-compiler" -version = "1.8.0" +version = "1.8.1" dependencies = [ "capitalize", "colored", @@ -400,6 +420,19 @@ dependencies = [ "quote 1.0.37", ] +[[package]] +name = "indicatif" +version = "0.17.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -469,6 +502,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.20.2" @@ -481,7 +520,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -490,6 +529,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "predicates" version = "3.1.2" @@ -750,6 +795,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -826,6 +877,16 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wildmatch" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index a05a8229..72784bb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ glob = "0.3" heraclitus-compiler = { path = "../heraclitus", features = ["serde"] } home = "0.5.9" include_dir = "0.7.4" +indicatif = "0.17.9" itertools = "0.13.0" rmp-serde = "1.3.0" serde = { version = "1.0.215", features = ["derive"] } diff --git a/src/compiler/cache/tokenize.rs b/src/compiler/cache/tokenize.rs index 18eaa2d4..fbd96d46 100644 --- a/src/compiler/cache/tokenize.rs +++ b/src/compiler/cache/tokenize.rs @@ -33,13 +33,11 @@ impl PretokenizedFile { match file_meta.source { FileSource::File => { if ! filename.is_file() { - println!("not a file: {filename:?}"); return Ok(None) } }, FileSource::Stdlib => { if Self::get_stdlib(&filename).is_none() { - println!("not a stdlib: {filename:?}"); return Ok(None) } }, @@ -51,16 +49,14 @@ impl PretokenizedFile { if let Some(cache_filename) = cache_filename { if ! cache_filename.is_file() { - println!("not a cachefile: {filename:?}"); return Ok(None) } - let parsed = Self::try_from(fs::read(cache_filename)?)?; + let parsed = Self::try_from(fs::read(&cache_filename)?)?; if parsed.validate(&filename, file_meta) { Ok(Some(parsed)) } else { - println!("invalid cachefile for {filename:?}"); Ok(None) } diff --git a/src/main.rs b/src/main.rs index 8da1c96a..ed454f8a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,12 +50,20 @@ pub struct Cli { minify: bool, #[arg(long, short)] - no_cache: bool + no_cache: bool, + + /// If you want to speed up your compile time, use this to precompile stdlib + /// It needs to be done only once per update, until you remove ~/.cache/amber + /// If you pass this, all other arguments will be discarded + #[arg(long, verbatim_doc_comment)] + precompile: bool } fn main() -> Result<(), Box> { let cli = Cli::parse(); - if cli.docs { + if cli.precompile { + handle_precompile(); + } else if cli.docs { handle_docs(cli)?; } else if let Some(ref code) = cli.eval { handle_eval(code.to_string(), cli)?; @@ -212,6 +220,10 @@ fn handle_docs(cli: Cli) -> Result<(), Box> { } } +fn handle_precompile() { + stdlib::precompile_all(); +} + #[cfg(windows)] fn set_file_permission(_file: &fs::File, _output: String) { // We don't need to set permission on Windows diff --git a/src/stdlib.rs b/src/stdlib.rs index 709f9d61..a13d9af7 100644 --- a/src/stdlib.rs +++ b/src/stdlib.rs @@ -1,4 +1,9 @@ -use include_dir::{include_dir, Dir}; +use std::{path::PathBuf, sync::mpsc::{channel, Sender}}; + +use include_dir::{include_dir, Dir, File}; +use indicatif::{ProgressBar, ProgressStyle}; + +use crate::{compiler::{file_source::{FileMeta, FileSource}, AmberCompiler}, Cli}; pub const STDLIB: Dir = include_dir!("src/std"); pub fn resolve>(path: T) -> Option { @@ -11,3 +16,61 @@ pub fn resolve>(path: T) -> Option { None } } + +pub fn precompile_all() { + + let mut threads = vec![]; + let (tx, rx) = channel::(); + + fn precompile_thread(file: File, tx: Sender) { + let prefix: PathBuf = "std/".into(); + let mut path = prefix.join(file.path()); + path.set_extension(""); + let path = path.to_str().unwrap().to_string(); + + let compiler = AmberCompiler::new( + file.contents_utf8().unwrap().into(), + Some(path.clone()), + Cli { + no_cache: false, + precompile: false, + ..Cli::default() + }, + FileMeta { + is_import: true, + source: FileSource::Stdlib + } + ); + + compiler.tokenize().unwrap(); + + tx.send(path).unwrap(); + } + + for file in STDLIB.files() { + let tx = tx.clone(); + + threads.push(std::thread::spawn(move || { + let caught = std::panic::catch_unwind(|| { + precompile_thread(file.clone(), tx.clone()); + }); + if caught.is_err() { + tx.send("error".into()).unwrap() + } + })); + } + + let mut i = 0; + let progress = ProgressBar::new(threads.len() as u64) + .with_style(ProgressStyle::with_template("{progress} [{elapsed}] {bar} {wide_msg}").unwrap()); + + while let Ok(file) = rx.recv() { + progress.inc(1); + progress.set_message(file); + + i += 1; + if i == threads.len() { break } + } + + progress.finish_with_message("done"); +} From 096fb5fd915674066781f76768a0ec161e652340 Mon Sep 17 00:00:00 2001 From: b1ek Date: Mon, 25 Nov 2024 14:00:41 +1000 Subject: [PATCH 08/10] get it to actually compile --- src/compiler/file_source.rs | 25 ++++++++++++++++++++++-- src/main.rs | 32 +++++++++++++++++++++++-------- src/modules/builtin/lines.rs | 3 ++- src/modules/variable/constinit.rs | 3 ++- src/stdlib.rs | 9 ++++----- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/compiler/file_source.rs b/src/compiler/file_source.rs index d5d0b2a6..f7a76992 100644 --- a/src/compiler/file_source.rs +++ b/src/compiler/file_source.rs @@ -1,5 +1,3 @@ -use std::default; - use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize)] @@ -8,6 +6,29 @@ pub struct FileMeta { pub source: FileSource } +impl FileMeta { + pub fn file(is_import: bool) -> Self { + Self { + is_import, + source: FileSource::File + } + } + + pub fn stdlib(is_import: bool) -> Self { + Self { + is_import, + source: FileSource::Stdlib + } + } + + pub fn stream(is_import: bool) -> Self { + Self { + is_import, + source: FileSource::Stream + } + } +} + impl Default for FileMeta { fn default() -> Self { Self { diff --git a/src/main.rs b/src/main.rs index fe33d348..9c302149 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,7 +39,7 @@ struct Cli { /// To select multiple, pass multiple times with different values /// Argument also supports a wilcard match, like "*" or "s*mt" #[arg(long, verbatim_doc_comment)] - no_proc: Vec, + no_proc: Vec } #[derive(Subcommand, Clone, Debug)] @@ -62,6 +62,10 @@ enum CommandKind { struct EvalCommand { /// Code to evaluate code: String, + + /// Disable cache + #[arg(long, verbatim_doc_comment)] + no_cache: bool } #[derive(Args, Clone, Debug)] @@ -79,6 +83,10 @@ struct RunCommand { /// Argument also supports a wilcard match, like "*" or "s*mt" #[arg(long, verbatim_doc_comment)] no_proc: Vec, + + /// Disable cache + #[arg(long, verbatim_doc_comment)] + no_cache: bool } #[derive(Args, Clone, Debug)] @@ -92,6 +100,10 @@ struct CheckCommand { /// Argument also supports a wilcard match, like "*" or "s*mt" #[arg(long, verbatim_doc_comment)] no_proc: Vec, + + /// Disable cache + #[arg(long, verbatim_doc_comment)] + no_cache: bool } #[derive(Args, Clone, Debug)] @@ -113,6 +125,7 @@ struct BuildCommand { #[arg(long)] minify: bool, + /// Disable cache #[arg(long, short)] no_cache: bool, @@ -144,17 +157,17 @@ fn main() -> Result<(), Box> { handle_eval(command)?; } CommandKind::Run(command) => { - let options = CompilerOptions::from_args(&command.no_proc, false); + let options = CompilerOptions::from_args(&command.no_proc, command.no_cache, false); let (code, messages) = compile_input(command.input, options); execute_output(code, command.args, messages)?; } CommandKind::Check(command) => { - let options = CompilerOptions::from_args(&command.no_proc, false); + let options = CompilerOptions::from_args(&command.no_proc, command.no_cache, false); compile_input(command.input, options); } CommandKind::Build(command) => { let output = create_output(&command); - let options = CompilerOptions::from_args(&command.no_proc, command.minify); + let options = CompilerOptions::from_args(&command.no_proc, command.no_cache, command.minify); let (code, _) = compile_input(command.input, options); write_output(output, code); } @@ -166,7 +179,7 @@ fn main() -> Result<(), Box> { } } } else if let Some(input) = cli.input { - let options = CompilerOptions::from_args(&cli.no_proc, false); + let options = CompilerOptions::from_args(&cli.no_proc, true, false); let (code, messages) = compile_input(input, options); execute_output(code, cli.args, messages)?; } @@ -198,7 +211,8 @@ fn compile_input(input: PathBuf, options: CompilerOptions) -> (String, bool) { } }; - let compiler = AmberCompiler::new(amber_code, Some(input), options); + let meta = FileMeta::file(false); + let compiler = AmberCompiler::new(amber_code, Some(input), options, meta); let (messages, bash_code) = match compiler.compile() { Ok(result) => result, Err(err) => { @@ -237,8 +251,9 @@ fn write_output(output: PathBuf, code: String) { } fn handle_eval(command: EvalCommand) -> Result<(), Box> { + let meta = FileMeta::stream(false); let options = CompilerOptions::default(); - let compiler = AmberCompiler::new(command.code, None, options); + let compiler = AmberCompiler::new(command.code, None, options, meta); match compiler.compile() { Ok((messages, code)) => { messages.iter().for_each(|m| m.show()); @@ -263,8 +278,9 @@ fn handle_docs(command: DocsCommand) -> Result<(), Box> { } }; + let meta = FileMeta::file(true); let options = CompilerOptions::default(); - let compiler = AmberCompiler::new(code, Some(input), options); + let compiler = AmberCompiler::new(code, Some(input), options, meta); let output = command.output.unwrap_or_else(|| PathBuf::from("docs")); let output = output.to_string_lossy().to_string(); match compiler.generate_docs(output, command.usage) { diff --git a/src/modules/builtin/lines.rs b/src/modules/builtin/lines.rs index df730291..25c85344 100644 --- a/src/modules/builtin/lines.rs +++ b/src/modules/builtin/lines.rs @@ -1,11 +1,12 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::expression::expr::Expr; use crate::modules::types::{Type, Typed}; use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct LinesInvocation { path: Box>, } diff --git a/src/modules/variable/constinit.rs b/src/modules/variable/constinit.rs index ebdfc525..aec570e2 100644 --- a/src/modules/variable/constinit.rs +++ b/src/modules/variable/constinit.rs @@ -1,4 +1,5 @@ use heraclitus_compiler::prelude::*; +use serde::{Deserialize, Serialize}; use crate::docs::module::DocumentationModule; use crate::modules::types::{Typed, Type}; use crate::modules::expression::expr::Expr; @@ -6,7 +7,7 @@ use crate::translate::module::TranslateModule; use crate::utils::metadata::{ParserMetadata, TranslateMetadata}; use super::{variable_name_extensions, handle_identifier_name}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConstInit { name: String, expr: Box, diff --git a/src/stdlib.rs b/src/stdlib.rs index a13d9af7..ede4e6cc 100644 --- a/src/stdlib.rs +++ b/src/stdlib.rs @@ -3,7 +3,7 @@ use std::{path::PathBuf, sync::mpsc::{channel, Sender}}; use include_dir::{include_dir, Dir, File}; use indicatif::{ProgressBar, ProgressStyle}; -use crate::{compiler::{file_source::{FileMeta, FileSource}, AmberCompiler}, Cli}; +use crate::compiler::{file_source::{FileMeta, FileSource}, AmberCompiler, CompilerOptions}; pub const STDLIB: Dir = include_dir!("src/std"); pub fn resolve>(path: T) -> Option { @@ -18,7 +18,6 @@ pub fn resolve>(path: T) -> Option { } pub fn precompile_all() { - let mut threads = vec![]; let (tx, rx) = channel::(); @@ -31,10 +30,10 @@ pub fn precompile_all() { let compiler = AmberCompiler::new( file.contents_utf8().unwrap().into(), Some(path.clone()), - Cli { + CompilerOptions { no_cache: false, - precompile: false, - ..Cli::default() + no_proc: vec![String::from("*")], + minify: false }, FileMeta { is_import: true, From 968c9b60f5937876c335f24f928e7902216f0565 Mon Sep 17 00:00:00 2001 From: b1ek Date: Sun, 1 Dec 2024 20:58:15 +1000 Subject: [PATCH 09/10] add precompile command --- src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 9c302149..2566ef5d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,7 @@ use crate::compiler::{AmberCompiler, CompilerOptions}; use clap::{Args, CommandFactory, Parser, Subcommand}; use clap_complete::Shell; use colored::Colorize; -use compiler::file_source::{FileMeta, FileSource}; +use compiler::file_source::FileMeta; use heraclitus_compiler::prelude::*; use std::error::Error; use std::io::{prelude::*, stdin}; @@ -54,6 +54,8 @@ enum CommandKind { Build(BuildCommand), /// Generate Amber script documentation Docs(DocsCommand), + /// Preparse stdlib + Precompile, /// Generate Bash completion script Completion, } @@ -176,6 +178,9 @@ fn main() -> Result<(), Box> { } CommandKind::Completion => { handle_completion(); + }, + CommandKind::Precompile => { + handle_precompile(); } } } else if let Some(input) = cli.input { From eabfbf3767c751e0a4360b4b243121f57d73f90f Mon Sep 17 00:00:00 2001 From: b1ek Date: Sun, 1 Dec 2024 20:58:33 +1000 Subject: [PATCH 10/10] set secure cache permissions on unix systems --- src/compiler/cache.rs | 5 ++++- src/compiler/cache/parse.rs | 7 +++++-- src/compiler/cache/tokenize.rs | 8 +++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/compiler/cache.rs b/src/compiler/cache.rs index cd28b053..b34ec7ce 100644 --- a/src/compiler/cache.rs +++ b/src/compiler/cache.rs @@ -1,4 +1,4 @@ -use std::{fs, path::PathBuf}; +use std::{fs::{self, Permissions}, os::unix::fs::PermissionsExt, path::PathBuf}; const GIT_HASH: &'static str = env!("GIT_HASH"); @@ -11,6 +11,9 @@ pub fn home_cache() -> Option { home.push("amber"); if ! home.is_dir() { fs::create_dir_all(&home).expect("Couldn't create ~/.cache/amber"); + + #[cfg(unix)] + fs::set_permissions(&home, Permissions::from_mode(0o700)).expect("Couldn't set permissions to ~/.cache/amber") } Some(home) } else { diff --git a/src/compiler/cache/parse.rs b/src/compiler/cache/parse.rs index ac205824..792cea07 100644 --- a/src/compiler/cache/parse.rs +++ b/src/compiler/cache/parse.rs @@ -1,4 +1,4 @@ -use std::{error::Error, fs::{self, File, Metadata}, path::PathBuf, time::SystemTime}; +use std::{error::Error, fs::{self, File, Metadata, Permissions}, os::unix::fs::PermissionsExt, path::PathBuf, time::SystemTime}; use serde::{Serialize, Deserialize}; use crate::{modules::block::Block, utils::ParserMetadata}; @@ -69,7 +69,10 @@ impl PreparsedFile { filename.set_extension(FILE_EXT); let serialized: Vec = self.try_into()?; - fs::write(filename, serialized)?; + fs::write(&filename, serialized)?; + + #[cfg(unix)] + fs::set_permissions(&filename, Permissions::from_mode(0o700)).map_err(|x| format!("Cannot set perms to {filename:?}: {x}"))?; Ok(()) } diff --git a/src/compiler/cache/tokenize.rs b/src/compiler/cache/tokenize.rs index fbd96d46..37749ae2 100644 --- a/src/compiler/cache/tokenize.rs +++ b/src/compiler/cache/tokenize.rs @@ -1,4 +1,5 @@ -use std::fs::File; +use std::fs::{File, Permissions}; +use std::os::unix::fs::PermissionsExt; use std::{fs, path::PathBuf}; use std::time::SystemTime; @@ -109,7 +110,12 @@ impl PretokenizedFile { if let Some(cache_file) = Self::get_path(&filename, file_meta) { let serialized: Vec = self.try_into()?; + fs::write(&cache_file, serialized).map_err(|x| format!("Cannot write to {cache_file:?}: {x}"))?; + + #[cfg(unix)] + fs::set_permissions(&filename, Permissions::from_mode(0o700)).map_err(|x| format!("Cannot set perms to {cache_file:?}: {x}"))?; + Ok(()) } else { Err(String::from("Couldn't get path to saved directory").into())