diff --git a/.idea/runConfigurations/clippy-parser-scala-wasm-linux.xml b/.idea/runConfigurations/clippy-parser-scala-wasm-linux.xml deleted file mode 100644 index ee8051bb01db..000000000000 --- a/.idea/runConfigurations/clippy-parser-scala-wasm-linux.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/clippy-parser-scala-wasm-macos.xml b/.idea/runConfigurations/clippy-parser-scala-wasm-macos.xml deleted file mode 100644 index e52454623d7f..000000000000 --- a/.idea/runConfigurations/clippy-parser-scala-wasm-macos.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/clippy-parser-scala-wasm-windows.xml b/.idea/runConfigurations/clippy-parser-scala-wasm-windows.xml deleted file mode 100644 index fc596a9a55a9..000000000000 --- a/.idea/runConfigurations/clippy-parser-scala-wasm-windows.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/clippy-parser-scala-wasm.xml b/.idea/runConfigurations/clippy-parser-scala-wasm.xml deleted file mode 100644 index 92fa83f7aca3..000000000000 --- a/.idea/runConfigurations/clippy-parser-scala-wasm.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/doc-test-parser-scala-native.xml b/.idea/runConfigurations/doc-test-parser-scala-native.xml deleted file mode 100644 index eba868e36b81..000000000000 --- a/.idea/runConfigurations/doc-test-parser-scala-native.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/test-parser-scala-native.xml b/.idea/runConfigurations/test-parser-scala-native.xml deleted file mode 100644 index c26f0b996b77..000000000000 --- a/.idea/runConfigurations/test-parser-scala-native.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb931d9aff1..16478de5fccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,8 @@ open them in the IDE (which is not part of the React app, but can be switched to from the dashboard). The PR also adds authentication+authorization (i.e., sign up and sign in for users), via either email/password or GitHub/Google. +- [New Enso documentation parser][5917]. Smaller and faster; enables planned + improvements to internal documentation representation. #### EnsoGL (rendering engine) @@ -527,6 +529,7 @@ [5802]: https://github.com/enso-org/enso/pull/5802 [5850]: https://github.com/enso-org/enso/pull/5850 [5863]: https://github.com/enso-org/enso/pull/5863 +[5917]: https://github.com/enso-org/enso/pull/5917 #### Enso Compiler diff --git a/Cargo.lock b/Cargo.lock index bbf6a72b328a..0f3733615ab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1603,6 +1603,7 @@ dependencies = [ name = "debug-scene-documentation" version = "0.1.0" dependencies = [ + "enso-doc-parser", "enso-suggestion-database", "ensogl", "ensogl-hardcoded-theme", @@ -2100,6 +2101,19 @@ dependencies = [ "debug-scene-visualization", ] +[[package]] +name = "enso-doc-parser" +version = "0.1.0" +dependencies = [ + "enso-metamodel", + "enso-metamodel-lexpr", + "enso-parser", + "enso-prelude", + "enso-profiler", + "enso-reflect", + "lexpr", +] + [[package]] name = "enso-executor" version = "0.1.0" @@ -2162,6 +2176,7 @@ dependencies = [ "enso-data-structures", "enso-debug-api", "enso-debug-scene", + "enso-doc-parser", "enso-executor", "enso-frp", "enso-notification", @@ -2189,7 +2204,6 @@ dependencies = [ "mockall", "nalgebra", "parser", - "parser-scala", "regex", "semver 1.0.16", "serde", @@ -2520,6 +2534,7 @@ dependencies = [ "double-representation", "engine-protocol", "enso-data-structures", + "enso-doc-parser", "enso-executor", "enso-notification", "enso-prelude", @@ -2529,7 +2544,6 @@ dependencies = [ "flo_stream", "futures 0.3.26", "parser", - "parser-scala", "span-tree", "wasm-bindgen-test", ] @@ -4238,10 +4252,16 @@ dependencies = [ name = "ide-view-component-browser" version = "0.1.0" dependencies = [ + "enso-frp", "enso-prelude", + "ensogl", + "ensogl-gui-component", + "ensogl-hardcoded-theme", "ensogl-text", "ide-view-component-list-panel", "ide-view-component-list-panel-breadcrumbs", + "ide-view-documentation", + "ide-view-graph-editor", ] [[package]] @@ -4308,8 +4328,10 @@ name = "ide-view-documentation" version = "0.1.0" dependencies = [ "double-representation", + "enso-doc-parser", "enso-frp", "enso-prelude", + "enso-profiler", "enso-suggestion-database", "ensogl", "ensogl-component", @@ -5362,29 +5384,6 @@ dependencies = [ "uuid 0.8.2", ] -[[package]] -name = "parser-scala" -version = "0.1.0" -dependencies = [ - "ast", - "bytes 1.1.0", - "console_error_panic_hook", - "enso-prelude", - "enso-profiler", - "failure", - "futures 0.3.26", - "ide-ci", - "js-sys", - "matches", - "reqwest", - "serde", - "serde_json", - "tokio", - "wasm-bindgen", - "wasm-bindgen-test", - "websocket", -] - [[package]] name = "paste" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index 9d41e1d49a19..95d7563700a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "build/deprecated/rust-scripts", "build/shader-tools", "lib/rust/*", + "lib/rust/parser/doc-parser", "lib/rust/parser/src/syntax/tree/visitor", "lib/rust/parser/jni", "lib/rust/parser/generate-java", diff --git a/app/gui/Cargo.toml b/app/gui/Cargo.toml index be07ab4f1708..a6aa3080e08d 100644 --- a/app/gui/Cargo.toml +++ b/app/gui/Cargo.toml @@ -16,6 +16,7 @@ enso-data-structures = { path = "../../lib/rust/data-structures" } enso-debug-api = { path = "../../lib/rust/debug-api" } enso-debug-scene = { path = "view/examples" } enso-frp = { path = "../../lib/rust/frp" } +enso-doc-parser = { path = "../../lib/rust/parser/doc-parser" } enso-prelude = { path = "../../lib/rust/prelude" } enso-profiler = { path = "../../lib/rust/profiler" } enso-executor = { path = "../../lib/rust/executor" } @@ -34,7 +35,6 @@ ensogl-drop-manager = { path = "../../lib/rust/ensogl/component/drop-manager" } fuzzly = { path = "../../lib/rust/fuzzly" } ast = { path = "language/ast/impl" } parser = { path = "language/parser" } -parser-scala = { path = "language/parser-scala" } ide-view = { path = "view" } engine-protocol = { path = "controller/engine-protocol" } json-rpc = { path = "../../lib/rust/json-rpc" } diff --git a/app/gui/controller/engine-protocol/src/language_server/types.rs b/app/gui/controller/engine-protocol/src/language_server/types.rs index 3bd1dab1084a..5a580451cdd9 100644 --- a/app/gui/controller/engine-protocol/src/language_server/types.rs +++ b/app/gui/controller/engine-protocol/src/language_server/types.rs @@ -847,62 +847,6 @@ pub enum RegisterOptions { -// =================== -// === Doc Section === -// =================== - -/// Text rendered as HTML (may contain HTML tags). -pub type HtmlString = String; - -/// Documentation section mark. -#[derive(Hash, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[allow(missing_docs)] -pub enum Mark { - Important, - Info, - Example, -} - -/// A single section of the documentation. -#[derive(Hash, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[allow(missing_docs)] -#[serde(tag = "type")] -#[serde(rename_all = "camelCase")] -pub enum DocSection { - /// The documentation tag. - #[serde(rename_all = "camelCase")] - Tag { - /// The tag name. - name: String, - /// The tag text. - body: HtmlString, - }, - /// The paragraph of the text. - #[serde(rename_all = "camelCase")] - Paragraph { - /// The elements that make up this paragraph. - body: HtmlString, - }, - /// The section that starts with the key followed by the colon and the body. - #[serde(rename_all = "camelCase")] - Keyed { - /// The section key. - key: String, - /// The elements that make up the body of the section. - body: HtmlString, - }, - /// The section that starts with the mark followed by the header and the body. - #[serde(rename_all = "camelCase")] - Marked { - /// The section mark. - mark: Mark, - /// The section header. - header: Option, - /// The elements that make up the body of the section. - body: HtmlString, - }, -} - // =========================== // === Suggestion Database === // =========================== @@ -988,79 +932,57 @@ pub enum SuggestionEntryType { pub enum SuggestionEntry { #[serde(rename_all = "camelCase")] Type { - external_id: Option, - name: String, - module: String, - params: Vec, - parent_type: Option, - reexport: Option, - documentation: Option, - documentation_html: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, + external_id: Option, + name: String, + module: String, + params: Vec, + parent_type: Option, + reexport: Option, + documentation: Option, }, #[serde(rename_all = "camelCase")] Constructor { - external_id: Option, - name: String, - module: String, - arguments: Vec, - return_type: String, - reexport: Option, - documentation: Option, - documentation_html: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, + external_id: Option, + name: String, + module: String, + arguments: Vec, + return_type: String, + reexport: Option, + documentation: Option, }, #[serde(rename_all = "camelCase")] Method { - external_id: Option, - name: String, - module: String, - arguments: Vec, - self_type: String, - return_type: String, - is_static: bool, - reexport: Option, - documentation: Option, - documentation_html: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, + external_id: Option, + name: String, + module: String, + arguments: Vec, + self_type: String, + return_type: String, + is_static: bool, + reexport: Option, + documentation: Option, }, #[serde(rename_all = "camelCase")] Function { - external_id: Option, - name: String, - module: String, - arguments: Vec, - return_type: String, - scope: SuggestionEntryScope, - documentation: Option, - documentation_html: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, + external_id: Option, + name: String, + module: String, + arguments: Vec, + return_type: String, + scope: SuggestionEntryScope, + documentation: Option, }, #[serde(rename_all = "camelCase")] Local { - external_id: Option, - name: String, - module: String, - return_type: String, - scope: SuggestionEntryScope, - documentation: Option, - documentation_html: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, + external_id: Option, + name: String, + module: String, + return_type: String, + scope: SuggestionEntryScope, + documentation: Option, }, #[serde(rename_all = "camelCase")] - Module { - module: String, - documentation: Option, - documentation_html: Option, - reexport: Option, - #[serde(default, deserialize_with = "enso_prelude::deserialize_null_as_default")] - documentation_sections: Vec, - }, + Module { module: String, documentation: Option, reexport: Option }, } impl SuggestionEntry { @@ -1116,7 +1038,7 @@ impl FieldUpdate { } /// Maps the field update by applying `f` on the underlying value. - pub fn map(self, f: impl Fn(T) -> U) -> FieldUpdate { + pub fn map(self, f: impl FnOnce(T) -> U) -> FieldUpdate { FieldUpdate { tag: self.tag, value: self.value.map(f) } } @@ -1124,7 +1046,7 @@ impl FieldUpdate { /// Otherwise returns the error returned by `f`. pub fn try_map( self, - f: impl Fn(T) -> std::result::Result, + f: impl FnOnce(T) -> std::result::Result, ) -> std::result::Result, E> { Ok(FieldUpdate { tag: self.tag, value: self.value.map(f).transpose()? }) } @@ -1182,14 +1104,13 @@ pub enum SuggestionsDatabaseUpdate { #[serde(rename_all = "camelCase")] pub struct SuggestionsDatabaseModification { #[serde(default)] - pub arguments: Vec, - pub module: Option>, - pub self_type: Option>, - pub return_type: Option>, - pub documentation: Option>, - pub documentation_sections: Option>>, - pub scope: Option>, - pub reexport: Option>, + pub arguments: Vec, + pub module: Option>, + pub self_type: Option>, + pub return_type: Option>, + pub documentation: Option>, + pub scope: Option>, + pub reexport: Option>, } /// Notification about change in the suggestions database. diff --git a/app/gui/language/parser-scala/.gitignore b/app/gui/language/parser-scala/.gitignore deleted file mode 100644 index 9110e4072ac4..000000000000 --- a/app/gui/language/parser-scala/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Downloaded parser artifacts -pkg diff --git a/app/gui/language/parser-scala/Cargo.toml b/app/gui/language/parser-scala/Cargo.toml deleted file mode 100644 index e93083c63d55..000000000000 --- a/app/gui/language/parser-scala/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "parser-scala" -version = "0.1.0" -authors = ["Enso Team "] -edition = "2021" -build = "build.rs" - -[lib] -crate-type = ["cdylib", "rlib"] - -[dependencies] -ast = { path = "../ast/impl" } -enso-prelude = { path = "../../../../lib/rust/prelude" } -enso-profiler = { path = "../../../../lib/rust/profiler" } -console_error_panic_hook = { workspace = true } -failure = { workspace = true } -js-sys = { workspace = true } -matches = { workspace = true } -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0", features = ["unbounded_depth"] } -wasm-bindgen = { workspace = true } - -[dev-dependencies] -wasm-bindgen-test = { workspace = true } - -[build-dependencies] -ide-ci = { path = "../../../../build/ci_utils" } -bytes = { workspace = true } -futures = { workspace = true } -reqwest = { workspace = true } -tokio = { workspace = true } - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -websocket = "0.26.5" diff --git a/app/gui/language/parser-scala/build.rs b/app/gui/language/parser-scala/build.rs deleted file mode 100644 index 925f2b248fdd..000000000000 --- a/app/gui/language/parser-scala/build.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! This build script is responsible for ensuring that if parser targets wasm, -//! the JS Parser package is available at the expected location for -//! `wasm_bindgen` tool. - -// === Features === -#![feature(option_result_contains)] - -use ide_ci::prelude::*; - - - -// ========================= -// == Hardcoded constants == -// ========================= - -/// Where the crate expects to find file with compiled parser. -/// Path relative to the crate directory. -const PARSER_PATH: &str = "./pkg/scala-parser.js"; - -/// Commit from `enso` repository that will be used to obtain parser from. -const PARSER_COMMIT: &str = "649fe33ccf148d47deb6ba6a06f3babc48078e3e"; - -/// Magic code that needs to be prepended to ScalaJS generated parser due to: -/// https://github.com/scala-js/scala-js/issues/3677/ -const PARSER_PREAMBLE: &str = "var __ScalaJSEnv = { global: window };"; - -/// Obtains a URL where this parser version can be downloaded. -pub fn parser_url(version: &ParserVersion) -> reqwest::Url { - let url_string = format!( - "https://packages.luna-lang.org/parser-js/nightly/{}/scala-parser.js", - version.commit - ); - let invalid_url_msg = format!("{url_string} is an invalid URL."); - reqwest::Url::parse(&url_string).expect(&invalid_url_msg) -} - - - -// =================== -// == ParserVersion == -// =================== - -/// Parser version described as commit hash from `enso` repository. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ParserVersion { - pub commit: String, -} - -impl ParserVersion { - /// Create a version described by given commit hash. - pub fn from_commit(commit: String) -> ParserVersion { - ParserVersion { commit } - } - - /// The JS parser version required for this crate. - pub fn required() -> ParserVersion { - ParserVersion { commit: PARSER_COMMIT.into() } - } -} - - - -// ======================== -// == Downloading parser == -// ======================== - -/// Stores information which parser version should be provided where. -/// -/// Implementation provides methods that download desired parser version, patch it and store to the -/// file, so parser can be consumed by `wasm_bindgen`. -struct ParserProvider { - /// Required parser version. - version: ParserVersion, - /// The path where JS file needs to be provided. - parser_path: PathBuf, -} - -impl ParserProvider { - /// Creates a provider that obtains given parser version to a given path. - pub fn new(version: ParserVersion, parser_path: impl AsRef) -> ParserProvider { - let parser_path = PathBuf::from(parser_path.as_ref()); - ParserProvider { version, parser_path } - } - - /// Downloads contents of JS parser into memory. - pub async fn download(&self) -> Result { - let url = parser_url(&self.version); - ide_ci::io::download_all(url.clone()).await.context("Failed to download the parser.") - } - - /// Stores JS parser into file, after patching with a `PARSER_PREAMBLE`. - pub async fn patch_and_store(&self, js_parser: bytes::Bytes) -> Result { - ide_ci::fs::tokio::write_iter(&self.parser_path, [ - PARSER_PREAMBLE.as_bytes(), - js_parser.as_ref(), - ]) - .await - } - - /// Places required parser version in the target location. - pub async fn run(&self) -> Result { - let fingerprint = self.parser_path.with_file_name("parser.fingerprint"); - let opt_version = ide_ci::fs::tokio::read_to_string(&fingerprint).await; - let changed = match opt_version { - Err(_) => true, - Ok(hash) => hash != PARSER_COMMIT, - }; - if changed { - let parser_js = self.download().await?; - self.patch_and_store(parser_js).await?; - ide_ci::fs::tokio::write(&fingerprint, PARSER_COMMIT).await?; - } - Ok(()) - } -} - - - -// ========== -// == main == -// ========== - -#[tokio::main] -async fn main() -> Result { - if ide_ci::programs::cargo::build_env::targeting_wasm() { - let required_version = ParserVersion::required(); - let parser_path = Path::new(PARSER_PATH).absolutize()?; - let provider = ParserProvider::new(required_version, &parser_path); - provider.run().await?; - } - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed={PARSER_PATH}"); - - Ok(()) -} diff --git a/app/gui/language/parser-scala/src/api.rs b/app/gui/language/parser-scala/src/api.rs deleted file mode 100644 index f057d7a1dfce..000000000000 --- a/app/gui/language/parser-scala/src/api.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! A module containing structures and traits used in parser API. - -use crate::prelude::*; - - - -// =========== -// == Error == -// =========== - -/// A result of parsing code. -pub type Result = std::result::Result; - -/// An error which may be result of parsing code. -#[derive(Debug, Fail)] -pub enum Error { - /// Error due to inner workings of the parser. - #[fail(display = "Internal parser error: {:?}.", _0)] - ParsingError(String), - /// Error related to wrapping = communication with the parser service. - #[fail(display = "Interop error: {}.", _0)] - InteropError(#[cause] Box), -} - -/// Wraps an arbitrary `std::error::Error` as an `InteropError.` -pub fn interop_error(error: T) -> Error -where T: Fail { - Error::InteropError(Box::new(error)) -} diff --git a/app/gui/language/parser-scala/src/jsclient.rs b/app/gui/language/parser-scala/src/jsclient.rs deleted file mode 100644 index f93255497384..000000000000 --- a/app/gui/language/parser-scala/src/jsclient.rs +++ /dev/null @@ -1,66 +0,0 @@ -#![cfg(target_arch = "wasm32")] - -use crate::prelude::*; -use wasm_bindgen::prelude::*; - -use crate::api; - - - -pub type Result = std::result::Result; - -#[derive(Debug, Fail)] -pub enum Error { - #[fail(display = "JSON (de)serialization failed: {:?}", _0)] - JsonSerializationError(#[cause] serde_json::error::Error), - - #[fail(display = "Scala parser failed: {:?}.", _0)] - ScalaException(String), -} - -impl From for api::Error { - fn from(e: Error) -> Self { - api::interop_error(e) - } -} - -impl From for Error { - fn from(error: serde_json::error::Error) -> Self { - Error::JsonSerializationError(error) - } -} - -impl From for Error { - fn from(jsvalue: JsValue) -> Self { - Error::ScalaException(format!("{jsvalue:?}")) - } -} - -#[wasm_bindgen(module = "/pkg/scala-parser.js")] -extern "C" { - #[wasm_bindgen(catch)] - fn doc_parser_generate_html_from_doc(content: String) -> std::result::Result; -} - -/// Wrapper over the JS-compiled parser. -/// -/// Can only be used when targeting WebAssembly. -#[derive(Debug, Clone, Copy)] -pub struct Client {} - -impl Client { - /// Creates a `Client`. - pub fn new() -> Result { - Ok(Client {}) - } - - /// Calls JS doc parser to generate HTML from pure doc code without Enso's AST. - #[profile(Detail)] - pub fn generate_html_doc_pure(&self, code: String) -> api::Result { - let html_code = || { - let html_code = doc_parser_generate_html_from_doc(code)?; - Result::Ok(html_code) - }; - Ok(html_code()?) - } -} diff --git a/app/gui/language/parser-scala/src/lib.rs b/app/gui/language/parser-scala/src/lib.rs deleted file mode 100644 index 94b090111d67..000000000000 --- a/app/gui/language/parser-scala/src/lib.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Crate wrapping parser API in nice-to-use Rust code. -//! -//! The Parser is a library written in scala. There are two implementations of Rust wrappers to -//! this parser: one for local parser which binds scala parser compiled to WebAssembly to the Rust -//! crate. The second is calling a Parser running remotely using WebSockets. - -// === Features === -#![feature(trait_alias)] -// === Standard Linter Configuration === -#![deny(non_ascii_idents)] -#![warn(unsafe_code)] -#![allow(clippy::bool_to_int_with_if)] -#![allow(clippy::let_and_return)] -// === Non-Standard Linter Configuration === -#![warn(missing_docs)] -#![warn(trivial_casts)] -#![warn(trivial_numeric_casts)] -#![warn(unused_import_braces)] -#![warn(unused_qualifications)] -#![warn(missing_copy_implementations)] -#![warn(missing_debug_implementations)] - - -// ============== -// === Export === -// ============== - -pub mod api; - - - -mod jsclient; -mod wsclient; - -use crate::prelude::*; - -use std::panic; - - - -#[allow(missing_docs)] -pub mod prelude { - pub use ast::traits::*; - pub use enso_prelude::*; - pub use enso_profiler as profiler; - pub use enso_profiler::prelude::*; -} - - - -// ========================================== -// === Documentation Parser and Generator === -// ========================================== - -/// Handle to a doc parser implementation. -/// -/// Currently this component is implemented as a wrapper over documentation -/// parser written in Scala. Depending on compilation target (native or wasm) -/// it uses either implementation provided by `wsclient` or `jsclient`. -#[derive(Clone, CloneRef, Debug, Deref, DerefMut)] -pub struct DocParser(pub Rc>); - -impl DocParser { - /// Obtains a default doc parser implementation. - #[cfg(not(target_arch = "wasm32"))] - pub fn new() -> api::Result { - let client = wsclient::Client::new()?; - let doc_parser = Rc::new(RefCell::new(client)); - Ok(DocParser(doc_parser)) - } - - /// Obtains a default doc parser implementation. - #[cfg(target_arch = "wasm32")] - pub fn new() -> api::Result { - let client = jsclient::Client::new()?; - let doc_parser = Rc::new(RefCell::new(client)); - Ok(DocParser(doc_parser)) - } - - /// Obtains a default doc parser implementation, panicking in case of failure. - pub fn new_or_panic() -> DocParser { - DocParser::new().unwrap_or_else(|e| panic!("Failed to create doc parser: {e:?}")) - } - - /// Parses pure documentation code and generates HTML code. - /// Will return empty string for empty entry. - pub fn generate_html_doc_pure(&self, code: String) -> api::Result { - self.borrow_mut().generate_html_doc_pure(code) - } -} - - -// === Support === - -/// Websocket parser client. -/// Used as an interface for our (scala) parser. -#[cfg(not(target_arch = "wasm32"))] -type Client = wsclient::Client; -/// Javascript parser client. -/// Used as an interface for our (scala) parser. -#[cfg(target_arch = "wasm32")] -type Client = jsclient::Client; diff --git a/app/gui/language/parser-scala/src/test_utils.rs b/app/gui/language/parser-scala/src/test_utils.rs deleted file mode 100644 index 126f7a4759c7..000000000000 --- a/app/gui/language/parser-scala/src/test_utils.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! Utilities for writing tests using parser. Should not be used in production parts. - -use crate::prelude::*; -use enso_text::unit::*; - -use crate::Parser; - -use ast::test_utils::expect_shape; -use ast::test_utils::expect_single_line; -use ast::test_utils::validate_spans; -use ast::Ast; -use ast::HasRepr; -use ast::Shape; - - - -/// Additional methods for parser to ease writing tests. -pub trait ParserTestExts { - /// Program is expected to be a module with a single non-emty line. Its AST - /// is reinterpret as given `Shape`. - fn parse_shape(&self, program: impl Str) -> T - where - for<'t> &'t Shape: TryInto<&'t T>, - T: Clone + 'static; - - /// Runs parser on given input, panics on any error. - fn parse_testing(&self, program: impl Str) -> Ast; -} - -impl ParserTestExts for Parser { - fn parse_shape(&self, program: impl Str) -> T - where - for<'t> &'t Shape: TryInto<&'t T>, - T: Clone + 'static, { - let ast = self.parse_testing(program); - let line = expect_single_line(&ast); - let shape = expect_shape(line); - shape.clone() - } - - fn parse_testing(&self, program: impl Str) -> Ast { - let program = program.into(); - debug!("parsing {}", program); - let ast = self.parse(program.clone(), default()).unwrap(); - assert_eq!(ast.shape().len(), program.len().bytes()); - validate_spans(&ast); - assert_eq!(ast.repr(), program, "{ast:?}"); - ast - } -} diff --git a/app/gui/language/parser-scala/src/wsclient.rs b/app/gui/language/parser-scala/src/wsclient.rs deleted file mode 100644 index 03fb536e010d..000000000000 --- a/app/gui/language/parser-scala/src/wsclient.rs +++ /dev/null @@ -1,229 +0,0 @@ -#![cfg(not(target_arch = "wasm32"))] - -use crate::api::Error::*; -use crate::prelude::*; - -use crate::api; - -use std::fmt::Formatter; -use websocket::stream::sync::TcpStream; -use websocket::ClientBuilder; -use websocket::Message; - - - -type WsTcpClient = websocket::sync::Client; - - - -// ========================== -// == Constants & literals == -// ========================== - -pub const LOCALHOST: &str = "localhost"; -pub const DEFAULT_PORT: i32 = 30615; -pub const DEFAULT_HOSTNAME: &str = LOCALHOST; - -pub const HOSTNAME_VAR: &str = "ENSO_PARSER_HOSTNAME"; -pub const PORT_VAR: &str = "ENSO_PARSER_PORT"; - - - -// =========== -// == Error == -// =========== - -pub type Result = std::result::Result; - -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Fail)] -pub enum Error { - #[fail(display = "Failed to parse given address url: {}", _0)] - WrongUrl(#[cause] websocket::client::ParseError), - - #[fail(display = "Connection error: {}", _0)] - ConnectivityError(#[cause] websocket::WebSocketError), - - #[fail(display = "Peer has closed the connection")] - PeerClosedConnection, - - #[fail(display = "Received non-text response: {:?}", _0)] - NonTextResponse(websocket::OwnedMessage), - - #[fail(display = "JSON (de)serialization failed: {:?}", _0)] - JsonSerializationError(#[cause] serde_json::error::Error), - - #[fail(display = "JSON deserialization failed: {:?}, JSON was: {}", _0, _1)] - JsonDeserializationError(#[cause] serde_json::error::Error, String), -} - -impl From for api::Error { - fn from(e: Error) -> Self { - api::interop_error(e) - } -} -impl From for Error { - fn from(error: websocket::client::ParseError) -> Self { - Error::WrongUrl(error) - } -} -impl From for Error { - fn from(error: websocket::WebSocketError) -> Self { - Error::ConnectivityError(error) - } -} -impl From for Error { - fn from(error: serde_json::error::Error) -> Self { - Error::JsonSerializationError(error) - } -} - - - -// ============== -// == Protocol == -// ============== - -/// All request supported by the Parser Service. -#[allow(clippy::enum_variant_names)] -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub enum Request { - DocParserGenerateHtmlFromDoc { code: String }, -} - -/// All responses that Doc Parser Service might reply with. -#[derive(Debug, serde::Deserialize)] -pub enum ResponseDoc { - SuccessDoc { code: String }, - Error { message: String }, -} - - - -// ============ -// == Config == -// ============ - -/// Describes a WS endpoint. -#[derive(Debug, Clone)] -pub struct Config { - pub host: String, - pub port: i32, -} - -impl Config { - /// Formats URL String describing a WS endpoint. - pub fn address_string(&self) -> String { - format!("ws://{}:{}", self.host, self.port) - } - - /// Obtains a default WS endpoint to use to connect to parser service - /// using environment variables or, if they are not set, hardcoded - /// defaults. - pub fn from_env() -> Config { - let host = env::env_var_or(HOSTNAME_VAR, DEFAULT_HOSTNAME); - let port = env::parse_var_or(PORT_VAR, DEFAULT_PORT); - Config { host, port } - } -} - - - -// ============ -// == Client == -// ============ - -/// Client to the Parser Service written in Scala. -/// -/// Connects through WebSocket to the running service. -pub struct Client { - connection: WsTcpClient, -} - -mod internal { - use super::*; - impl Client { - /// Serializes `Request` to JSON and sends to peer as a text message. - pub fn send_request(&mut self, request: Request) -> Result<()> { - let request_txt = serde_json::to_string(&request)?; - let message = Message::text(request_txt); - self.connection.send_message(&message)?; - Ok(()) - } - - /// Obtains a text message from peer and deserializes it using JSON - /// into a `ResponseDoc`. - /// - /// Should be called exactly once after each `send_request` invocation. - pub fn recv_response_doc(&mut self) -> Result { - let response = self.connection.recv_message()?; - match response { - websocket::OwnedMessage::Text(code) => Ok(serde_json::from_str(&code)?), - _ => Err(Error::NonTextResponse(response)), - } - } - - /// Sends given `Request` to peer and receives a `ResponseDoc`. - /// - /// Both request and response are exchanged in JSON using text messages - /// over WebSocket. - pub fn rpc_call_doc(&mut self, request: Request) -> Result { - self.send_request(request)?; - self.recv_response_doc() - } - } -} - -impl Client { - /// Creates a new `Client` connected to the already running parser service. - pub fn from_conf(config: &Config) -> Result { - let address = config.address_string(); - let mut builder = ClientBuilder::new(&address)?; - let connection = builder.connect_insecure()?; - Ok(Client { connection }) - } - - /// Creates a `Client` using configuration defined by environment or - /// defaults if environment is not set. - pub fn new() -> Result { - // This parser is used only for native debugging, it is not used in production. - // As such, we can use debug macros here. - let config = Config::from_env(); - debug!("Connecting to {}", config.address_string()); - let client = Client::from_conf(&config)?; - debug!("Established connection with {}", config.address_string()); - Ok(client) - } - - /// Sends a request to parser service to generate HTML code from pure documentation code. - #[profile(Detail)] - pub fn generate_html_doc_pure(&mut self, code: String) -> api::Result { - let request = Request::DocParserGenerateHtmlFromDoc { code }; - let response_doc = self.rpc_call_doc(request)?; - match response_doc { - ResponseDoc::SuccessDoc { code } => Ok(code), - ResponseDoc::Error { message } => Err(ParsingError(message)), - } - } -} - -impl Debug for Client { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "") - } -} - - - -// =========== -// == tests == -// =========== - -#[test] -fn wrong_url_reported() { - let invalid_hostname = String::from("bgjhkb 7"); - let wrong_config = Config { host: invalid_hostname, port: 8080 }; - let client = Client::from_conf(&wrong_config); - let got_wrong_url_error = matches::matches!(client, Err(Error::WrongUrl(_))); - assert!(got_wrong_url_error, "expected WrongUrl error"); -} diff --git a/app/gui/src/controller/searcher.rs b/app/gui/src/controller/searcher.rs index ad51bdd2a2c6..bf011c48dbef 100644 --- a/app/gui/src/controller/searcher.rs +++ b/app/gui/src/controller/searcher.rs @@ -1424,18 +1424,17 @@ impl Searcher { let self_type = module.clone(); for method in &["data", "root"] { let entry = model::suggestion_database::Entry { - name: (*method).to_owned(), - kind: model::suggestion_database::entry::Kind::Method, - defined_in: module.clone(), - arguments: vec![], - return_type: "Standard.Base.System.File.File".try_into().unwrap(), - documentation: vec![], - documentation_html: None, - self_type: Some(self_type.clone()), - is_static: true, - scope: model::suggestion_database::entry::Scope::Everywhere, - icon_name: None, - reexported_in: None, + name: (*method).to_owned(), + kind: model::suggestion_database::entry::Kind::Method, + defined_in: module.clone(), + arguments: vec![], + return_type: "Standard.Base.System.File.File".try_into().unwrap(), + documentation: vec![], + self_type: Some(self_type.clone()), + is_static: true, + scope: model::suggestion_database::entry::Scope::Everywhere, + icon_name: None, + reexported_in: None, }; let action = Action::Suggestion(action::Suggestion::FromDatabase(Rc::new(entry))); libraries_cat_builder.add_action(action); diff --git a/app/gui/src/controller/searcher/action.rs b/app/gui/src/controller/searcher/action.rs index d052681b196b..22c1a689a937 100644 --- a/app/gui/src/controller/searcher/action.rs +++ b/app/gui/src/controller/searcher/action.rs @@ -60,11 +60,10 @@ impl Suggestion { /// Return the documentation assigned to the suggestion. pub fn documentation_html(&self) -> Option<&str> { - let doc_html = match self { - Suggestion::FromDatabase(s) => &s.documentation_html, - Suggestion::Hardcoded(s) => &s.documentation_html, - }; - doc_html.as_ref().map(AsRef::::as_ref) + // This module is mostly obsolete and used as a test API (#5661). This functionality has not + // been ported to the new documentation parser, but is not needed for anything the old + // searcher is still used for. + None } /// The Id of the method called by a suggestion, or [`None`] if the suggestion is not a method diff --git a/app/gui/src/controller/searcher/component.rs b/app/gui/src/controller/searcher/component.rs index 1c32d6f59ed6..2c22986e29ff 100644 --- a/app/gui/src/controller/searcher/component.rs +++ b/app/gui/src/controller/searcher/component.rs @@ -10,7 +10,8 @@ use controller::searcher::action::MatchKind; use convert_case::Case; use convert_case::Casing; use double_representation::name::QualifiedName; -use engine_protocol::language_server::DocSection; +use enso_doc_parser::DocSection; + // ============== @@ -223,7 +224,7 @@ impl Component { match &self.data { Data::FromDatabase { entry, .. } => entry.documentation.iter().any(|doc| match doc { DocSection::Tag { name, .. } => - name == ast::constants::PRIVATE_DOC_SECTION_TAG_NAME, + name == &ast::constants::PRIVATE_DOC_SECTION_TAG_NAME, _ => false, }), _ => false, @@ -237,7 +238,7 @@ impl Component { Data::FromDatabase { entry, .. } => { let aliases = entry.documentation.iter().filter_map(|doc| match doc { DocSection::Tag { name, body } - if name == ast::constants::ALIAS_DOC_SECTION_TAG_NAME => + if name == &ast::constants::ALIAS_DOC_SECTION_TAG_NAME => Some(body.as_str().split(',').map(|s| s.trim())), _ => None, }); diff --git a/app/gui/src/controller/searcher/component/hardcoded.rs b/app/gui/src/controller/searcher/component/hardcoded.rs index 435b565b6039..ff94aa03db0b 100644 --- a/app/gui/src/controller/searcher/component/hardcoded.rs +++ b/app/gui/src/controller/searcher/component/hardcoded.rs @@ -8,6 +8,7 @@ use crate::prelude::*; use double_representation::name::QualifiedName; +use enso_doc_parser::DocSection; use ide_view::component_browser::component_list_panel::grid::entry::icon::Id as IconId; @@ -72,18 +73,18 @@ pub fn input_snippets_with_matching_return_type( #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Snippet { /// The name displayed in the [Component Browser](crate::controller::searcher). - pub name: &'static str, + pub name: &'static str, /// The code inserted when picking the snippet. - pub code: &'static str, + pub code: &'static str, /// A list of types that the return value of this snippet's code typechecks as. Used by the /// [Component Browser](crate::controller::searcher) to decide whether to display the /// snippet when filtering components by return type. - pub return_types: Vec, + pub return_types: Vec, /// The documentation bound to the snippet. - pub documentation_html: Option, + pub documentation: Option>, /// The ID of the icon bound to this snippet's entry in the [Component /// Browser](crate::controller::searcher). - pub icon: IconId, + pub icon: IconId, } impl Snippet { @@ -101,15 +102,10 @@ impl Snippet { self } - /// Returns a modified suggestion with [`Snippet::documentation_html`] field set. This method - /// is only intended to be used when defining hardcoded suggestions and panics if a - /// documentation parser cannot be created or the argument fails to parse as valid - /// documentation. + /// Returns a modified suggestion with the [`Snippet::documentation`] field set. This method + /// is only intended to be used when defining hardcoded suggestions. fn with_documentation(mut self, documentation: &str) -> Self { - let doc_parser = parser_scala::DocParser::new().unwrap(); - let doc_string = documentation.to_string(); - let documentation_html = doc_parser.generate_html_doc_pure(doc_string); - self.documentation_html = Some(documentation_html.unwrap()); + self.documentation = Some(enso_doc_parser::parse(documentation)); self } } diff --git a/app/gui/src/presenter/searcher.rs b/app/gui/src/presenter/searcher.rs index 1d5bf1c20d04..d837ca99b60a 100644 --- a/app/gui/src/presenter/searcher.rs +++ b/app/gui/src/presenter/searcher.rs @@ -8,7 +8,6 @@ use crate::controller::searcher::action::Suggestion; use crate::controller::searcher::component; use crate::controller::searcher::Mode; use crate::controller::searcher::Notification; -use crate::controller::searcher::UserAction; use crate::executor::global::spawn_stream_handler; use crate::model::module::NodeMetadata; use crate::model::suggestion_database::entry::Kind; @@ -27,7 +26,6 @@ use ide_view::component_browser::component_list_panel::SECTION_NAME_CRUMB_INDEX; use ide_view::graph_editor::component::node as node_view; use ide_view::graph_editor::GraphEditor; use ide_view::project::SearcherParams; -use ide_view::project::SearcherVariant; // ============== @@ -104,31 +102,6 @@ impl Model { } } - fn entry_used_as_suggestion( - &self, - entry_id: view::searcher::entry::Id, - ) -> Option<(ViewNodeId, node_view::Expression)> { - match self.controller.use_as_suggestion(entry_id) { - Ok(new_code) => { - let new_code_and_trees = node_view::Expression::new_plain(new_code); - self.update_breadcrumbs(); - Some((self.input_view, new_code_and_trees)) - } - Err(err) => { - error!("Error while applying suggestion: {err}."); - None - } - } - } - - /// Should be called if an entry is selected but not used yet. Only used for the old searcher - /// API. - fn entry_selected_as_suggestion(&self, entry_id: view::searcher::entry::Id) { - if let Err(error) = self.controller.preview_entry_as_suggestion(entry_id) { - warn!("Failed to preview entry {entry_id:?} because of error: {error:?}."); - } - } - fn commit_editing(&self, entry_id: Option) -> Option { let result = match entry_id { Some(id) => self.controller.execute_action_by_index(id), @@ -140,10 +113,6 @@ impl Model { }) } - fn create_providers(&self) -> provider::Any { - provider::create_providers_from_controller(&self.controller) - } - fn suggestion_for_entry_id( &self, id: component_grid::GroupEntryId, @@ -205,26 +174,23 @@ impl Model { fn update_breadcrumbs(&self) { let names = self.controller.breadcrumbs().into_iter(); - if let SearcherVariant::ComponentBrowser(browser) = self.view.searcher() { - // We only update the breadcrumbs starting from the second element because the first - // one is reserved as a section name. - let from = 1; - let breadcrumbs_from = (names.map(Into::into).collect(), from); - browser.model().list.model().breadcrumbs.set_entries_from(breadcrumbs_from); - } + let browser = self.view.searcher(); + // We only update the breadcrumbs starting from the second element because the first + // one is reserved as a section name. + let from = 1; + let breadcrumbs_from = (names.map(Into::into).collect(), from); + browser.model().list.model().breadcrumbs.set_entries_from(breadcrumbs_from); } fn show_breadcrumbs_ellipsis(&self, show: bool) { - if let SearcherVariant::ComponentBrowser(browser) = self.view.searcher() { - browser.model().list.model().breadcrumbs.show_ellipsis(show); - } + let browser = self.view.searcher(); + browser.model().list.model().breadcrumbs.show_ellipsis(show); } fn set_section_name_crumb(&self, text: ImString) { - if let SearcherVariant::ComponentBrowser(browser) = self.view.searcher() { - let breadcrumbs = &browser.model().list.model().breadcrumbs; - breadcrumbs.set_entry((SECTION_NAME_CRUMB_INDEX, text.into())); - } + let browser = self.view.searcher(); + let breadcrumbs = &browser.model().list.model().breadcrumbs; + breadcrumbs.set_entry((SECTION_NAME_CRUMB_INDEX, text.into())); } fn on_active_section_change(&self, section_id: component_grid::SectionId) { @@ -281,13 +247,8 @@ impl Model { match component.data { component::Data::FromDatabase { id, .. } => self.controller.documentation_for_entry(*id), - component::Data::Virtual { snippet } => { - if let Some(documentation) = &snippet.documentation_html { - EntryDocumentation::Builtin(documentation.into()) - } else { - default() - } - } + component::Data::Virtual { snippet } => + snippet.documentation.as_ref().map_or_default(EntryDocumentation::builtin), } } else { default() @@ -306,15 +267,6 @@ impl Model { default() } } - - fn should_auto_select_first_action(&self) -> bool { - let user_action = self.controller.current_user_action(); - let list_not_empty = matches!(self.controller.actions(), controller::searcher::Actions::Loaded {list} if list.matching_count() > 0); - // Usually we don't want to select first entry and display docs when user finished typing - // function or argument. - let starting_typing = user_action == UserAction::StartingTypingArgument; - !starting_typing && list_not_empty - } } /// The Searcher presenter, synchronizing state between searcher view and searcher controller. @@ -345,81 +297,61 @@ impl Searcher { eval model.view.searcher_input_changed ((expr) model.input_changed(expr)); action_list_changed <- source::<()>(); - select_entry <- action_list_changed.filter(f_!(model.should_auto_select_first_action())); eval_ model.view.toggle_component_browser_private_entries_visibility ( model.controller.reload_list()); } - match model.view.searcher() { - SearcherVariant::ComponentBrowser(browser) => { - let grid = &browser.model().list.model().grid; - let navigator = &browser.model().list.model().section_navigator; - let breadcrumbs = &browser.model().list.model().breadcrumbs; - let documentation = &browser.model().documentation; - frp::extend! { network - eval_ action_list_changed ([model, grid, navigator] { - model.provider.take(); - let controller_provider = model.controller.provider(); - let namespace_section_count = controller_provider.namespace_section_count(); - navigator.set_namespace_section_count.emit(namespace_section_count); - let provider = provider::Component::provide_new_list(controller_provider, &grid); - *model.provider.borrow_mut() = Some(provider); - }); - new_input <- grid.suggestion_accepted.filter_map(f!((e) model.suggestion_accepted(*e))); - graph.set_node_expression <+ new_input; - - entry_selected <- grid.active.filter_map(|&s| s?.as_entry_id()); - selected_entry_changed <- entry_selected.on_change().constant(()); - grid.unhover_element <+ any2( - &selected_entry_changed, - &model.view.toggle_component_browser_private_entries_visibility, - ); - hovered_not_selected <- all_with(&grid.hovered, &grid.active, |h, s| { - match (h, s) { - (Some(h), Some(s)) => h != s, - _ => false, - } - }); - documentation.frp.show_hovered_item_preview_caption <+ hovered_not_selected; - docs_params <- all3(&action_list_changed, &grid.active, &grid.hovered); - docs <- docs_params.filter_map(f!([model]((_, selected, hovered)) { - let entry = hovered.as_ref().or(selected.as_ref()); - entry.map(|entry| { - if let Some(group_id) = entry.as_header() { - model.documentation_of_group(group_id) - } else { - let entry_id = entry.as_entry_id().expect("GroupEntryId"); - model.documentation_of_component(entry_id) - } - }) - })); - documentation.frp.display_documentation <+ docs; - - eval_ grid.suggestion_accepted([]analytics::remote_log_event("component_browser::suggestion_accepted")); - eval entry_selected((entry) model.suggestion_selected(*entry)); - eval grid.module_entered((id) model.module_entered(*id)); - eval breadcrumbs.selected((id) model.breadcrumb_selected(*id)); - active_section <- grid.active_section.filter_map(|s| *s); - eval active_section((section) model.on_active_section_change(*section)); - } - } - SearcherVariant::OldNodeSearcher(searcher) => { - let searcher = &searcher.frp; - - frp::extend! { network - new_providers <- action_list_changed.map(f_!(model.create_providers())); - searcher.set_actions <+ new_providers; - searcher.select_action <+ select_entry.constant(0); - used_as_suggestion <- searcher.used_as_suggestion.filter_map(|entry| *entry); - new_input <- used_as_suggestion.filter_map(f!((e) model.entry_used_as_suggestion(*e))); - graph.set_node_expression <+ new_input; - eval searcher.selected_entry([model](entry) - if let Some(id) = entry { model.entry_selected_as_suggestion(*id)}); - - eval_ searcher.used_as_suggestion([]analytics::remote_log_event("searcher::used_as_suggestion")); + let browser = model.view.searcher(); + let grid = &browser.model().list.model().grid; + let navigator = &browser.model().list.model().section_navigator; + let breadcrumbs = &browser.model().list.model().breadcrumbs; + let documentation = &browser.model().documentation; + frp::extend! { network + eval_ action_list_changed ([model, grid, navigator] { + model.provider.take(); + let controller_provider = model.controller.provider(); + let namespace_section_count = controller_provider.namespace_section_count(); + navigator.set_namespace_section_count.emit(namespace_section_count); + let provider = provider::Component::provide_new_list(controller_provider, &grid); + *model.provider.borrow_mut() = Some(provider); + }); + new_input <- grid.suggestion_accepted.filter_map(f!((e) model.suggestion_accepted(*e))); + graph.set_node_expression <+ new_input; + + entry_selected <- grid.active.filter_map(|&s| s?.as_entry_id()); + selected_entry_changed <- entry_selected.on_change().constant(()); + grid.unhover_element <+ any2( + &selected_entry_changed, + &model.view.toggle_component_browser_private_entries_visibility, + ); + hovered_not_selected <- all_with(&grid.hovered, &grid.active, |h, s| { + match (h, s) { + (Some(h), Some(s)) => h != s, + _ => false, } - } + }); + documentation.frp.show_hovered_item_preview_caption <+ hovered_not_selected; + docs_params <- all3(&action_list_changed, &grid.active, &grid.hovered); + docs <- docs_params.filter_map(f!([model]((_, selected, hovered)) { + let entry = hovered.as_ref().or(selected.as_ref()); + entry.map(|entry| { + if let Some(group_id) = entry.as_header() { + model.documentation_of_group(group_id) + } else { + let entry_id = entry.as_entry_id().expect("GroupEntryId"); + model.documentation_of_component(entry_id) + } + }) + })); + documentation.frp.display_documentation <+ docs; + + eval_ grid.suggestion_accepted([]analytics::remote_log_event("component_browser::suggestion_accepted")); + eval entry_selected((entry) model.suggestion_selected(*entry)); + eval grid.module_entered((id) model.module_entered(*id)); + eval breadcrumbs.selected((id) model.breadcrumb_selected(*id)); + active_section <- grid.active_section.filter_map(|s| *s); + eval active_section((section) model.on_active_section_change(*section)); } let weak_model = Rc::downgrade(&model); diff --git a/app/gui/suggestion-database/Cargo.toml b/app/gui/suggestion-database/Cargo.toml index a5f5a67ca0e0..4a518646f741 100644 --- a/app/gui/suggestion-database/Cargo.toml +++ b/app/gui/suggestion-database/Cargo.toml @@ -9,11 +9,11 @@ crate-type = ["cdylib", "rlib"] [dependencies] enso-prelude = { path = "../../../lib/rust/prelude" } +enso-doc-parser = { path = "../../../lib/rust/parser/doc-parser" } convert_case = { workspace = true } span-tree = { path = "../language/span-tree" } ast = { path = "../language/ast/impl" } parser = { path = "../language/parser" } -parser-scala = { path = "../language/parser-scala" } enso-profiler = { path = "../../../lib/rust/profiler" } enso-text = { path = "../../../lib/rust/text" } double-representation = { path = "../controller/double-representation" } diff --git a/app/gui/suggestion-database/src/documentation_ir.rs b/app/gui/suggestion-database/src/documentation_ir.rs index 6a6590b2afd8..51ecbc89eb94 100644 --- a/app/gui/suggestion-database/src/documentation_ir.rs +++ b/app/gui/suggestion-database/src/documentation_ir.rs @@ -15,8 +15,8 @@ use crate::NoSuchEntry; use crate::SuggestionDatabase; use double_representation::name::QualifiedName; -use engine_protocol::language_server::DocSection; -use engine_protocol::language_server::Mark; +use enso_doc_parser::DocSection; +use enso_doc_parser::Mark; use std::cmp::Ordering; @@ -32,10 +32,8 @@ use std::cmp::Ordering; pub enum EntryDocumentation { /// No documentation available. Placeholder(Placeholder), - /// Documentation of the entry provided by the Engine. + /// Documentation for the entry. Docs(Documentation), - /// Documentation of builtin components that are not included in the suggestion database. - Builtin(ImString), } impl Default for EntryDocumentation { @@ -85,6 +83,12 @@ impl EntryDocumentation { } } + /// Create documentation for a hard-coded builtin entry. + pub fn builtin(sections: impl IntoIterator) -> Self { + let sections = Rc::new(BuiltinDocumentation::from_doc_sections(sections.into_iter())); + Self::Docs(Documentation::Builtin(sections)) + } + fn method_docs( db: &SuggestionDatabase, entry: &Entry, @@ -169,6 +173,7 @@ pub enum Documentation { ModuleMethod { name: Rc, module_docs: Rc }, Function(Rc), Local(Rc), + Builtin(Rc), } impl Documentation { @@ -329,6 +334,31 @@ impl LocalDocumentation { } + +// ============================ +// === BuiltinDocumentation === +// ============================ + +/// Documentation of hard-coded builtin entries. +#[derive(Debug, Clone, CloneRef, PartialEq)] +#[allow(missing_docs)] +pub struct BuiltinDocumentation { + pub synopsis: Synopsis, +} + +impl BuiltinDocumentation { + /// Constructor. + pub fn from_doc_sections(sections: impl IntoIterator) -> Self { + let FilteredDocSections { tags, synopsis, examples } = + FilteredDocSections::new(sections.into_iter()); + debug_assert!(tags.is_empty()); + debug_assert!(examples.is_empty()); + Self { synopsis } + } +} + + + // ============ // === Tags === // ============ diff --git a/app/gui/suggestion-database/src/entry.rs b/app/gui/suggestion-database/src/entry.rs index bf66203e54df..26184161ebd0 100644 --- a/app/gui/suggestion-database/src/entry.rs +++ b/app/gui/suggestion-database/src/entry.rs @@ -13,9 +13,9 @@ use double_representation::name; use double_representation::name::QualifiedName; use double_representation::name::QualifiedNameRef; use engine_protocol::language_server; -use engine_protocol::language_server::DocSection; use engine_protocol::language_server::FieldUpdate; use engine_protocol::language_server::SuggestionsDatabaseModification; +use enso_doc_parser::DocSection; use enso_text::Location; use language_server::types::FieldAction; @@ -226,30 +226,28 @@ pub enum Scope { #[derive(Clone, Debug, Eq, PartialEq)] pub struct Entry { /// A type of suggestion. - pub kind: Kind, + pub kind: Kind, /// A module where the suggested object is defined. - pub defined_in: QualifiedName, + pub defined_in: QualifiedName, /// A name of suggested object. - pub name: String, + pub name: String, /// Argument lists of suggested object (atom or function). If the object does not take any /// arguments, the list is empty. - pub arguments: Vec, + pub arguments: Vec, /// A type returned by the suggested object. - pub return_type: QualifiedName, + pub return_type: QualifiedName, /// A module reexporting this entity. - pub reexported_in: Option, - /// A HTML documentation associated with object. - pub documentation_html: Option, + pub reexported_in: Option, /// A list of documentation sections associated with object. - pub documentation: Vec, + pub documentation: Vec, /// A type of the "self" argument. This field is `None` for non-method suggestions. - pub self_type: Option, + pub self_type: Option, /// A flag set to true if the method is a static or module method. - pub is_static: bool, + pub is_static: bool, /// A scope where this suggestion is visible. - pub scope: Scope, + pub scope: Scope, /// A name of a custom icon to use when displaying the entry. - pub icon_name: Option, + pub icon_name: Option, } @@ -258,7 +256,7 @@ pub struct Entry { impl Entry { /// Create new entry with required parameters only. /// - /// The entry will be flagget as non-static, with [`Scope::Everywhere`] and all optional fields + /// The entry will be flagged as non-static, with [`Scope::Everywhere`] and all optional fields /// will be [`None`]. pub fn new( kind: Kind, @@ -274,7 +272,6 @@ impl Entry { return_type: return_type.into(), is_static: false, reexported_in: None, - documentation_html: None, documentation: default(), self_type: None, scope: Scope::Everywhere, @@ -392,12 +389,6 @@ impl Entry { self } - /// Takes self and returns it with new [`documentation_html`] value. - pub fn with_documentation(mut self, html: impl Into) -> Self { - self.documentation_html = Some(html.into()); - self - } - /// Takes self and returns it with new [`documentation`] value; pub fn with_doc_sections(mut self, sections: Vec) -> Self { self.documentation = sections; @@ -604,19 +595,17 @@ impl Entry { }) } - let (documentation, icon_name, doc_sections) = match &mut entry { - Type { documentation, documentation_html, documentation_sections, .. } - | Constructor { documentation, documentation_html, documentation_sections, .. } - | Method { documentation, documentation_html, documentation_sections, .. } - | Module { documentation, documentation_html, documentation_sections, .. } - | Function { documentation, documentation_html, documentation_sections, .. } - | Local { documentation, documentation_html, documentation_sections, .. } => { - let documentation = - Self::make_html_docs(mem::take(documentation), mem::take(documentation_html)); - let icon_name = find_icon_name_in_doc_sections(&*documentation_sections); - (documentation, icon_name, mem::take(documentation_sections)) - } + let documentation = match &entry { + Type { documentation, .. } + | Constructor { documentation, .. } + | Method { documentation, .. } + | Module { documentation, .. } + | Function { documentation, .. } + | Local { documentation, .. } => + documentation.as_ref().map(|s| s.as_ref()).unwrap_or_default(), }; + let doc_sections = enso_doc_parser::parse(documentation); + let icon_name = find_icon_name_in_doc_sections(&doc_sections); let reexported_in: Option = match &mut entry { Type { reexport: Some(reexport), .. } | Constructor { reexport: Some(reexport), .. } @@ -654,36 +643,12 @@ impl Entry { ), Module { module, .. } => Self::new_module(to_qualified_name(module)), }; - this.documentation_html = documentation; this.documentation = doc_sections; this.icon_name = icon_name; this.reexported_in = reexported_in; this } - /// Returns the documentation in html depending on the information received from the Engine. - /// - /// Depending on the engine version, we may receive the documentation in HTML format already, - /// or the raw text which needs to be parsed. This function takes two fields of - /// [`language_server::types::SuggestionEntry`] and depending on availability, returns the - /// HTML docs fields, or parsed raw docs field. - fn make_html_docs(docs: Option, docs_html: Option) -> Option { - if docs_html.is_some() { - docs_html - } else { - docs.map(|docs| { - let parser = parser_scala::DocParser::new(); - match parser { - Ok(p) => { - let output = p.generate_html_doc_pure((*docs).to_string()); - output.unwrap_or(docs) - } - Err(_) => docs, - } - }) - } - } - /// Apply modification to the entry. pub fn apply_modifications( &mut self, @@ -694,19 +659,11 @@ impl Entry { let return_type = m.return_type.map(|f| f.try_map(QualifiedName::from_text)).transpose(); let self_type = m.self_type.map(|f| f.try_map(QualifiedName::from_text)).transpose(); let reexport = m.reexport.map(|f| f.try_map(QualifiedName::from_text)).transpose(); + let docs = m.documentation.map(|docs| docs.map(|docs| enso_doc_parser::parse(&docs))); let update_results = [ return_type .and_then(|m| Entry::apply_field_update("return_type", &mut self.return_type, m)), - Entry::apply_opt_field_update( - "documentation", - &mut self.documentation_html, - m.documentation, - ), - Entry::apply_field_update( - "documentation_sections", - &mut self.documentation, - m.documentation_sections, - ), + Entry::apply_default_field_update("documentation", &mut self.documentation, docs), module.and_then(|m| Entry::apply_field_update("module", &mut self.defined_in, m)), self_type .and_then(|s| Entry::apply_opt_field_update("self_type", &mut self.self_type, s)), @@ -815,11 +772,21 @@ impl Entry { } Ok(()) } -} -impl From for Entry { - fn from(entry: language_server::types::SuggestionEntry) -> Self { - Self::from_ls_entry(entry) + /// Apply an update to a field that can be removed by settings its value to its type's default. + fn apply_default_field_update( + field_name: &'static str, + field: &mut T, + update: Option>, + ) -> FallibleResult { + let err = InvalidFieldUpdate(field_name); + if let Some(update) = update { + *field = match update.tag { + FieldAction::Set => update.value.ok_or(err)?, + FieldAction::Remove => default(), + }; + } + Ok(()) } } @@ -1132,7 +1099,7 @@ mod test { /// of a keyed [`DocSection`] which has its key equal to the `Icon` string. #[test] fn find_icon_name_in_doc_section_with_icon_key() { - use language_server::types::DocSection; + use enso_doc_parser::DocSection; let doc_sections = [ DocSection::Paragraph { body: "Some paragraph.".into() }, DocSection::Keyed { key: "NotIcon".into(), body: "example_not_icon_body".into() }, @@ -1174,8 +1141,7 @@ mod test { tag_values: Vec::new(), }; let entry = Entry::new_method(defined_in, on_type, "entry", return_type, true) - .with_arguments(vec![argument]) - .with_documentation("Some docs"); + .with_arguments(vec![argument]); Self { modified_entry: entry.clone(), expected_entry: entry } } @@ -1199,25 +1165,23 @@ mod test { #[test] fn applying_simple_fields_modification() { let mut test = ApplyModificationTest::new(); + let new_documentation = "NewDocumentation"; let modification = SuggestionsDatabaseModification { - arguments: vec![], - module: Some(FieldUpdate::set("local.Project.NewModule".to_owned())), - self_type: Some(FieldUpdate::set( - "local.Project.NewModule.NewType".to_owned(), - )), - return_type: Some(FieldUpdate::set( + arguments: vec![], + module: Some(FieldUpdate::set("local.Project.NewModule".to_owned())), + self_type: Some(FieldUpdate::set("local.Project.NewModule.NewType".to_owned())), + return_type: Some(FieldUpdate::set( "local.Project.NewModule.NewReturnType".to_owned(), )), - documentation: Some(FieldUpdate::set("NewDocumentation".to_owned())), - documentation_sections: None, - scope: None, - reexport: Some(FieldUpdate::set("local.Project.NewReexport".to_owned())), + documentation: Some(FieldUpdate::set(new_documentation.to_owned())), + scope: None, + reexport: Some(FieldUpdate::set("local.Project.NewReexport".to_owned())), }; test.expected_entry.defined_in = "local.Project.NewModule".try_into().unwrap(); test.expected_entry.self_type = Some("local.Project.NewModule.NewType".try_into().unwrap()); test.expected_entry.return_type = "local.Project.NewModule.NewReturnType".try_into().unwrap(); - test.expected_entry.documentation_html = Some("NewDocumentation".to_owned()); + test.expected_entry.documentation = enso_doc_parser::parse(new_documentation); test.expected_entry.reexported_in = Some("local.Project.NewReexport".try_into().unwrap()); let result = test.check_modification(modification); assert!(result.is_empty()); @@ -1230,7 +1194,6 @@ mod test { documentation: Some(FieldUpdate::remove()), ..default() }; - test.expected_entry.documentation_html = None; let result = test.check_modification(modification); assert!(result.is_empty()); } @@ -1249,7 +1212,6 @@ mod test { ..default() }; test.expected_entry.defined_in = "local.Project.NewModule".try_into().unwrap(); - test.expected_entry.documentation_html = None; let result = test.check_modification(modification); assert_eq!(result.len(), 1); } diff --git a/app/gui/suggestion-database/src/lib.rs b/app/gui/suggestion-database/src/lib.rs index 4c6f76fce864..28ad0421ac77 100644 --- a/app/gui/suggestion-database/src/lib.rs +++ b/app/gui/suggestion-database/src/lib.rs @@ -2,6 +2,7 @@ #![recursion_limit = "512"] // === Features === +#![feature(anonymous_lifetime_in_impl_trait)] #![feature(arc_unwrap_or_clone)] #![feature(async_closure)] #![feature(associated_type_bounds)] @@ -316,7 +317,7 @@ pub enum Notification { /// often-called Language Server methods returns the list of keys of this database instead of the /// whole entries. Additionally the suggestions contains information about functions and their /// argument names and types. -#[derive(Clone, Debug, Default)] +#[derive(Debug, Default)] pub struct SuggestionDatabase { entries: RefCell>>, qualified_name_to_id_map: RefCell, @@ -420,17 +421,13 @@ impl SuggestionDatabase { let mut mp_to_id_map = self.method_pointer_to_id_map.borrow_mut(); let mut hierarchy_index = self.hierarchy_index.borrow_mut(); match update { - entry::Update::Add { id, suggestion } => match (*suggestion).try_into() { - Ok(entry) => { - qn_to_id_map.set_and_warn_if_existed(&Entry::qualified_name(&entry), id); - mp_to_id_map.set(&entry, id); - hierarchy_index.add(id, &entry, &qn_to_id_map); - entries.insert(id, Rc::new(entry)); - } - Err(err) => { - error!("Discarding update for {id}: {err}") - } - }, + entry::Update::Add { id, suggestion } => { + let entry = Entry::from_ls_entry(*suggestion); + qn_to_id_map.set_and_warn_if_existed(&Entry::qualified_name(&entry), id); + mp_to_id_map.set(&entry, id); + hierarchy_index.add(id, &entry, &qn_to_id_map); + entries.insert(id, Rc::new(entry)); + } entry::Update::Remove { id } => { let removed = entries.remove(&id); match removed { @@ -672,15 +669,13 @@ pub mod test { // Non-empty db let entry = SuggestionEntry::Constructor { - name: "TextAtom".to_string(), - module: "test.TestProject.TestModule".to_string(), - arguments: vec![], - return_type: "test.TestProject.TestModule.TestAtom".to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, - reexport: None, + name: "TextAtom".to_string(), + module: "test.TestProject.TestModule".to_string(), + arguments: vec![], + return_type: "test.TestProject.TestModule.TestAtom".to_string(), + documentation: None, + external_id: None, + reexport: None, }; let db_entry = SuggestionsDatabaseEntry { id: 12, suggestion: entry }; let response = language_server::response::GetSuggestionDatabase { @@ -701,15 +696,13 @@ pub mod test { let replaced_entry = QualifiedName::from_text("Standard.Base.Number").unwrap(); let (replaced_id, _) = db.lookup_by_qualified_name(&replaced_entry).unwrap(); let new_entry = SuggestionEntry::Constructor { - name: "NewEntry".to_owned(), - module: "test.TestProject.TestModule".to_owned(), - arguments: vec![], - return_type: "test.TestProject.TestModule.TestAtom".to_owned(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, - reexport: None, + name: "NewEntry".to_owned(), + module: "test.TestProject.TestModule".to_owned(), + arguments: vec![], + return_type: "test.TestProject.TestModule.TestAtom".to_owned(), + documentation: None, + external_id: None, + reexport: None, }; let new_entry_modification = SuggestionsDatabaseModification { module: Some(FieldUpdate::set("test.TestProject.TestModule2".to_owned())), @@ -812,67 +805,55 @@ pub mod test { fn lookup_by_fully_qualified_name_in_db_created_from_ls_response() { // Initialize a suggestion database with sample entries. let entry0 = SuggestionEntry::Type { - name: "TestType".to_string(), - module: "test.TestProject.TestModule".to_string(), - params: vec![], - parent_type: Some("Any".to_string()), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "TestType".to_string(), + module: "test.TestProject.TestModule".to_string(), + params: vec![], + parent_type: Some("Any".to_string()), + reexport: None, + documentation: None, + external_id: None, }; let entry1 = SuggestionEntry::Constructor { - name: "TestAtom".to_string(), - module: "test.TestProject.TestModule".to_string(), - arguments: vec![], - return_type: "test.TestProject.TestModule.TestType".to_string(), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "TestAtom".to_string(), + module: "test.TestProject.TestModule".to_string(), + arguments: vec![], + return_type: "test.TestProject.TestModule.TestType".to_string(), + reexport: None, + documentation: None, + external_id: None, }; let entry2 = SuggestionEntry::Method { - name: "create_process".to_string(), - module: "Standard.Builtins.Main".to_string(), - self_type: "Standard.Builtins.Main.System".to_string(), - arguments: vec![], - return_type: "Standard.Builtins.Main.System_Process_Result".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "create_process".to_string(), + module: "Standard.Builtins.Main".to_string(), + self_type: "Standard.Builtins.Main.System".to_string(), + arguments: vec![], + return_type: "Standard.Builtins.Main.System_Process_Result".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let entry3 = SuggestionEntry::Module { - module: "local.Unnamed_6.Main".to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - reexport: None, + module: "local.Unnamed_6.Main".to_string(), + documentation: None, + reexport: None, }; let entry4 = SuggestionEntry::Local { - module: "local.Unnamed_6.Main".to_string(), - name: "operator1".to_string(), - return_type: "Standard.Base.Data.Vector.Vector".to_string(), - external_id: None, - scope: (default()..=default()).into(), - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "local.Unnamed_6.Main".to_string(), + name: "operator1".to_string(), + return_type: "Standard.Base.Data.Vector.Vector".to_string(), + external_id: None, + scope: (default()..=default()).into(), + documentation: None, }; let entry5 = SuggestionEntry::Function { - module: "NewProject.NewModule".to_string(), - name: "testFunction1".to_string(), - arguments: vec![], - return_type: "Standard.Base.Data.Vector.Vector".to_string(), - scope: (default()..=default()).into(), - external_id: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "NewProject.NewModule".to_string(), + name: "testFunction1".to_string(), + arguments: vec![], + return_type: "Standard.Base.Data.Vector.Vector".to_string(), + scope: (default()..=default()).into(), + external_id: None, + documentation: None, }; let ls_response = language_server::response::GetSuggestionDatabase { entries: vec![ @@ -909,54 +890,46 @@ pub mod test { fn lookup_by_method_pointer_in_db_created_from_ls_response() { // Initialize a suggestion database with sample entries. let method1 = SuggestionEntry::Method { - name: "create_process".to_string(), - module: "Standard.Base".to_string(), - self_type: "Standard.Base.System".to_string(), - arguments: vec![], - return_type: "Standard.Base.System.System_Process_Result".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "create_process".to_string(), + module: "Standard.Base".to_string(), + self_type: "Standard.Base.System".to_string(), + arguments: vec![], + return_type: "Standard.Base.System.System_Process_Result".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let method2 = SuggestionEntry::Method { - name: "exit".to_string(), - module: "Standard.Base".to_string(), - self_type: "Standard.Base.System".to_string(), - arguments: vec![], - return_type: "Standard.Base.Data.Numbers.Integer".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "exit".to_string(), + module: "Standard.Base".to_string(), + self_type: "Standard.Base.System".to_string(), + arguments: vec![], + return_type: "Standard.Base.Data.Numbers.Integer".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let method3 = SuggestionEntry::Method { - name: "println".to_string(), - module: "Standard.Base.IO".to_string(), - self_type: "Standard.Base.IO".to_string(), - arguments: vec![], - return_type: "Standard.Base.Nothing.Nothing".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "println".to_string(), + module: "Standard.Base.IO".to_string(), + self_type: "Standard.Base.IO".to_string(), + arguments: vec![], + return_type: "Standard.Base.Nothing.Nothing".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let function = SuggestionEntry::Function { - module: "NewProject.NewModule".to_string(), - name: "test_function".to_string(), - arguments: vec![], - return_type: "Standard.Base.Data.Vector.Vector".to_string(), - scope: (default()..=default()).into(), - external_id: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "NewProject.NewModule".to_string(), + name: "test_function".to_string(), + arguments: vec![], + return_type: "Standard.Base.Data.Vector.Vector".to_string(), + scope: (default()..=default()).into(), + external_id: None, + documentation: None, }; let ls_response = language_server::response::GetSuggestionDatabase { entries: vec![ @@ -1009,32 +982,26 @@ pub mod test { fn initialize_database_with_invalid_entries() { // Prepare some nonsense inputs from the Engine. let entry_with_empty_name = SuggestionEntry::Constructor { - name: "".to_string(), - module: "Empty.Entry".to_string(), - arguments: vec![], - return_type: "".to_string(), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "".to_string(), + module: "Empty.Entry".to_string(), + arguments: vec![], + return_type: "".to_string(), + reexport: None, + documentation: None, + external_id: None, }; let empty_entry = SuggestionEntry::Local { - module: "".to_string(), - name: "".to_string(), - return_type: "".to_string(), - external_id: None, - scope: (default()..=default()).into(), - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "".to_string(), + name: "".to_string(), + return_type: "".to_string(), + external_id: None, + scope: (default()..=default()).into(), + documentation: None, }; let gibberish_entry = SuggestionEntry::Module { - module: GIBBERISH_MODULE_NAME.to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - reexport: None, + module: GIBBERISH_MODULE_NAME.to_string(), + documentation: None, + reexport: None, }; let ls_response = language_server::response::GetSuggestionDatabase { @@ -1067,33 +1034,27 @@ pub mod test { let entry_added_id = 123; // Modify the database contents by applying an update event. let entry_doc_change = Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: None, - self_type: None, - return_type: None, - documentation: Some(FieldUpdate::set("New doc".to_string())), - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: None, + self_type: None, + return_type: None, + documentation: Some(FieldUpdate::set("New doc".to_string())), + scope: None, + reexport: None, }); let entry_mod_change = Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: Some(FieldUpdate::set( - "new_project.NewProject.NewModule".to_string(), - )), - self_type: None, - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: Some(FieldUpdate::set("new_project.NewProject.NewModule".to_string())), + self_type: None, + return_type: None, + documentation: None, + scope: None, + reexport: None, }); let new_entry = SuggestionEntry::Module { - module: "local.Unnamed_6.Main".to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - reexport: None, + module: "local.Unnamed_6.Main".to_string(), + documentation: None, + reexport: None, }; let update = SuggestionDatabaseUpdatesEvent { updates: vec![ @@ -1131,43 +1092,37 @@ pub mod test { // Modify the database contents by applying an update event. let method1 = SuggestionEntry::Method { - name: "create_process".to_string(), - module: "Standard.Base".to_string(), - self_type: "Standard.Base.System".to_string(), - arguments: vec![], - return_type: "Standard.Base.System.System_Process_Result".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "create_process".to_string(), + module: "Standard.Base".to_string(), + self_type: "Standard.Base.System".to_string(), + arguments: vec![], + return_type: "Standard.Base.System.System_Process_Result".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let method2 = SuggestionEntry::Method { - name: "exit".to_string(), - module: "Standard.Base".to_string(), - self_type: "Standard.Base.System".to_string(), - arguments: vec![], - return_type: "Standard.Base.Data.Numbers.Integer".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "exit".to_string(), + module: "Standard.Base".to_string(), + self_type: "Standard.Base.System".to_string(), + arguments: vec![], + return_type: "Standard.Base.Data.Numbers.Integer".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let method3 = SuggestionEntry::Method { - name: "println".to_string(), - module: "Standard.Base.IO".to_string(), - self_type: "Standard.Base.IO".to_string(), - arguments: vec![], - return_type: "Standard.Base.Nothing.Nothing".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "println".to_string(), + module: "Standard.Base.IO".to_string(), + self_type: "Standard.Base.IO".to_string(), + arguments: vec![], + return_type: "Standard.Base.Nothing.Nothing".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let update1 = SuggestionDatabaseUpdatesEvent { updates: vec![ @@ -1200,14 +1155,13 @@ pub mod test { lookup_method_pointer(&db, &method_pointer3); let method_change = Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: Some(FieldUpdate::set("Standard.Base".to_owned())), - self_type: None, - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: Some(FieldUpdate::set("Standard.Base".to_owned())), + self_type: None, + return_type: None, + documentation: None, + scope: None, + reexport: None, }); let update2 = SuggestionDatabaseUpdatesEvent { updates: vec![entry::Update::Modify { @@ -1250,15 +1204,13 @@ pub mod test { fn lookup_by_fully_qualified_name_after_db_update_reuses_id() { // Initialize a suggestion database with a sample entry. let entry1 = SuggestionEntry::Constructor { - name: "TextAtom".to_string(), - module: "TestProject.TestModule".to_string(), - arguments: vec![], - return_type: "TestAtom".to_string(), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "TextAtom".to_string(), + module: "TestProject.TestModule".to_string(), + arguments: vec![], + return_type: "TestAtom".to_string(), + reexport: None, + documentation: None, + external_id: None, }; let id = 1; let response = language_server::response::GetSuggestionDatabase { @@ -1276,11 +1228,9 @@ pub mod test { // Apply a DB update adding a different entry at the same `id`. let entry2 = SuggestionEntry::Module { - module: "local.Unnamed_6.Main".to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - reexport: None, + module: "local.Unnamed_6.Main".to_string(), + documentation: None, + reexport: None, }; let update = SuggestionDatabaseUpdatesEvent { updates: vec![entry::Update::Add { id, suggestion: Box::new(entry2) }], @@ -1380,67 +1330,55 @@ pub mod test { fn hierarchy_index_is_populated_after_construction() { // Initialize a suggestion database with sample entries. let entry0 = SuggestionEntry::Type { - name: "TestType".to_string(), - module: "test.TestProject.TestModule".to_string(), - params: vec![], - parent_type: Some("Any".to_string()), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "TestType".to_string(), + module: "test.TestProject.TestModule".to_string(), + params: vec![], + parent_type: Some("Any".to_string()), + reexport: None, + documentation: None, + external_id: None, }; let entry1 = SuggestionEntry::Constructor { - name: "Empty".to_string(), - module: "test.TestProject.TestModule".to_string(), - arguments: vec![], - return_type: "test.TestProject.TestModule.TestType".to_string(), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "Empty".to_string(), + module: "test.TestProject.TestModule".to_string(), + arguments: vec![], + return_type: "test.TestProject.TestModule.TestType".to_string(), + reexport: None, + documentation: None, + external_id: None, }; let entry2 = SuggestionEntry::Method { - name: "test_method".to_string(), - module: "test.TestProject.TestModule".to_string(), - self_type: "test.TestProject.TestModule.TestType".to_string(), - arguments: vec![], - return_type: "Standard.Builtins.Main.System_Process_Result".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "test_method".to_string(), + module: "test.TestProject.TestModule".to_string(), + self_type: "test.TestProject.TestModule.TestType".to_string(), + arguments: vec![], + return_type: "Standard.Builtins.Main.System_Process_Result".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }; let entry3 = SuggestionEntry::Module { - module: "test.TestProject.TestModule".to_string(), - documentation: None, - documentation_html: None, - documentation_sections: default(), - reexport: None, + module: "test.TestProject.TestModule".to_string(), + documentation: None, + reexport: None, }; let entry4 = SuggestionEntry::Local { - module: "test.TestProject.TestModule".to_string(), - name: "operator1".to_string(), - return_type: "Standard.Base.Data.Vector.Vector".to_string(), - external_id: None, - scope: (default()..=default()).into(), - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "test.TestProject.TestModule".to_string(), + name: "operator1".to_string(), + return_type: "Standard.Base.Data.Vector.Vector".to_string(), + external_id: None, + scope: (default()..=default()).into(), + documentation: None, }; let entry5 = SuggestionEntry::Function { - module: "test.TestProject.TestModule".to_string(), - name: "testFunction1".to_string(), - arguments: vec![], - return_type: "Standard.Base.Data.Vector.Vector".to_string(), - scope: (default()..=default()).into(), - external_id: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), + module: "test.TestProject.TestModule".to_string(), + name: "testFunction1".to_string(), + arguments: vec![], + return_type: "Standard.Base.Data.Vector.Vector".to_string(), + scope: (default()..=default()).into(), + external_id: None, + documentation: None, }; let ls_response = language_server::response::GetSuggestionDatabase { entries: vec![ @@ -1507,17 +1445,15 @@ pub mod test { updates: vec![entry::Update::Add { id: added_id, suggestion: Box::new(SuggestionEntry::Method { - name: "new_method".to_string(), - module: "local.Project.Submodule".to_string(), - self_type: "local.Project.Submodule.TestType".to_string(), - arguments: vec![], - return_type: "Some.Type".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: default(), - external_id: None, + name: "new_method".to_string(), + module: "local.Project.Submodule".to_string(), + self_type: "local.Project.Submodule.TestType".to_string(), + arguments: vec![], + return_type: "Some.Type".to_string(), + is_static: false, + reexport: None, + documentation: None, + external_id: None, }), }], current_version: 1, @@ -1537,16 +1473,13 @@ pub mod test { id: added_id, external_id: None, modification: Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: None, - self_type: Some(FieldUpdate::set( - "Standard.Base.Maybe".to_string(), - )), - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: None, + self_type: Some(FieldUpdate::set("Standard.Base.Maybe".to_string())), + return_type: None, + documentation: None, + scope: None, + reexport: None, }), }], current_version: 2, @@ -1593,11 +1526,9 @@ pub mod test { entry::Update::Add { id: 20, suggestion: Box::new(SuggestionEntry::Module { - module: "Standard.NewModule".to_string(), - documentation: None, - documentation_html: None, - reexport: None, - documentation_sections: vec![], + module: "Standard.NewModule".to_string(), + documentation: None, + reexport: None, }), }, // Move Number and Boolean to the new module. @@ -1605,32 +1536,26 @@ pub mod test { id: modify_id_1, external_id: None, modification: Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: Some(FieldUpdate::set( - "Standard.NewModule".to_string(), - )), - self_type: None, - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: Some(FieldUpdate::set("Standard.NewModule".to_string())), + self_type: None, + return_type: None, + documentation: None, + scope: None, + reexport: None, }), }, entry::Update::Modify { id: modify_id_2, external_id: None, modification: Box::new(SuggestionsDatabaseModification { - arguments: vec![], - module: Some(FieldUpdate::set( - "Standard.NewModule".to_string(), - )), - self_type: None, - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], + module: Some(FieldUpdate::set("Standard.NewModule".to_string())), + self_type: None, + return_type: None, + documentation: None, + scope: None, + reexport: None, }), }, // Modify the definition of Maybe. It should not affect the hierarchy index. @@ -1638,14 +1563,13 @@ pub mod test { id: maybe_id, external_id: None, modification: Box::new(SuggestionsDatabaseModification { - arguments: vec![], // remove arguments - module: None, - self_type: None, - return_type: None, - documentation: None, - documentation_sections: None, - scope: None, - reexport: None, + arguments: vec![], // remove arguments + module: None, + self_type: None, + return_type: None, + documentation: None, + scope: None, + reexport: None, }), }, ], @@ -1676,17 +1600,15 @@ pub mod test { updates: vec![entry::Update::Add { id: 20, suggestion: Box::new(SuggestionEntry::Method { - external_id: None, - name: "new_method".to_string(), - module: "Standard.Base".to_string(), - arguments: vec![], - self_type: "Standard.Base.NewType".to_string(), - return_type: "Standard.Base.Maybe".to_string(), - is_static: false, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: vec![], + external_id: None, + name: "new_method".to_string(), + module: "Standard.Base".to_string(), + arguments: vec![], + self_type: "Standard.Base.NewType".to_string(), + return_type: "Standard.Base.Maybe".to_string(), + is_static: false, + reexport: None, + documentation: None, }), }], current_version: 1, @@ -1697,15 +1619,13 @@ pub mod test { updates: vec![entry::Update::Add { id: 21, suggestion: Box::new(SuggestionEntry::Type { - external_id: None, - name: "NewType".to_string(), - module: "Standard.Base".to_string(), - params: vec![], - parent_type: None, - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: vec![], + external_id: None, + name: "NewType".to_string(), + module: "Standard.Base".to_string(), + params: vec![], + parent_type: None, + reexport: None, + documentation: None, }), }], current_version: 2, @@ -1724,15 +1644,13 @@ pub mod test { updates: vec![entry::Update::Add { id: 20, suggestion: Box::new(SuggestionEntry::Type { - external_id: None, - name: "NewType".to_string(), - module: "Standard.NewModule".to_string(), - params: vec![], - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: vec![], - parent_type: None, + external_id: None, + name: "NewType".to_string(), + module: "Standard.NewModule".to_string(), + params: vec![], + reexport: None, + documentation: None, + parent_type: None, }), }], current_version: 1, @@ -1743,11 +1661,9 @@ pub mod test { updates: vec![entry::Update::Add { id: 21, suggestion: Box::new(SuggestionEntry::Module { - module: "Standard.NewModule".to_string(), - reexport: None, - documentation: None, - documentation_html: None, - documentation_sections: vec![], + module: "Standard.NewModule".to_string(), + reexport: None, + documentation: None, }), }], current_version: 2, diff --git a/app/gui/suggestion-database/src/mock.rs b/app/gui/suggestion-database/src/mock.rs index 6f1d451ce521..3b9b2d9df1be 100644 --- a/app/gui/suggestion-database/src/mock.rs +++ b/app/gui/suggestion-database/src/mock.rs @@ -13,6 +13,15 @@ use enso_text::Location; +// =============== +// === Exports === +// =============== + +// Re-exported here so it can be referenced by macro-generated code. +pub use enso_doc_parser; + + + // ================= // === Constants === // ================= @@ -49,9 +58,7 @@ pub const DEFAULT_TYPE: &str = "Standard.Base.Any"; /// .add_and_enter_type("Type", vec![], |e| e.with_icon(IconName::from_snake_case("an_icon"))); /// builder.add_constructor("Constructor", vec![], |e| e); /// builder.leave(); -/// builder.add_method("module_method", vec![], "local.Project.Type", true, |e| { -/// e.with_documentation("A module method") -/// }); +/// builder.add_method("module_method", vec![], "local.Project.Type", true, |e| e); /// builder.leave(); /// let db = builder.result; /// @@ -311,7 +318,6 @@ macro_rules! mock_suggestion_database_entries { /// } /// local.Project { /// mod Submodule { -/// #[with_documentation("Some test type")] /// type TestType (a: Standard.Base.Number, b: Standard.Base.Maybe) { /// static fn static_method(x) -> Standard.Base.Number; /// } @@ -380,13 +386,13 @@ macro_rules! mock_suggestion_database { #[macro_export] macro_rules! doc_section_mark { (!) => { - $crate::engine_protocol::language_server::Mark::Important + $crate::mock::enso_doc_parser::Mark::Important }; (>) => { - $crate::engine_protocol::language_server::Mark::Example + $crate::mock::enso_doc_parser::Mark::Example }; (?) => { - $crate::engine_protocol::language_server::Mark::Info + $crate::mock::enso_doc_parser::Mark::Info }; } @@ -423,33 +429,27 @@ macro_rules! doc_section_mark { #[macro_export] macro_rules! doc_section { (@ $tag:expr, $body:expr) => { - $crate::engine_protocol::language_server::DocSection::Tag { - name: $tag.into(), - body: $body.into(), - } + $crate::mock::enso_doc_parser::DocSection::Tag { name: $tag.into(), body: $body.into() } }; ($mark:tt $body:expr) => { - $crate::engine_protocol::language_server::DocSection::Marked { + $crate::mock::enso_doc_parser::DocSection::Marked { mark: $crate::doc_section_mark!($mark), header: None, body: $body.into(), } }; ($mark:tt $header:expr, $body:expr) => { - $crate::engine_protocol::language_server::DocSection::Marked { + $crate::mock::enso_doc_parser::DocSection::Marked { mark: $crate::doc_section_mark!($mark), header: Some($header.into()), body: $body.into(), } }; ($paragraph:expr) => { - $crate::engine_protocol::language_server::DocSection::Paragraph { body: $paragraph.into() } + $crate::mock::enso_doc_parser::DocSection::Paragraph { body: $paragraph.into() } }; ($key:expr => $body:expr) => { - $crate::engine_protocol::language_server::DocSection::Keyed { - key: $key.into(), - body: $body.into(), - } + $crate::mock::enso_doc_parser::DocSection::Keyed { key: $key.into(), body: $body.into() } }; } @@ -475,7 +475,6 @@ pub fn standard_db_mock() -> SuggestionDatabase { } local.Project { mod Submodule { - #[with_documentation("Some test type")] type TestType (a: Standard.Base.Number, b: Standard.Base.Maybe) { static fn static_method(x) -> Standard.Base.Number; } diff --git a/app/gui/view/component-browser/Cargo.toml b/app/gui/view/component-browser/Cargo.toml index e2da82b2562b..a4bd13f0129d 100644 --- a/app/gui/view/component-browser/Cargo.toml +++ b/app/gui/view/component-browser/Cargo.toml @@ -8,7 +8,13 @@ edition = "2021" crate-type = ["cdylib", "rlib"] [dependencies] +enso-frp = { path = "../../../../lib/rust/frp" } +ensogl = { path = "../../../../lib/rust/ensogl" } +ensogl-gui-component = { path = "../../../../lib/rust/ensogl/component/gui" } +ensogl-hardcoded-theme = { path = "../../../../lib/rust/ensogl/app/theme/hardcoded" } enso-prelude = { path = "../../../../lib/rust/prelude" } ide-view-component-list-panel-breadcrumbs = { path = "component-list-panel/breadcrumbs" } ide-view-component-list-panel = { path = "component-list-panel" } +ide-view-documentation = { path = "../documentation" } +ide-view-graph-editor = { path = "../graph-editor" } ensogl-text = { path = "../../../../lib/rust/ensogl/component/text" } diff --git a/app/gui/view/component-browser/src/lib.rs b/app/gui/view/component-browser/src/lib.rs index dc725ac96695..ffc4bf7b711e 100644 --- a/app/gui/view/component-browser/src/lib.rs +++ b/app/gui/view/component-browser/src/lib.rs @@ -4,6 +4,7 @@ //! in `ide_view` crate, because the Documentation panel is used by old node searcher as well, and //! we need to avoid crates' circular dependencies. +#![recursion_limit = "256"] // === Standard Linter Configuration === #![deny(non_ascii_idents)] #![warn(unsafe_code)] @@ -18,6 +19,21 @@ #![warn(unused_import_braces)] #![warn(unused_qualifications)] +use enso_prelude::*; +use ensogl::prelude::*; + +use ide_view_documentation as documentation; +use ide_view_graph_editor::component::node::HEIGHT as NODE_HEIGHT; + +use enso_frp as frp; +use ensogl::application::frp::API; +use ensogl::application::Application; +use ensogl::display; +use ensogl::display::shape::StyleWatchFrp; +use ensogl_gui_component::component; +use ensogl_hardcoded_theme::application::component_browser as theme; + + // ============== // === Export === @@ -25,3 +41,151 @@ pub use ide_view_component_list_panel as component_list_panel; pub use ide_view_component_list_panel_breadcrumbs as breadcrumbs; + + + +/// The Model of Component Browser View. +#[allow(missing_docs)] +#[derive(Clone, CloneRef, Debug)] +pub struct Model { + display_object: display::object::Instance, + pub list: component_list_panel::View, + pub documentation: documentation::View, +} + +impl component::Model for Model { + fn label() -> &'static str { + "ComponentBrowser" + } + + fn new(app: &Application) -> Self { + let display_object = display::object::Instance::new(); + let list = app.new_view::(); + let documentation = documentation::View::new(app); + app.display.default_scene.layers.node_searcher.add(&display_object); + display_object.add_child(&list); + display_object.add_child(&documentation); + + // We need to set the shapes order for the documentation panel overlay to be on top of the + // CB background; otherwise, the background would block some mouse events. The CB background + // is big because of the surrounding shadow and partially overlaps the left side of the + // documentation panel. + let scene = &app.display.default_scene; + shapes_order_dependencies! { + scene => { + component_list_panel::background -> documentation::overlay; + } + } + + Self { display_object, list, documentation } + } +} + +impl Model { + fn expression_input_position( + size: &Vector2, + vertical_gap: &f32, + snap_to_pixel_offset: &Vector2, + ) -> Vector2 { + let half_node_height = NODE_HEIGHT / 2.0; + let panel_left = -size.x / 2.0; + let panel_bottom = -size.y / 2.0; + let x = panel_left; + let y = panel_bottom - vertical_gap - half_node_height; + Vector2(x, y) + snap_to_pixel_offset + } + + fn snap_to_pixel_offset(size: Vector2, scene_shape: &display::scene::Shape) -> Vector2 { + let device_size = scene_shape.device_pixels(); + let origin_left_top_pos = Vector2(device_size.width, device_size.height) / 2.0; + let origin_snapped = Vector2(origin_left_top_pos.x.floor(), origin_left_top_pos.y.floor()); + let origin_offset = origin_snapped - origin_left_top_pos; + let panel_left_top_pos = (size * scene_shape.pixel_ratio) / 2.0; + let panel_snapped = Vector2(panel_left_top_pos.x.floor(), panel_left_top_pos.y.floor()); + let panel_offset = panel_snapped - panel_left_top_pos; + origin_offset - panel_offset + } +} + +impl display::Object for Model { + fn display_object(&self) -> &display::object::Instance { + &self.display_object + } +} + +ensogl::define_endpoints_2! { + Input { + show(), + hide(), + } + Output { + is_visible(bool), + size(Vector2), + expression_input_position(Vector2), + is_hovered(bool), + editing_committed(), + is_empty(bool), + } +} + +impl component::Frp for Frp { + fn init( + network: &frp::Network, + frp_api: &::Private, + app: &Application, + model: &Model, + style: &StyleWatchFrp, + ) { + let scene = &app.display.default_scene; + let input = &frp_api.input; + let out = &frp_api.output; + let list_panel = &model.list.output; + let documentation = &model.documentation; + let grid = &model.list.model().grid; + + let gap = style.get_number(theme::panels_gap); + let doc_width = style.get_number(theme::documentation::width); + let doc_height = style.get_number(theme::documentation::height); + frp::extend! { network + init <- source_(); + + doc_size <- all_with3(&init, &doc_width, &doc_height, |_, w, h| Vector2(*w, *h)); + size <- all_with4(&init, &list_panel.size, &doc_size, &gap, |(), list_size, doc_size, gap| { + let width = list_size.x + gap + doc_size.x; + let height = max(list_size.y, doc_size.y); + Vector2(width, height) + }); + snap <- all_with(&size, &scene.frp.shape, |sz, sh| Model::snap_to_pixel_offset(*sz, sh)); + list_position_x <- + all_with3(&size, &list_panel.size, &snap, |sz, list_sz, snap| list_sz.x / 2.0 - sz.x / 2.0 + snap.x); + doc_position_x <- all_with3(&size, &doc_size, &snap, |sz, doc_sz, snap| sz.x / 2.0 - doc_sz.x / 2.0 + snap.x); + eval list_position_x ((x) model.list.set_x(*x)); + eval doc_position_x ((x) model.documentation.set_x(*x)); + + model.list.input.show <+ input.show; + model.list.input.hide <+ input.hide; + out.is_visible <+ bool(&input.hide, &input.show); + out.size <+ size; + out.expression_input_position <+ all_with3( + &size, + &gap, + &snap, + Model::expression_input_position + ); + + out.is_hovered <+ list_panel.is_hovered || documentation.frp.is_hovered; + out.is_hovered <+ input.hide.constant(false); + + out.editing_committed <+ grid.expression_accepted.constant(()); + } + init.emit(()); + } +} + +/// Component Browser View. +/// +/// The Component Browser is a panel where user searches for types, variables and methods which can +/// be used to construct new nodes. The components are arranged in sections and groups, and +/// displayed in Component List Panel. The Component Browser View contains also Documentation Panel, +/// displaying documentation of currently selected or hovered component. +pub type View = component::ComponentView; diff --git a/app/gui/view/documentation/Cargo.toml b/app/gui/view/documentation/Cargo.toml index bd3e5d35c1cc..ab1ba93e4386 100644 --- a/app/gui/view/documentation/Cargo.toml +++ b/app/gui/view/documentation/Cargo.toml @@ -8,8 +8,10 @@ edition = "2021" crate-type = ["cdylib", "rlib"] [dependencies] +enso-doc-parser = { path = "../../../../lib/rust/parser/doc-parser" } enso-frp = { path = "../../../../lib/rust/frp" } enso-prelude = { path = "../../../../lib/rust/prelude" } +enso-profiler = { path = "../../../../lib/rust/profiler" } ensogl = { path = "../../../../lib/rust/ensogl" } ensogl-component = { path = "../../../../lib/rust/ensogl/component" } ensogl-hardcoded-theme = { path = "../../../../lib/rust/ensogl/app/theme/hardcoded" } diff --git a/app/gui/view/documentation/src/html.rs b/app/gui/view/documentation/src/html.rs index 082fd5318375..4d28ce1de4b9 100644 --- a/app/gui/view/documentation/src/html.rs +++ b/app/gui/view/documentation/src/html.rs @@ -4,6 +4,11 @@ use enso_prelude::*; use horrorshow::prelude::*; use double_representation::name::QualifiedName; +use enso_doc_parser::DocSection; +use enso_doc_parser::Mark; +use enso_profiler as profiler; +use enso_profiler::profile; +use enso_suggestion_database::documentation_ir::BuiltinDocumentation; use enso_suggestion_database::documentation_ir::Constructors; use enso_suggestion_database::documentation_ir::Documentation; use enso_suggestion_database::documentation_ir::EntryDocumentation; @@ -17,8 +22,6 @@ use enso_suggestion_database::documentation_ir::Synopsis; use enso_suggestion_database::documentation_ir::Tag; use enso_suggestion_database::documentation_ir::TypeDocumentation; use enso_suggestion_database::documentation_ir::Types; -use enso_suggestion_database::engine_protocol::language_server::DocSection; -use enso_suggestion_database::engine_protocol::language_server::Mark; use enso_suggestion_database::entry::Argument; use horrorshow::box_html; use horrorshow::labels; @@ -58,6 +61,7 @@ fn svg_icon(content: &'static str) -> impl Render { // ============== /// Render entry documentation to HTML code with Tailwind CSS styles. +#[profile(Detail)] pub fn render(docs: EntryDocumentation) -> String { match docs { EntryDocumentation::Placeholder(placeholder) => match placeholder { @@ -66,7 +70,6 @@ pub fn render(docs: EntryDocumentation) -> String { render_virtual_component_group_docs(name), }, EntryDocumentation::Docs(docs) => render_documentation(docs), - EntryDocumentation::Builtin(docs) => render_builtin_docs(docs), } } @@ -82,6 +85,7 @@ fn render_documentation(docs: Documentation) -> String { render_type_documentation(&type_docs, Some(&name)), Documentation::ModuleMethod { module_docs, name } => render_module_documentation(&module_docs, Some(&name)), + Documentation::Builtin(builtin_docs) => render_builtin_documentation(&builtin_docs), } } @@ -95,13 +99,6 @@ fn render_virtual_component_group_docs(name: ImString) -> String { docs_content(content).into_string().unwrap() } -/// Render a documentation of a builtin entry that is not present in the suggestion database. -fn render_builtin_docs(html_docs: ImString) -> String { - let content = owned_html! { - : Raw(&*html_docs); - }; - docs_content(content).into_string().unwrap() -} // === Types === @@ -475,6 +472,20 @@ fn local_synopsis<'a>(synopsis: &'a Synopsis) -> Box { } +// === Builtin entries === + +/// Render documentation for built-in entries. +/// +/// Consists of only a synopsis. +fn render_builtin_documentation(docs: &BuiltinDocumentation) -> String { + let synopsis = section_content(module_synopsis(&docs.synopsis)); + let content = owned_html! { + : &synopsis; + }; + docs_content(content).into_string().unwrap() +} + + // ======================= // === Common elements === diff --git a/app/gui/view/examples/documentation/Cargo.toml b/app/gui/view/examples/documentation/Cargo.toml index 65fe66bb9a8e..85c823a6831f 100644 --- a/app/gui/view/examples/documentation/Cargo.toml +++ b/app/gui/view/examples/documentation/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" crate-type = ["cdylib", "rlib"] [dependencies] +enso-doc-parser = { path = "../../../../../lib/rust/parser/doc-parser" } ensogl = { path = "../../../../../lib/rust/ensogl" } ensogl-hardcoded-theme = { path = "../../../../../lib/rust/ensogl/app/theme/hardcoded" } enso-suggestion-database = { path = "../../../suggestion-database" } diff --git a/app/gui/view/examples/documentation/src/lib.rs b/app/gui/view/examples/documentation/src/lib.rs index 07bb4aea94ec..6c52b5f72301 100644 --- a/app/gui/view/examples/documentation/src/lib.rs +++ b/app/gui/view/examples/documentation/src/lib.rs @@ -10,11 +10,11 @@ use ensogl::display::shape::*; use ensogl::prelude::*; +use enso_doc_parser::DocSection; +use enso_doc_parser::Mark; use enso_suggestion_database as suggestion_database; use enso_suggestion_database::doc_section; use enso_suggestion_database::documentation_ir::EntryDocumentation; -use enso_suggestion_database::engine_protocol::language_server::DocSection; -use enso_suggestion_database::engine_protocol::language_server::Mark; use enso_suggestion_database::entry::Argument; use enso_suggestion_database::mock; use enso_suggestion_database::mock_suggestion_database; @@ -135,7 +135,7 @@ fn database() -> SuggestionDatabase { builder.add_function("bar", args, "Standard.Base.Boolean", scope.clone(), |e| { e.with_doc_sections(vec![ DocSection::Paragraph { body: "Documentation for the bar function.".into() }, - DocSection::Tag { name: "DEPRECATED".into(), body: default() }, + DocSection::Tag { name: "DEPRECATED", body: default() }, DocSection::Marked { mark: Mark::Example, header: None, @@ -147,7 +147,7 @@ fn database() -> SuggestionDatabase { builder.add_local("local1", "Standard.Base.Boolean", scope, |e| { e.with_doc_sections(vec![ DocSection::Paragraph { body: "Documentation for the local1 variable.".into() }, - DocSection::Tag { name: "SOMETAG".into(), body: default() }, + DocSection::Tag { name: "SOMETAG", body: default() }, ]) }); diff --git a/app/gui/view/src/component_browser.rs b/app/gui/view/src/component_browser.rs deleted file mode 100644 index 962d43f94281..000000000000 --- a/app/gui/view/src/component_browser.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! A module containing the Component Browser [`View`]. - -// TODO[ao]: This module should be extracted to a separate crate in the future; now it shares -// the documentation panel with old Node Searcher, that's why it must be in the view crate. - -use crate::prelude::*; - -use crate::documentation; -use crate::graph_editor::component::node::HEIGHT as NODE_HEIGHT; - -use enso_frp as frp; -use ensogl::application::frp::API; -use ensogl::application::Application; -use ensogl::display; -use ensogl::display::shape::StyleWatchFrp; -use ensogl_gui_component::component; -use ensogl_hardcoded_theme::application::component_browser as theme; - - -// ============== -// === Export === -// ============== - -pub use ide_view_component_browser::*; - - - -/// The Model of Component Browser View. -#[allow(missing_docs)] -#[derive(Clone, CloneRef, Debug)] -pub struct Model { - display_object: display::object::Instance, - pub list: component_list_panel::View, - pub documentation: documentation::View, -} - -impl component::Model for Model { - fn label() -> &'static str { - "ComponentBrowser" - } - - fn new(app: &Application) -> Self { - let display_object = display::object::Instance::new(); - let list = app.new_view::(); - let documentation = documentation::View::new(app); - app.display.default_scene.layers.node_searcher.add(&display_object); - display_object.add_child(&list); - display_object.add_child(&documentation); - - // We need to set the shapes order for the documentation panel overlay to be on top of the - // CB background; otherwise, the background would block some mouse events. The CB background - // is big because of the surrounding shadow and partially overlaps the left side of the - // documentation panel. - let scene = &app.display.default_scene; - shapes_order_dependencies! { - scene => { - component_list_panel::background -> documentation::overlay; - } - } - - Self { display_object, list, documentation } - } -} - -impl Model { - fn expression_input_position( - size: &Vector2, - vertical_gap: &f32, - snap_to_pixel_offset: &Vector2, - ) -> Vector2 { - let half_node_height = NODE_HEIGHT / 2.0; - let panel_left = -size.x / 2.0; - let panel_bottom = -size.y / 2.0; - let x = panel_left; - let y = panel_bottom - vertical_gap - half_node_height; - Vector2(x, y) + snap_to_pixel_offset - } - - fn snap_to_pixel_offset(size: Vector2, scene_shape: &display::scene::Shape) -> Vector2 { - let device_size = scene_shape.device_pixels(); - let origin_left_top_pos = Vector2(device_size.width, device_size.height) / 2.0; - let origin_snapped = Vector2(origin_left_top_pos.x.floor(), origin_left_top_pos.y.floor()); - let origin_offset = origin_snapped - origin_left_top_pos; - let panel_left_top_pos = (size * scene_shape.pixel_ratio) / 2.0; - let panel_snapped = Vector2(panel_left_top_pos.x.floor(), panel_left_top_pos.y.floor()); - let panel_offset = panel_snapped - panel_left_top_pos; - origin_offset - panel_offset - } -} - -impl display::Object for Model { - fn display_object(&self) -> &display::object::Instance { - &self.display_object - } -} - -ensogl::define_endpoints_2! { - Input { - show(), - hide(), - } - Output { - is_visible(bool), - size(Vector2), - expression_input_position(Vector2), - is_hovered(bool), - } -} - -impl component::Frp for Frp { - fn init( - network: &frp::Network, - frp_api: &::Private, - app: &Application, - model: &Model, - style: &StyleWatchFrp, - ) { - let scene = &app.display.default_scene; - let input = &frp_api.input; - let out = &frp_api.output; - let list_panel = &model.list.output; - let documentation = &model.documentation; - - let gap = style.get_number(theme::panels_gap); - let doc_width = style.get_number(theme::documentation::width); - let doc_height = style.get_number(theme::documentation::height); - frp::extend! { network - init <- source_(); - - doc_size <- all_with3(&init, &doc_width, &doc_height, |_, w, h| Vector2(*w, *h)); - size <- all_with4(&init, &list_panel.size, &doc_size, &gap, |(), list_size, doc_size, gap| { - let width = list_size.x + gap + doc_size.x; - let height = max(list_size.y, doc_size.y); - Vector2(width, height) - }); - snap <- all_with(&size, &scene.frp.shape, |sz, sh| Model::snap_to_pixel_offset(*sz, sh)); - list_position_x <- - all_with3(&size, &list_panel.size, &snap, |sz, list_sz, snap| list_sz.x / 2.0 - sz.x / 2.0 + snap.x); - doc_position_x <- all_with3(&size, &doc_size, &snap, |sz, doc_sz, snap| sz.x / 2.0 - doc_sz.x / 2.0 + snap.x); - eval list_position_x ((x) model.list.set_x(*x)); - eval doc_position_x ((x) model.documentation.set_x(*x)); - - model.list.input.show <+ input.show; - model.list.input.hide <+ input.hide; - out.is_visible <+ bool(&input.hide, &input.show); - out.size <+ size; - out.expression_input_position <+ all_with3( - &size, - &gap, - &snap, - Model::expression_input_position - ); - - out.is_hovered <+ list_panel.is_hovered || documentation.frp.is_hovered; - out.is_hovered <+ input.hide.constant(false); - } - init.emit(()); - } -} - -/// Component Browser View. -/// -/// The Component Browser is a panel where user searches for types, variables and methods which can -/// be used to construct new nodes. The components are arranged in sections and groups, and -/// displayed in Component List Panel. The Component Browser View contains also Documentation Panel, -/// displaying documentation of currently selected or hovered component. -pub type View = component::ComponentView; diff --git a/app/gui/view/src/lib.rs b/app/gui/view/src/lib.rs index 60eff01f1cd0..cb05d146633c 100644 --- a/app/gui/view/src/lib.rs +++ b/app/gui/view/src/lib.rs @@ -31,7 +31,6 @@ #[allow(clippy::option_map_unit_fn)] pub mod code_editor; -pub mod component_browser; pub mod debug_mode_popup; pub mod project; pub mod project_list; @@ -40,6 +39,7 @@ pub mod searcher; pub mod status_bar; pub mod window_control_buttons; +pub use ide_view_component_browser as component_browser; pub use ide_view_documentation as documentation; pub use ide_view_graph_editor as graph_editor; pub use welcome_screen; diff --git a/app/gui/view/src/project.rs b/app/gui/view/src/project.rs index 775e8e9c9895..319f8cf5e8dc 100644 --- a/app/gui/view/src/project.rs +++ b/app/gui/view/src/project.rs @@ -9,8 +9,6 @@ use crate::component_browser; use crate::component_browser::component_list_panel; use crate::debug_mode_popup; use crate::debug_mode_popup::DEBUG_MODE_SHORTCUT; -use crate::documentation; -use crate::graph_editor::component::node; use crate::graph_editor::component::node::Expression; use crate::graph_editor::component::visualization; use crate::graph_editor::GraphEditor; @@ -131,119 +129,6 @@ ensogl::define_endpoints! { // === Model === // ============= -/// Common FRP endpoints for both Searcher variants. See [`SearcherVariant`]. -#[derive(Clone, CloneRef, Debug)] -struct SearcherFrp { - editing_committed: frp::Stream, - is_visible: frp::Stream, - is_empty: frp::Stream, - is_hovered: frp::Stream, -} - -/// A structure containing the Searcher View: the old Node Searcher of a new Component Browser. -/// -/// As Component Browser is unstable, it is available only under the feature flag. Thus the Project -/// View must be able to handle any of those views. -#[allow(missing_docs)] -#[derive(Clone, CloneRef, Debug)] -pub enum SearcherVariant { - ComponentBrowser(component_browser::View), - /// We keep the old searcher in a Rc, as its memory size is much greater than the new one. - OldNodeSearcher(Rc), -} - -impl SearcherVariant { - fn new(app: &Application) -> Self { - if ARGS.groups.feature_preview.options.new_component_browser.value { - Self::ComponentBrowser(app.new_view::()) - } else { - Self::OldNodeSearcher(Rc::new(app.new_view::())) - } - } - - fn frp(&self, project_view_network: &frp::Network) -> SearcherFrp { - match self { - SearcherVariant::ComponentBrowser(view) => { - let grid = &view.model().list.model().grid; - frp::extend! {project_view_network - is_empty <- source::(); - editing_committed <- grid.expression_accepted.constant(()); - } - is_empty.emit(false); - SearcherFrp { - editing_committed, - is_visible: view.output.is_visible.clone_ref().into(), - is_empty: is_empty.into(), - is_hovered: view.output.is_hovered.clone_ref().into(), - } - } - SearcherVariant::OldNodeSearcher(view) => { - let documentation = view.documentation(); - frp::extend! {project_view_network - editing_committed <- view.editing_committed.constant(()); - } - SearcherFrp { - editing_committed, - is_visible: view.output.is_visible.clone_ref().into(), - is_empty: view.output.is_empty.clone_ref().into(), - is_hovered: documentation.frp.is_hovered.clone_ref().into(), - } - } - } - } - - fn documentation(&self) -> &documentation::View { - match self { - SearcherVariant::ComponentBrowser(view) => &view.model().documentation, - SearcherVariant::OldNodeSearcher(view) => view.documentation(), - } - } - - fn show(&self) { - match self { - SearcherVariant::ComponentBrowser(view) => view.show(), - SearcherVariant::OldNodeSearcher(view) => view.show(), - } - } - - fn hide(&self) { - match self { - SearcherVariant::ComponentBrowser(view) => view.hide(), - SearcherVariant::OldNodeSearcher(view) => view.hide(), - } - } - - fn setup_anchor(&self, network: &frp::Network, anchor: &frp::Stream>) { - match self { - SearcherVariant::ComponentBrowser(view) => { - frp::extend! {network - cb_position <- all_with(anchor, &view.expression_input_position, |anchor, pos| anchor - pos); - eval cb_position ((pos) view.set_xy(*pos)); - } - } - SearcherVariant::OldNodeSearcher(view) => { - frp::extend! {network - searcher_pos <- all_with(anchor, &view.size, |anchor, size| { - let x = anchor.x + size.x / 2.0; - let y = anchor.y - node::HEIGHT / 2.0 - size.y / 2.0; - Vector2(x, y) - }); - eval searcher_pos ((pos) view.set_xy(*pos)); - } - } - } - } -} - -impl display::Object for SearcherVariant { - fn display_object(&self) -> &display::object::Instance { - match self { - SearcherVariant::ComponentBrowser(view) => view.display_object(), - SearcherVariant::OldNodeSearcher(view) => view.display_object(), - } - } -} - #[derive(Clone, CloneRef, Debug)] struct Model { app: Application, @@ -251,7 +136,7 @@ struct Model { /// These buttons are present only in a cloud environment. window_control_buttons: Immutable>, graph_editor: Rc, - searcher: SearcherVariant, + searcher: component_browser::View, code_editor: code_editor::View, fullscreen_vis: Rc>>, project_list: Rc, @@ -262,7 +147,7 @@ impl Model { fn new(app: &Application) -> Self { let scene = &app.display.default_scene; let display_object = display::object::Instance::new(); - let searcher = SearcherVariant::new(app); + let searcher = app.new_view::(); let graph_editor = app.new_view::(); let code_editor = app.new_view::(); let fullscreen_vis = default(); @@ -471,7 +356,7 @@ impl View { let model = Model::new(app); let frp = Frp::new(); let network = &frp.network; - let searcher = &model.searcher.frp(network); + let searcher = &model.searcher.frp(); let graph = &model.graph_editor.frp; let project_list = &model.project_list; let searcher_anchor = DEPRECATED_Animation::>::new(network); @@ -555,24 +440,14 @@ impl View { frp.source.editing_aborted <+ aborted_in_searcher; } - match &model.searcher { - SearcherVariant::ComponentBrowser(browser) => { - let grid = &browser.model().list.model().grid; - frp::extend! { network - committed_in_browser <- grid.expression_accepted.map2(&last_searcher, |&entry, &s| (s.input, Some(entry))); - frp.source.editing_committed <+ committed_in_browser; - frp.source.editing_committed <+ finished_with_searcher.map(|id| (*id,None)); - } - } - SearcherVariant::OldNodeSearcher(searcher) => { - frp::extend! { network - committed_in_searcher <- searcher.editing_committed.map2(&last_searcher, |&entry, &s| (s.input, entry)); - frp.source.editing_committed_old_searcher <+ committed_in_searcher; - frp.source.editing_committed_old_searcher <+ finished_with_searcher.map(|id| (*id,None)); - } - } + let grid = &model.searcher.model().list.model().grid; + frp::extend! { network + committed_in_browser <- grid.expression_accepted.map2(&last_searcher, |&entry, &s| (s.input, Some(entry))); + frp.source.editing_committed <+ committed_in_browser; + frp.source.editing_committed <+ finished_with_searcher.map(|id| (*id,None)); } + let anchor = &searcher_anchor.value; frp::extend! { network committed_in_searcher_event <- searcher.editing_committed.constant(()); aborted_in_searcher_event <- aborted_in_searcher.constant(()); @@ -641,6 +516,8 @@ impl View { }) ); + cb_position <- all_with(anchor, &model.searcher.expression_input_position, |anchor, pos| anchor - pos); + eval cb_position ((pos) model.searcher.set_xy(*pos)); // === Project Dialog === @@ -669,7 +546,7 @@ impl View { // === Fullscreen Visualization === - // TODO[ao]: All DOM elements in visualizations ale displayed below canvas, because + // TODO[ao]: All DOM elements in visualizations are displayed below canvas, because // The mouse cursor must be displayed over them. But fullscreen visualization should // be displayed "above" nodes. The workaround is to hide whole graph editor except // fullscreen visualization, and bring it back when fullscreen is closed. @@ -688,7 +565,7 @@ impl View { // === Disabling Navigation === - let documentation = model.searcher.documentation(); + let documentation = &model.searcher.model().documentation; searcher_active <- searcher.is_hovered || documentation.frp.is_selected; disable_navigation <- searcher_active || frp.project_list_shown; graph.set_navigator_disabled <+ disable_navigation; @@ -706,7 +583,7 @@ impl View { model.debug_mode_popup.enabled <+ frp.enable_debug_mode; model.debug_mode_popup.disabled <+ frp.disable_debug_mode; } - model.searcher.setup_anchor(network, &searcher_anchor.value); + init.emit(()); Self { model, frp } @@ -718,7 +595,7 @@ impl View { } /// Searcher View. - pub fn searcher(&self) -> &SearcherVariant { + pub fn searcher(&self) -> &component_browser::View { &self.model.searcher } diff --git a/app/ide-desktop/lib/content-config/src/config.json b/app/ide-desktop/lib/content-config/src/config.json index 0a9ad320777e..e9d9bfbee26c 100644 --- a/app/ide-desktop/lib/content-config/src/config.json +++ b/app/ide-desktop/lib/content-config/src/config.json @@ -109,10 +109,6 @@ "featurePreview": { "description": "Options that enable experimental features that are not yet stable.", "options": { - "newComponentBrowser": { - "value": true, - "description": "Indicates whether the new component browser should be enabled." - }, "skipAndFreeze": { "value": false, "description": "Enable possibility to skip and freeze nodes.", diff --git a/build.sbt b/build.sbt index 445516f0a99d..acc7f31af2cf 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,6 @@ import sbt.complete.DefaultParsers._ import sbt.complete.Parser import sbt.nio.file.FileTreeView import sbt.internal.util.ManagedLogger -import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} import src.main.scala.licenses.{ DistributionDescription, SBTDistributionComponent @@ -222,10 +221,6 @@ ThisBuild / Test / testOptions ++= Tests.Argument(TestFrameworks.ScalaTest, "-u", junitDir) } -val jsSettings = Seq( - scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.ESModule) } -) - Compile / console / scalacOptions ~= (_ filterNot (_ == "-Xfatal-warnings")) // ============================================================================ @@ -255,16 +250,13 @@ lazy val enso = (project in file(".")) `json-rpc-server-test`, `json-rpc-server`, `language-server`, - `parser-service`, - `docs-generator`, `polyglot-api`, `project-manager`, - `syntax-definition`.jvm, + `syntax-definition`, `syntax-rust-definition`, `text-buffer`, - flexer.jvm, graph, - logger.jvm, + logger, pkg, cli, `task-progress-notifications`, @@ -295,7 +287,7 @@ lazy val enso = (project in file(".")) `library-manager`, `library-manager-test`, `connected-lock-manager`, - syntax.jvm, + syntax, testkit, `std-base`, `std-database`, @@ -479,7 +471,6 @@ val scalacheckVersion = "1.16.0" val scalacticVersion = "3.3.0-SNAP3" val scalaLoggingVersion = "3.9.4" val scalameterVersion = "0.19" -val scalatagsVersion = "0.11.1" val scalatestVersion = "3.3.0-SNAP3" val shapelessVersion = "2.4.0-M1" val slf4jVersion = "1.7.36" @@ -494,176 +485,44 @@ val netbeansApiVersion = "RELEASE140" // === Internal Libraries ===================================================== // ============================================================================ -lazy val logger = crossProject(JVMPlatform, JSPlatform) - .withoutSuffixFor(JVMPlatform) - .crossType(CrossType.Pure) - .in(file("lib/scala/logger")) +lazy val logger = (project in file("lib/scala/logger")) .settings( frgaalJavaCompilerSetting, version := "0.1", libraryDependencies ++= scalaCompiler ) - .jsSettings(jsSettings) - -lazy val flexer = crossProject(JVMPlatform, JSPlatform) - .withoutSuffixFor(JVMPlatform) - .crossType(CrossType.Pure) - .in(file("lib/scala/flexer")) - .dependsOn(logger) - .settings( - frgaalJavaCompilerSetting, - version := "0.1", - resolvers ++= Resolver.sonatypeOssRepos("releases"), - libraryDependencies ++= scalaCompiler ++ Seq( - "com.google.guava" % "guava" % guavaVersion exclude ("com.google.code.findbugs", "jsr305"), - "org.typelevel" %%% "cats-core" % catsVersion, - "org.typelevel" %%% "kittens" % kittensVersion - ) - ) - .jsSettings(jsSettings) -lazy val `syntax-definition` = crossProject(JVMPlatform, JSPlatform) - .withoutSuffixFor(JVMPlatform) - .crossType(CrossType.Pure) - .in(file("lib/scala/syntax/definition")) - .dependsOn(logger, flexer) - .settings( - scalacOptions ++= Seq("-Ypatmat-exhaust-depth", "off"), - libraryDependencies ++= monocle ++ scalaCompiler ++ Seq( - "org.typelevel" %%% "cats-core" % catsVersion, - "org.typelevel" %%% "kittens" % kittensVersion, - "com.lihaoyi" %%% "scalatags" % scalatagsVersion, - "io.circe" %%% "circe-core" % circeVersion, - "io.circe" %%% "circe-generic" % circeVersion, - "io.circe" %%% "circe-parser" % circeVersion +lazy val `syntax-definition` = + (project in file("lib/scala/syntax/definition")) + .dependsOn(logger) + .settings( + scalacOptions ++= Seq("-Ypatmat-exhaust-depth", "off"), + libraryDependencies ++= monocle ++ scalaCompiler ++ Seq( + "org.typelevel" %% "cats-core" % catsVersion, + "org.typelevel" %% "kittens" % kittensVersion + ) ) - ) - .jsSettings(jsSettings) -lazy val syntax = crossProject(JVMPlatform, JSPlatform) - .withoutSuffixFor(JVMPlatform) - .crossType(CrossType.Full) - .in(file("lib/scala/syntax/specialization")) - .dependsOn(logger, flexer, `syntax-definition`) - .configs(Test) - .configs(Benchmark) +lazy val syntax = (project in file("lib/scala/syntax/specialization")) + .dependsOn(`syntax-definition`) .settings( commands += WithDebugCommand.withDebug, - Test / fork := true, testFrameworks := Nil, scalacOptions ++= Seq("-Ypatmat-exhaust-depth", "off"), Compile / run / mainClass := Some("org.enso.syntax.text.Main"), version := "0.1", logBuffered := false, libraryDependencies ++= Seq( - "org.scalatest" %%% "scalatest" % scalatestVersion % Test, - "com.lihaoyi" %%% "pprint" % pprintVersion, - "io.circe" %%% "circe-core" % circeVersion, - "io.circe" %%% "circe-generic" % circeVersion, - "io.circe" %%% "circe-parser" % circeVersion + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "com.lihaoyi" %% "pprint" % pprintVersion, + "io.circe" %% "circe-core" % circeVersion, + "io.circe" %% "circe-generic" % circeVersion, + "io.circe" %% "circe-parser" % circeVersion ), (Compile / compile) := (Compile / compile) - .dependsOn(RecompileParser.run(`syntax-definition`)) - .value, - (Test / compile) := (Test / compile) - .dependsOn(RecompileParser.run(`syntax-definition`)) - .value, - (Benchmark / compile) := (Benchmark / compile) .dependsOn(RecompileParser.run(`syntax-definition`)) .value ) - .jvmSettings( - inConfig(Benchmark)(Defaults.testSettings), - Benchmark / unmanagedSourceDirectories += - baseDirectory.value.getParentFile / "shared/src/bench/scala", - libraryDependencies += - "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench", - testFrameworks := List( - new TestFramework("org.scalatest.tools.Framework"), - new TestFramework("org.scalameter.ScalaMeterFramework") - ), - bench := (Benchmark / test).tag(Exclusive).value - ) - .jsSettings( - scalaJSUseMainModuleInitializer := false, - scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.ESModule) }, - testFrameworks := List(new TestFramework("org.scalatest.tools.Framework")), - Compile / fullOptJS / artifactPath := file("target/scala-parser.js"), - libraryDependencies += - "org.scala-js" %%% "scalajs-java-securerandom" % "1.0.0" - ) - -lazy val `lexer-bench` = - (project in file("lib/scala/syntax/specialization/lexer-bench")) - .settings( - commands += WithDebugCommand.withDebug, - inConfig(Compile)(truffleRunOptionsSettings), - inConfig(Benchmark)(Defaults.testSettings), - Test / parallelExecution := false, - Test / logBuffered := false, - Test / fork := true, - libraryDependencies ++= jmh - ) - .configs(Test) - .configs(Benchmark) - .dependsOn(syntax.jvm) - .dependsOn(flexer.jvm) - .settings( - javaOptions ++= Seq( - "-Xms4096m", - "-Xmx4096m", - "-XX:+FlightRecorder" - ), - Benchmark / mainClass := Some("org.openjdk.jmh.Main"), - bench := Def.task { - (Benchmark / run).toTask("").value - }, - benchOnly := Def.inputTaskDyn { - import complete.Parsers.spaceDelimited - val name = spaceDelimited("").parsed match { - case List(name) => name - case _ => - throw new IllegalArgumentException("Expected one argument.") - } - Def.task { - (Benchmark / testOnly).toTask(" -- -z " + name).value - } - }.evaluated, - Benchmark / parallelExecution := false - ) - -lazy val `parser-service` = (project in file("lib/scala/parser-service")) - .dependsOn(syntax.jvm) - .settings( - libraryDependencies ++= akka ++ akkaTest, - mainClass := Some("org.enso.ParserServiceMain") - ) - -lazy val `docs-generator` = (project in file("lib/scala/docs-generator")) - .dependsOn(syntax.jvm) - .dependsOn(cli) - .dependsOn(`version-output`) - .dependsOn(`polyglot-api`) - .configs(Benchmark) - .settings( - frgaalJavaCompilerSetting, - libraryDependencies ++= Seq( - "commons-cli" % "commons-cli" % commonsCliVersion - ), - mainClass := Some("org.enso.docs.generator.Main"), - inConfig(Benchmark)(Defaults.testSettings), - Benchmark / unmanagedSourceDirectories += - baseDirectory.value.getParentFile / "bench" / "scala", - libraryDependencies ++= Seq( - "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench", - "org.scalatest" %% "scalatest" % scalatestVersion % Test - ), - testFrameworks := List( - new TestFramework("org.scalatest.tools.Framework"), - new TestFramework("org.scalameter.ScalaMeterFramework") - ), - bench := (Benchmark / test).tag(Exclusive).value - ) lazy val `text-buffer` = project .in(file("lib/scala/text-buffer")) @@ -777,7 +636,7 @@ lazy val `syntax-rust-definition` = project ) lazy val graph = (project in file("lib/scala/graph/")) - .dependsOn(logger.jvm) + .dependsOn(logger) .configs(Test) .settings( frgaalJavaCompilerSetting, @@ -871,8 +730,8 @@ lazy val `logging-service` = project "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, akkaStream, akkaHttp, - "io.circe" %%% "circe-core" % circeVersion, - "io.circe" %%% "circe-parser" % circeVersion, + "io.circe" %% "circe-core" % circeVersion, + "io.circe" %% "circe-parser" % circeVersion, "junit" % "junit" % junitVersion % Test, "com.novocode" % "junit-interface" % "0.11" % Test exclude ("junit", "junit-dep"), "org.scalatest" %% "scalatest" % scalatestVersion % Test, @@ -1253,7 +1112,6 @@ lazy val `language-server` = (project in file("engine/language-server")) .dependsOn(`text-buffer`) .dependsOn(`version-output`) .dependsOn(pkg) - .dependsOn(`docs-generator`) .dependsOn(`profiling-utils`) .dependsOn(testkit % Test) .dependsOn(`library-manager-test` % Test) @@ -1517,9 +1375,8 @@ lazy val runtime = (project in file("engine/runtime")) .dependsOn(pkg) .dependsOn(`edition-updater`) .dependsOn(`connected-lock-manager`) - .dependsOn(syntax.jvm) + .dependsOn(syntax) .dependsOn(`syntax-rust-definition`) - .dependsOn(`docs-generator`) .dependsOn(testkit % Test) lazy val `runtime-instrument-id-execution` = diff --git a/build/build/src/engine.rs b/build/build/src/engine.rs index d1a40c5e298d..a1e6fd5814d0 100644 --- a/build/build/src/engine.rs +++ b/build/build/src/engine.rs @@ -132,8 +132,6 @@ pub struct BuildConfigurationFlags { pub execute_benchmarks: BTreeSet, /// Used to check that benchmarks do not fail on runtime, rather than obtaining the results. pub execute_benchmarks_once: bool, - /// Whether the Scala-based parser should be compiled into JS. - pub build_js_parser: bool, pub build_engine_package: bool, pub build_launcher_package: bool, pub build_project_manager_package: bool, @@ -141,7 +139,6 @@ pub struct BuildConfigurationFlags { pub build_project_manager_bundle: bool, pub generate_java_from_rust: bool, pub test_java_generated_from_rust: bool, - pub generate_documentation: bool, /// Verify License Packages in Distributions. pub verify_packages: bool, } @@ -215,7 +212,6 @@ impl Default for BuildConfigurationFlags { check_enso_benchmarks: false, execute_benchmarks: default(), execute_benchmarks_once: false, - build_js_parser: false, build_engine_package: false, build_launcher_package: false, build_project_manager_package: false, @@ -223,7 +219,6 @@ impl Default for BuildConfigurationFlags { build_project_manager_bundle: false, generate_java_from_rust: true, test_java_generated_from_rust: false, - generate_documentation: false, verify_packages: false, } } diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index b022456f528e..4eeea5b49245 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -431,25 +431,6 @@ impl RunContext { perhaps_test_java_generated_from_rust_job.await.transpose()?; // === Build Distribution === - if self.config.generate_documentation { - // FIXME [mwu] - // docs-generator fails on Windows because it can't understand non-Unix-style paths. - if TARGET_OS != OS::Windows { - // Build the docs from standard library sources. - sbt.call_arg("docs-generator/run").await?; - } - } - - if self.config.build_js_parser { - // Build the Parser JS Bundle - sbt.call_arg("syntaxJS/fullOptJS").await?; - ide_ci::fs::copy_to( - &self.paths.repo_root.target.scala_parser_js, - &self.paths.repo_root.target.parser_upload, - )?; - } - - if self.config.test_standard_library { enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS).await?; } diff --git a/build/cli/src/lib.rs b/build/cli/src/lib.rs index bb6d6b658744..f881a1a02e8e 100644 --- a/build/cli/src/lib.rs +++ b/build/cli/src/lib.rs @@ -427,9 +427,7 @@ impl Processor { execute_benchmarks: once(Benchmarks::Runtime).collect(), execute_benchmarks_once: true, check_enso_benchmarks: true, - build_js_parser: matches!(TARGET_OS, OS::Linux), verify_packages: true, - generate_documentation: true, ..default() }; let context = self.prepare_backend_context(config); diff --git a/distribution/engine/THIRD-PARTY/NOTICE b/distribution/engine/THIRD-PARTY/NOTICE index 5d6645809cc0..9ea7650b13e5 100644 --- a/distribution/engine/THIRD-PARTY/NOTICE +++ b/distribution/engine/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'enumeratum-circe_2.13', licensed under the MIT, is distributed with the engine. The license information can be found along with the copyright notices. @@ -106,21 +106,11 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.lihaoyi.fansi_2.13-0.3.1`. -'geny_2.13', licensed under the MIT, is distributed with the engine. -The license information can be found along with the copyright notices. -Copyright notices related to this dependency can be found in the directory `com.lihaoyi.geny_2.13-0.6.10`. - - 'pprint_2.13', licensed under the MIT, is distributed with the engine. The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.lihaoyi.pprint_2.13-0.7.3`. -'scalatags_2.13', licensed under the MIT, is distributed with the engine. -The license information can be found along with the copyright notices. -Copyright notices related to this dependency can be found in the directory `com.lihaoyi.scalatags_2.13-0.11.1`. - - 'sourcecode_2.13', licensed under the MIT, is distributed with the engine. The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.lihaoyi.sourcecode_2.13-0.2.8`. @@ -267,7 +257,7 @@ Copyright notices related to this dependency can be found in the directory `io.c 'circe-yaml_2.13', licensed under the Apache 2.0, is distributed with the engine. -The license information can be found along with the copyright notices. +The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.circe.circe-yaml_2.13-0.14.1`. @@ -286,6 +276,11 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.spray.spray-json_2.13-1.3.6`. +'jna', licensed under the Apache License v2.0, is distributed with the engine. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `net.java.dev.jna.jna-5.5.0`. + + 'jna', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `net.java.dev.jna.jna-5.9.0`. diff --git a/distribution/engine/THIRD-PARTY/com.lihaoyi.geny_2.13-0.6.10/LICENSE b/distribution/engine/THIRD-PARTY/com.lihaoyi.geny_2.13-0.6.10/LICENSE deleted file mode 100644 index 1a2c5c062e55..000000000000 --- a/distribution/engine/THIRD-PARTY/com.lihaoyi.geny_2.13-0.6.10/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -License -======= - - -The MIT License (MIT) - -Copyright (c) 2016 Li Haoyi (haoyi.sg@gmail.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/LICENSE.txt b/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/LICENSE.txt deleted file mode 100644 index 2145e7996cd3..000000000000 --- a/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2018 Li Haoyi (haoyi.sg@gmail.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/NOTICES b/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/NOTICES deleted file mode 100644 index d881ec537678..000000000000 --- a/distribution/engine/THIRD-PARTY/com.lihaoyi.scalatags_2.13-0.11.1/NOTICES +++ /dev/null @@ -1 +0,0 @@ -Defines the footer for a page or section. It often contains a copyright diff --git a/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE b/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE deleted file mode 100644 index 7e7cb146f58d..000000000000 --- a/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE +++ /dev/null @@ -1,67 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and -You must cause any modified files to carry prominent notices stating that You changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS diff --git a/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES b/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES new file mode 100644 index 000000000000..bbf2d5f53dc8 --- /dev/null +++ b/distribution/engine/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES @@ -0,0 +1,16 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/distribution/engine/THIRD-PARTY/net.java.dev.jna.jna-5.5.0/NOTICES b/distribution/engine/THIRD-PARTY/net.java.dev.jna.jna-5.5.0/NOTICES new file mode 100644 index 000000000000..c89a59e31480 --- /dev/null +++ b/distribution/engine/THIRD-PARTY/net.java.dev.jna.jna-5.5.0/NOTICES @@ -0,0 +1,5 @@ +Copyright (c) 2007 Timothy Wall, All Rights Reserved + +Copyright (c) 2007 Wayne Meissner, All Rights Reserved + +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved diff --git a/distribution/launcher/THIRD-PARTY/NOTICE b/distribution/launcher/THIRD-PARTY/NOTICE index e46723d7687c..c9a028017002 100644 --- a/distribution/launcher/THIRD-PARTY/NOTICE +++ b/distribution/launcher/THIRD-PARTY/NOTICE @@ -102,7 +102,7 @@ Copyright notices related to this dependency can be found in the directory `io.c 'circe-yaml_2.13', licensed under the Apache 2.0, is distributed with the launcher. -The license information can be found along with the copyright notices. +The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.circe.circe-yaml_2.13-0.14.1`. diff --git a/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE b/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE deleted file mode 100644 index 7e7cb146f58d..000000000000 --- a/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE +++ /dev/null @@ -1,67 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and -You must cause any modified files to carry prominent notices stating that You changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS diff --git a/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES b/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES new file mode 100644 index 000000000000..bbf2d5f53dc8 --- /dev/null +++ b/distribution/launcher/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES @@ -0,0 +1,16 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE index f3c7d7830375..73d24445cf4f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'icu4j', licensed under the Unicode/ICU License, is distributed with the Base. The license information can be found along with the copyright notices. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index 4a31e64d9d4a..39529c3b9d91 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -22,7 +22,7 @@ import project.System.File_Format.File_Format from project.Data.Boolean import Boolean, True, False ## ALIAS Load, Open - Read's a file into Enso. + Reads a file into Enso. Uses the specified file format to parse the file into an Enso type. If not specified will use the file's extension to determine the file format. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE index 8c92e7e49474..3caa2f08e41c 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'redshift-jdbc42', licensed under the Apache License, Version 2.0, is distributed with the Database. The license information can be found along with the copyright notices. diff --git a/distribution/lib/Standard/Google_Api/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Google_Api/0.0.0-dev/THIRD-PARTY/NOTICE index 89dc5d86de8b..3f8bfdbdcac5 100644 --- a/distribution/lib/Standard/Google_Api/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Google_Api/0.0.0-dev/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'google-api-client', licensed under the The Apache Software License, Version 2.0, is distributed with the Google_Api. The license file can be found at `licenses/APACHE2.0`. diff --git a/distribution/lib/Standard/Image/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Image/0.0.0-dev/THIRD-PARTY/NOTICE index b0d2b75fb506..1469be8ddb86 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Image/0.0.0-dev/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'opencv', licensed under the BSD License, is distributed with the Image. The license file can be found at `licenses/BSD-3-Clause`. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE index e14dd46a6bda..a834c4dd9a27 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE @@ -1,5 +1,5 @@ Enso -Copyright 2020 - 2022 New Byte Order sp. z o. o. +Copyright 2020 - 2023 New Byte Order sp. z o. o. 'curvesapi', licensed under the BSD License, is distributed with the Table. The license file can be found at `licenses/BSD-3-Clause`. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso index 16d64492261e..029f9c235ca2 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso @@ -11,7 +11,7 @@ type Expression - expression: the expression to evaluate - get_column: a function that takes a column name and returns the associated Column object. - - make_constant`: a function that takes an object and returns a + - make_constant: a function that takes an object and returns a constant Column object. - module_name: the name of the Column module that the expression is being evaluated against. diff --git a/distribution/project-manager/THIRD-PARTY/NOTICE b/distribution/project-manager/THIRD-PARTY/NOTICE index 448851c35cc2..2b88e2320529 100644 --- a/distribution/project-manager/THIRD-PARTY/NOTICE +++ b/distribution/project-manager/THIRD-PARTY/NOTICE @@ -217,7 +217,7 @@ Copyright notices related to this dependency can be found in the directory `io.c 'circe-yaml_2.13', licensed under the Apache 2.0, is distributed with the project-manager. -The license information can be found along with the copyright notices. +The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.circe.circe-yaml_2.13-0.14.1`. diff --git a/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE b/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE deleted file mode 100644 index 7e7cb146f58d..000000000000 --- a/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/LICENSE +++ /dev/null @@ -1,67 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and -You must cause any modified files to carry prominent notices stating that You changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS diff --git a/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES b/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES new file mode 100644 index 000000000000..bbf2d5f53dc8 --- /dev/null +++ b/distribution/project-manager/THIRD-PARTY/io.circe.circe-yaml_2.13-0.14.1/NOTICES @@ -0,0 +1,16 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/docs/flexer/README.md b/docs/flexer/README.md deleted file mode 100644 index b69ee05fd4c8..000000000000 --- a/docs/flexer/README.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -layout: docs-index -title: Enso's Parser -category: summary -tags: [parser, readme] -order: 0 ---- - -# The Flexer - -The flexer is a finite-automaton-based engine for the generation of -highly-efficient lexers. It is capable of supporting the lexing of unrestricted -grammars as it provides the author with the ability to manipulate the lexing -state after every match. - -The various components of the flexer's design and architecture are described -below: - -- [**Overview of the Flexer:**](./flexer.md) A brief overview of the flexer, - including what it is and how it works. diff --git a/docs/flexer/flexer.md b/docs/flexer/flexer.md deleted file mode 100644 index 22509bd5f72c..000000000000 --- a/docs/flexer/flexer.md +++ /dev/null @@ -1,199 +0,0 @@ ---- -layout: developer-doc -title: Flexer -category: syntax -tags: [parser, flexer, lexer, dfa] -order: 1 ---- - -# Flexer - -The flexer is a finite-automata-based engine for the definition and generation -of lexers. Akin to `flex`, and other lexer generators, the user may use it to -define a series of rules for lexing their language, which are then used by the -flexer to generate a highly-efficient lexer implementation. - -Where the flexer differs from other programs in this space, however, is the -power that it gives users. When matching a rule, the flexer allows its users to -execute _arbitrary_ Rust code, which may even manipulate the lexer's state and -position. This means that the languages that can be lexed by the flexer extend -from the simplest regular grammars right up to unrestricted grammars (but please -don't write a programming language whose syntax falls into this category). It -also differs in that it chooses the first complete match for a rule, rather than -the longest one, which makes lexers much easier to define and maintain. - -For detailed library documentation, please see the -[crate documentation](../../lib/rust/flexer/src/lib.rs) itself. This includes a -comprehensive tutorial on how to define a lexer using the flexer. - - - -- [The Lexing Process](#the-lexing-process) -- [Lexing Rules](#lexing-rules) - - [Groups](#groups) - - [Patterns](#patterns) - - [Transition Functions](#transition-functions) -- [Code Generation](#code-generation) - - [Automated Code Generation](#automated-code-generation) -- [Structuring the Flexer Code](#structuring-the-flexer-code) - - [Supporting Code Generation](#supporting-code-generation) - - - -## The Lexing Process - -In the flexer, the lexing process proceeds from the top to the bottom of the -user-defined rules, and selects the first expression that _matches fully_. Once -a pattern has been matched against the input, the associated code is executed -and the process starts again until the input stream has been consumed. - -This point about _matching fully_ is particularly important to keep in mind, as -it differs from other lexer generators that tend to prefer the _longest_ match -instead. - -## Lexing Rules - -A lexing rule for the flexer is a combination of three things: - -1. A group. -2. A pattern. -3. A transition function. - -An example of defining a rule is as follows: - -```rust -fn define() -> Self { - let mut lexer = TestLexer::new(); - let a_word = Pattern::char('a').many1(); - let root_group_id = lexer.initial_state; - let root_group = lexer.groups_mut().group_mut(root_group_id); - // Here is the rule definition. - root_group.create_rule(&a_word,"self.on_first_word(reader)"); - lexer -} -``` - -### Groups - -A group is a mechanism that the flexer provides to allow grouping of rules -together. The flexer has a concept of a "state stack", which records the -currently active state at the current time, that can be manipulated by the -user-defined [transition functions](#transition-functions). - -A state can be made active by using `flexer::push_state(state)`, and can be -deactivated by using `flexer::pop_state(state)` or -`flexer::pop_states_until(state)`. In addition, states may also have _parents_, -from which they can inherit rules. This is fantastic for removing the need to -repeat yourself when defining the lexer. - -When inheriting rules from a parent group, the rules from the parent group are -matched strictly _after_ the rules from the child group. This means that groups -are able to selectively "override" the rules of their parents. Rules are still -matched in order for each group's set of rules. - -### Patterns - -Rules are defined to match _patterns_. Patterns are regular-grammar-like -descriptions of the textual content (as characters) that should be matched. For -a description of the various patterns provided by the flexer, see -[pattern.rs](../../lib/rust/flexer/src/automata/pattern.rs). - -When a pattern is matched, the associated -[transition function](#transition-functions) is executed. - -### Transition Functions - -The transition function is a piece of arbitrary rust code that is executed when -the pattern for a given rule is matched by the flexer. This code may perform -arbitrary manipulations of the lexer state, and is where the majority of the -power of the flexer stems from. - -## Code Generation - -While it would be possible to interpret the flexer definition directly at -runtime, this would involve far too much dynamicism and non-cache-local lookup -to be at all fast. - -Instead, the flexer includes -[`generate.rs`](../../lib/rust/flexer/src/generate.rs), a library for generating -highly-specialized lexer implementations based on the definition provided by the -user. The transformation that it implements operates as follows for each group -of rules. - -1. The set of rules in a group is used to generate a - [Nondeterministic Finite Automaton](https://en.wikipedia.org/wiki/Nondeterministic_finite_automaton), - (NFA). -2. The NFA is ttransformed into a - [Deterministic Finite Automaton](https://en.wikipedia.org/wiki/Deterministic_finite_automaton) - (DFA), using a variant of the standard - [powerset construction](https://en.wikipedia.org/wiki/Powerset_construction) - algorithm. This variant has been modified to ensure that the following - additional properties hold: - - Patterns are matched in the order in which they are defined. - - The associated transition functions are maintained correctly through the - transformation. - - The lexing process is `O(n)`, where `n` is the size of the input. -3. The DFA is then used to generate the rust code that implements that lexer. - -The generated lexer contains a main loop that consumes the input stream -character-by-character, evaluating what is effectively a big `match` expression -that processes the input to evaluate the user-provided transition functions as -appropriate. - -### Automated Code Generation - -In order to avoid the lexer definition getting out of sync with its -implementation (the generated engine), it is necessary to create a separate -crate for the generated engine that has the lexer definition as one of its -dependencies. - -This separation enables a call to `flexer::State::specialize()` in the crate's -`build.rs` (or a macro) during compilation. The output can be stored in a new -file i.e. `engine.rs` and exported from the library as needed. The project -structure would therefore appear as follows. - -``` -- lib/rust/lexer/ - - definition/ - - src/ - - lib.rs - - cargo.toml - - - generation/ - - src/ - - engine.rs <-- the generated file - - lib.rs <-- `pub mod engine` - - build.rs <-- calls `flexer::State::specialize()` and saves its output to - `src/engine.rs` - - cargo.toml <-- lexer-definition is in dependencies and build-dependencies -``` - -With this design, `flexer.generate_specialized_code()` is going to be executed -on each rebuild of `lexer/generation`. Therefore, `generation` should contain -only the minimum amount of logic, and should endeavor to minimize any -unnecessary dependencies to avoid recompiling too often. - -## Structuring the Flexer Code - -In order to unify the API between the definition and generated usages of the -flexer, the API is separated into the following components: - -- `Flexer`: The main flexer definition itself, providing functionality common to - the definition and implementation of all lexers. -- `flexer::State`: The stateful components of a lexer definition. This trait is - implemented for a particular lexer definition, allowing the user to store - arbitrary data in their lexer, as needed. -- **User-Defined Lexer:** The user can then define a lexer that _wraps_ the - flexer, specialised to the particular `flexer::State` that the user has - defined. It is recommended to implement `Deref` and `DerefMut` between the - defined lexer and the `Flexer`, to allow for ease of use. - -### Supporting Code Generation - -This architecture separates out the generated code (which can be defined purely -on the user-defined lexer), from the code that is defined as part of the lexer -definition. This means that the same underlying structures can be used to both -_define_ the lexer, and be used by the generated code from that definition. - -For an example of how these components are used in the generated lexer, please -see [`generated_api_test`](../../lib/rust/flexer/tests/generated_api_test.rs). diff --git a/docs/language-server/protocol-language-server.md b/docs/language-server/protocol-language-server.md index b4dcaa58a3d2..7d015ec42474 100644 --- a/docs/language-server/protocol-language-server.md +++ b/docs/language-server/protocol-language-server.md @@ -505,12 +505,6 @@ interface Module { /** The fully qualified module name re-exporting this module. */ reexport?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } interface Type { @@ -534,12 +528,6 @@ interface Type { /** The documentation string. */ documentation?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } interface Constructor { @@ -563,12 +551,6 @@ interface Constructor { /** The documentation string. */ documentation?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } interface Method { @@ -598,12 +580,6 @@ interface Method { /** The documentation string. */ documentation?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } interface Function { @@ -627,12 +603,6 @@ interface Function { /** The documentation string. */ documentation?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } interface Local { @@ -653,12 +623,6 @@ interface Local { /** The documentation string. */ documentation?: string; - - /** The rendered HTML of the documentation string. */ - documentationHtml?: string; - - /** The documentation string divided into sections. */ - documentationSections?: DocSection[]; } ``` @@ -689,119 +653,6 @@ The suggestion entry id of the suggestions database. type SuggestionId = number; ``` -### `DocSection` - -A single section of the documentation. - -#### Format - -```typescript -type DocSection = Tag | Paragraph | Keyed | Marked; - -/** The documentation tag. - * - * {{{ - * name text - * }}} - * - * @example - * - * {{{ - * UNSTABLE - * DEPRECATED - * ALIAS Length - * }}} - * - */ -interface Tag { - /** The tag name. */ - name: string; - - /** The tag text. */ - body: HTMLString; -} - -/** The paragraph of the text. - * - * @example - * - * {{{ - * Arbitrary text in the documentation comment. - * - * This is another paragraph. - * }}} - * - */ -interface Paragraph { - /** The elements that make up this paragraph. */ - body: HTMLString; -} - -/** The section that starts with the key followed by the colon and the body. - * - * {{{ - * key: body - * }}} - * - * @example - * - * {{{ - * Arguments: - * - one: the first - * - two: the second - * }}} - * - * - * {{{ - * Icon: table-from-rows - * }}} - * - */ -interface Keyed { - /** The section key. */ - key: string; - - /** The elements that make up the body of the section. */ - body: HTMLString; -} - -/** The section that starts with the mark followed by the header and the body. - * - * {{{ - * mark header - * body - * }}} - * - * @example - * - * {{{ - * > Example - * This is how it's done. - * foo = bar baz - * }}} - * - * {{{ - * ! Notice - * This is important. - * }}} - */ -interface Marked { - /** The section mark. */ - mark: Mark; - - /** The section header. */ - header?: string; - - /** The elements that make up the body of the section. */ - body: HTMLString; -} - -/** Text rendered as HTML (may contain HTML tags). */ -type HTMLString = string; - -type Mark = "Important" | "Info" | "Example"; -``` - ### `SuggestionsDatabaseEntry` The entry in the suggestions database. @@ -980,11 +831,6 @@ interface Modify { */ documentation?: FieldUpdate; - /** - * New documentation sections. - */ - documentationSections?: FieldUpdate; - /** * The scope to update. */ diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/SearchProtocol.scala b/engine/language-server/src/main/scala/org/enso/languageserver/search/SearchProtocol.scala index 5d80626e009e..1eebd7fe9323 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/SearchProtocol.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/search/SearchProtocol.scala @@ -6,7 +6,7 @@ import io.circe.syntax._ import io.circe.{Decoder, Encoder, Json} import org.enso.languageserver.filemanager.{FileSystemFailure, Path} import org.enso.pkg.QualifiedName -import org.enso.polyglot.{DocSection, Suggestion} +import org.enso.polyglot.Suggestion import org.enso.searcher.SuggestionEntry import org.enso.text.editing.model.Position @@ -32,10 +32,6 @@ object SearchProtocol { val Documentation = "documentation" - val DocumentationHtml = "documentationHtml" - - val DocumentationSections = "documentationSections" - val Reexport = "reexport" } @@ -63,87 +59,6 @@ object SearchProtocol { val Local = "local" } - object DocSectionType { - - val Tag = "tag" - - val Paragraph = "paragraph" - - val Keyed = "keyed" - - val Marked = "marked" - } - - object DocSectionMarkType { - - val Important = "Important" - - val Info = "Info" - - val Example = "Example" - } - - implicit val docSectionEncoder: Encoder[DocSection] = - Encoder.instance[DocSection] { - case tag: DocSection.Tag => - Encoder[DocSection.Tag] - .apply(tag) - .deepMerge(Json.obj(CodecField.Type -> DocSectionType.Tag.asJson)) - - case paragraph: DocSection.Paragraph => - Encoder[DocSection.Paragraph] - .apply(paragraph) - .deepMerge( - Json.obj(CodecField.Type -> DocSectionType.Paragraph.asJson) - ) - - case keyed: DocSection.Keyed => - Encoder[DocSection.Keyed] - .apply(keyed) - .deepMerge(Json.obj(CodecField.Type -> DocSectionType.Keyed.asJson)) - - case marked: DocSection.Marked => - Encoder[DocSection.Marked] - .apply(marked) - .deepMerge(Json.obj(CodecField.Type -> DocSectionType.Marked.asJson)) - } - - implicit val docSectionDecoder: Decoder[DocSection] = - Decoder.instance { cursor => - cursor.downField(CodecField.Type).as[String].flatMap { - case DocSectionType.Tag => - Decoder[DocSection.Tag].tryDecode(cursor) - - case DocSectionType.Paragraph => - Decoder[DocSection.Paragraph].tryDecode(cursor) - - case DocSectionType.Keyed => - Decoder[DocSection.Keyed].tryDecode(cursor) - - case DocSectionType.Marked => - Decoder[DocSection.Marked].tryDecode(cursor) - } - } - - implicit val docSectionMarkEncoder: Encoder[DocSection.Mark] = - Encoder.instance { - case _: DocSection.Mark.Important => - DocSectionMarkType.Important.asJson - case _: DocSection.Mark.Info => - DocSectionMarkType.Info.asJson - case _: DocSection.Mark.Example => - DocSectionMarkType.Example.asJson - } - - implicit val docSectionMarkDecoder: Decoder[DocSection.Mark] = - Decoder.instance { cursor => - cursor.as[String].map { - case DocSectionMarkType.Important => DocSection.Mark.Important() - case DocSectionMarkType.Info => DocSection.Mark.Info() - case DocSectionMarkType.Example => DocSection.Mark.Example() - } - } - implicit val suggestionEncoder: Encoder[Suggestion] = Encoder.instance[Suggestion] { case module: Suggestion.Module => @@ -220,8 +135,6 @@ object SearchProtocol { conversion.returnType, isStatic = false, conversion.documentation, - conversion.documentationHtml, - None, conversion.reexport ) } @@ -241,12 +154,6 @@ object SearchProtocol { documentation <- cursor .downField(CodecField.Documentation) .as[Option[String]] - documentationHtml <- cursor - .downField(CodecField.DocumentationHtml) - .as[Option[String]] - documentationSections <- cursor - .downField(CodecField.DocumentationSections) - .as[Option[List[DocSection]]] reexport <- cursor.downField(CodecField.Reexport).as[Option[String]] } yield { val returnType = @@ -259,8 +166,6 @@ object SearchProtocol { returnType, parentType, documentation, - documentationHtml, - documentationSections, reexport ) } @@ -407,23 +312,19 @@ object SearchProtocol { * @param selfType the self type to update * @param returnType the return type to update * @param documentation the documentation string to update - * @param documentationHtml the HTML documentation to update - * @param documentationSections the documentation sections to update * @param scope the scope to update * @param reexport the module reexporting the suggestion */ case class Modify( id: SuggestionId, - externalId: Option[FieldUpdate[Suggestion.ExternalId]] = None, - arguments: Option[Seq[SuggestionArgumentUpdate]] = None, - module: Option[FieldUpdate[String]] = None, - selfType: Option[FieldUpdate[String]] = None, - returnType: Option[FieldUpdate[String]] = None, - documentation: Option[FieldUpdate[String]] = None, - documentationHtml: Option[FieldUpdate[String]] = None, - documentationSections: Option[FieldUpdate[Seq[DocSection]]] = None, - scope: Option[FieldUpdate[Suggestion.Scope]] = None, - reexport: Option[FieldUpdate[String]] = None + externalId: Option[FieldUpdate[Suggestion.ExternalId]] = None, + arguments: Option[Seq[SuggestionArgumentUpdate]] = None, + module: Option[FieldUpdate[String]] = None, + selfType: Option[FieldUpdate[String]] = None, + returnType: Option[FieldUpdate[String]] = None, + documentation: Option[FieldUpdate[String]] = None, + scope: Option[FieldUpdate[Suggestion.Scope]] = None, + reexport: Option[FieldUpdate[String]] = None ) extends SuggestionsDatabaseUpdate implicit val decoder: Decoder[SuggestionsDatabaseUpdate] = diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala index c4bdd467855b..040cc5c33ed3 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala @@ -5,7 +5,6 @@ import java.util.concurrent.Executors import akka.actor.{Actor, ActorRef, Props, Stash, Status} import akka.pattern.pipe import com.typesafe.scalalogging.LazyLogging -import org.enso.docs.sections.DocSectionsBuilder import org.enso.languageserver.capability.CapabilityProtocol.{ AcquireCapability, CapabilityAcquired, @@ -82,7 +81,6 @@ import scala.util.{Failure, Success} * @param versionsRepo the versions repo * @param sessionRouter the session router * @param runtimeConnector the runtime connector - * @param docSectionsBuilder the builder of documentation sections */ final class SuggestionsHandler( config: Config, @@ -90,8 +88,7 @@ final class SuggestionsHandler( suggestionsRepo: SuggestionsRepo[Future], versionsRepo: VersionsRepo[Future], sessionRouter: ActorRef, - runtimeConnector: ActorRef, - docSectionsBuilder: DocSectionsBuilder + runtimeConnector: ActorRef ) extends Actor with Stash with LazyLogging @@ -357,11 +354,7 @@ final class SuggestionsHandler( .map { case (version, entries) => GetSuggestionsDatabaseResult( version, - entries.map { entry => - SuggestionDatabaseEntry( - entry.copy(suggestion = generateDocumentation(entry.suggestion)) - ) - } + entries.map { entry => SuggestionDatabaseEntry(entry) } ) } .pipeTo(sender()) @@ -580,7 +573,7 @@ final class SuggestionsHandler( ids.map( SuggestionsDatabaseUpdate.Add( _, - generateDocumentation(suggestion) + suggestion ) ) case action => @@ -628,7 +621,7 @@ final class SuggestionsHandler( ids.map( SuggestionsDatabaseUpdate.Add( _, - generateDocumentation(suggestion) + suggestion ) ) case Api.SuggestionAction.Remove() => @@ -644,10 +637,7 @@ final class SuggestionsHandler( arguments = m.arguments.map(_.map(toApiArgumentAction)), returnType = m.returnType.map(fieldUpdate), scope = m.scope.map(fieldUpdate), - documentation = m.documentation.map(fieldUpdateOption), - documentationSections = m.documentation.map( - fieldUpdateMapOption(docSectionsBuilder.build) - ) + documentation = m.documentation.map(fieldUpdateOption) ) } } @@ -688,20 +678,6 @@ final class SuggestionsHandler( case None => FieldUpdate(FieldAction.Remove, None) } - /** Construct the field update object from an optional value. - * - * @param f the mapping function - * @param value the optional value - * @return the field update object representing the value update - */ - private def fieldUpdateMapOption[A, B]( - f: A => B - )(value: Option[A]): FieldUpdate[B] = - value match { - case Some(value) => FieldUpdate(FieldAction.Set, Some(f(value))) - case None => FieldUpdate(FieldAction.Remove, None) - } - /** Construct the field update object from and update value. * * @param value the update value @@ -771,43 +747,6 @@ final class SuggestionsHandler( */ private def toPosition(pos: Position): Suggestion.Position = Suggestion.Position(pos.line, pos.character) - - /** Generate the documentation for the given suggestion. - * - * @param suggestion the initial suggestion - * @return the suggestion with documentation fields set - */ - private def generateDocumentation(suggestion: Suggestion): Suggestion = - suggestion match { - case module: Suggestion.Module => - val docSections = module.documentation.map(docSectionsBuilder.build) - module.copy(documentationSections = docSections) - - case constructor: Suggestion.Constructor => - val docSections = - constructor.documentation.map(docSectionsBuilder.build) - constructor.copy(documentationSections = docSections) - - case tpe: Suggestion.Type => - val docSections = tpe.documentation.map(docSectionsBuilder.build) - tpe.copy(documentationSections = docSections) - - case method: Suggestion.Method => - val docSections = method.documentation.map(docSectionsBuilder.build) - method.copy(documentationSections = docSections) - - case conversion: Suggestion.Conversion => - val docSections = conversion.documentation.map(docSectionsBuilder.build) - conversion.copy(documentationSections = docSections) - - case function: Suggestion.Function => - val docSections = function.documentation.map(docSectionsBuilder.build) - function.copy(documentationSections = docSections) - - case local: Suggestion.Local => - val docSections = local.documentation.map(docSectionsBuilder.build) - local.copy(documentationSections = docSections) - } } object SuggestionsHandler { @@ -898,7 +837,6 @@ object SuggestionsHandler { * @param versionsRepo the versions repo * @param sessionRouter the session router * @param runtimeConnector the runtime connector - * @param docSectionsBuilder the builder of documentation sections */ def props( config: Config, @@ -906,8 +844,7 @@ object SuggestionsHandler { suggestionsRepo: SuggestionsRepo[Future], versionsRepo: VersionsRepo[Future], sessionRouter: ActorRef, - runtimeConnector: ActorRef, - docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder() + runtimeConnector: ActorRef ): Props = Props( new SuggestionsHandler( @@ -916,8 +853,7 @@ object SuggestionsHandler { suggestionsRepo, versionsRepo, sessionRouter, - runtimeConnector, - docSectionsBuilder + runtimeConnector ) ) } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala index 9fcc9843762d..7d3261a12ab5 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala @@ -1,7 +1,5 @@ package org.enso.languageserver.search -import org.enso.docs.generator.DocsGenerator -import org.enso.docs.sections.DocSectionsBuilder import org.enso.polyglot.Suggestion import java.util.UUID @@ -26,39 +24,28 @@ object Suggestions { |""".stripMargin.linesIterator.mkString("\n") } - val htmlDocsGenerator: DocsGenerator = - DocsGenerator - val docSectionsBuilder: DocSectionsBuilder = - DocSectionsBuilder() - val module: Suggestion.Module = Suggestion.Module( - module = "local.Test.Main", - documentation = Some("Module doc"), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build("Module doc")) + module = "local.Test.Main", + documentation = Some("Module doc") ) val tpe: Suggestion.Type = Suggestion.Type( - externalId = None, - module = "local.Test.Main", - name = "Newtype", - params = Vector(Suggestion.Argument("a", "Any", false, false, None)), - returnType = "Newtype", - parentType = Some("Any"), - documentation = None, - documentationHtml = None, - documentationSections = None + externalId = None, + module = "local.Test.Main", + name = "Newtype", + params = Vector(Suggestion.Argument("a", "Any", false, false, None)), + returnType = "Newtype", + parentType = Some("Any"), + documentation = None ) val constructor: Suggestion.Constructor = Suggestion.Constructor( - externalId = None, - module = "local.Test.Main", - name = "MyType", - arguments = Vector(Suggestion.Argument("a", "Any", false, false, None)), - returnType = "MyAtom", - documentation = Some(comment.atom), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build(comment.atom)) + externalId = None, + module = "local.Test.Main", + name = "MyType", + arguments = Vector(Suggestion.Argument("a", "Any", false, false, None)), + returnType = "MyAtom", + documentation = Some(comment.atom) ) val method: Suggestion.Method = Suggestion.Method( @@ -69,12 +56,10 @@ object Suggestions { Suggestion.Argument("this", "MyType", false, false, None), Suggestion.Argument("foo", "Number", false, true, Some("42")) ), - selfType = "MyType", - returnType = "Number", - isStatic = false, - documentation = Some("Lovely"), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build("Lovely")) + selfType = "MyType", + returnType = "Number", + isStatic = false, + documentation = Some("Lovely") ) val function: Suggestion.Function = Suggestion.Function( @@ -89,9 +74,7 @@ object Suggestions { returnType = "IO", scope = Suggestion.Scope(Suggestion.Position(1, 9), Suggestion.Position(1, 22)), - documentation = Some("My Function"), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build("My Function")) + documentation = Some("My Function") ) val local: Suggestion.Local = Suggestion.Local( @@ -112,12 +95,10 @@ object Suggestions { Suggestion.Argument("this", "Any", false, false, None), Suggestion.Argument("that", "Any", false, false, None) ), - selfType = "Any", - returnType = "Any", - isStatic = false, - documentation = Some("Lovely"), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build("Lovely")) + selfType = "Any", + returnType = "Any", + isStatic = false, + documentation = Some("Lovely") ) val methodOnNumber: Suggestion.Method = Suggestion.Method( @@ -127,12 +108,10 @@ object Suggestions { arguments = Vector( Suggestion.Argument("this", "Number", false, false, None) ), - selfType = "Number", - returnType = "Number", - isStatic = false, - documentation = None, - documentationHtml = None, - documentationSections = None + selfType = "Number", + returnType = "Number", + isStatic = false, + documentation = None ) val methodOnInteger: Suggestion.Method = Suggestion.Method( @@ -142,12 +121,10 @@ object Suggestions { arguments = Vector( Suggestion.Argument("that", "Number", false, false, None) ), - selfType = "Integer", - returnType = "Number", - isStatic = false, - documentation = Some("Blah, blah"), - documentationHtml = None, - documentationSections = Some(docSectionsBuilder.build("Blah, blah")) + selfType = "Integer", + returnType = "Number", + isStatic = false, + documentation = Some("Blah, blah") ) val all = Seq( diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala index 6f235df00a5b..6b2ab795d088 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala @@ -3,8 +3,6 @@ package org.enso.languageserver.search import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{ImplicitSender, TestKit, TestProbe} import org.apache.commons.io.FileUtils -import org.enso.docs.generator.DocsGenerator -import org.enso.docs.sections.DocSectionsBuilder import org.enso.languageserver.boot.ProfilingConfig import org.enso.languageserver.capability.CapabilityProtocol.{ AcquireCapability, @@ -364,20 +362,16 @@ class SuggestionsHandlerSpec val moduleName = "Test.Foo" val fooAtom = Suggestion.Constructor( - externalId = None, - module = moduleName, - name = "Foo", - arguments = Vector(), - returnType = moduleName, - documentation = None, - documentationHtml = None, - documentationSections = None + externalId = None, + module = moduleName, + name = "Foo", + arguments = Vector(), + returnType = moduleName, + documentation = None ) val module = Suggestion.Module( - module = moduleName, - documentation = None, - documentationHtml = None, - documentationSections = None + module = moduleName, + documentation = None ) val tree = Tree.Root( @@ -1149,11 +1143,6 @@ class SuggestionsHandlerSpec object TestSuggestion { - val htmlDocsGenerator: DocsGenerator = - DocsGenerator - val docSectionsBuilder: DocSectionsBuilder = - DocSectionsBuilder() - val atom: Suggestion.Constructor = Suggestion.Constructor( externalId = None, @@ -1163,24 +1152,20 @@ class SuggestionsHandlerSpec Suggestion.Argument("a", "Any", false, false, None), Suggestion.Argument("b", "Any", false, false, None) ), - returnType = "Pair", - documentation = Some("Awesome"), - documentationHtml = Some(htmlDocsGenerator.generate("Awesome", "Pair")), - documentationSections = Some(docSectionsBuilder.build("Awesome")) + returnType = "Pair", + documentation = Some("Awesome") ) val method: Suggestion.Method = Suggestion.Method( - externalId = Some(UUID.randomUUID()), - module = "Test.Main", - name = "main", - arguments = Seq(), - selfType = "Test.Main", - returnType = "IO", - isStatic = true, - documentation = None, - documentationHtml = None, - documentationSections = None + externalId = Some(UUID.randomUUID()), + module = "Test.Main", + name = "main", + arguments = Seq(), + selfType = "Test.Main", + returnType = "IO", + isStatic = true, + documentation = None ) } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala index 4fa272830570..85056674b3f2 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala @@ -114,29 +114,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { } ], "returnType" : "MyAtom", - "documentation" : " PRIVATE\n\n A key-value store. This type assumes all keys are pairwise comparable,\n using the `<`, `>` and `==` operators.\n\n Arguments:\n - one: The first.\n - two_three: The *second*.\n\n ? Info\n Here is a thing.", - "documentationSections" : [ - { - "type" : "tag", - "name" : "PRIVATE", - "body" : "" - }, - { - "type" : "paragraph", - "body" : "A key-value store. This type assumes all keys are pairwise comparable, using the <, > and == operators. " - }, - { - "type" : "keyed", - "key" : "Arguments", - "body" : "
  • one: The first.
  • two_three: The second.
" - }, - { - "type" : "marked", - "mark" : "Info", - "header" : "Info", - "body" : " Here is a thing." - } - ] + "documentation" : " PRIVATE\n\n A key-value store. This type assumes all keys are pairwise comparable,\n using the `<`, `>` and `==` operators.\n\n Arguments:\n - one: The first.\n - two_three: The *second*.\n\n ? Info\n Here is a thing." } } ], @@ -207,13 +185,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { "selfType" : "MyType", "returnType" : "Number", "isStatic" : false, - "documentation" : "Lovely", - "documentationSections" : [ - { - "type" : "paragraph", - "body" : "Lovely" - } - ] + "documentation" : "Lovely" } } ], @@ -308,13 +280,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { "character" : 22 } }, - "documentation" : "My Function", - "documentationSections" : [ - { - "type" : "paragraph", - "body" : "My Function" - } - ] + "documentation" : "My Function" } } ], @@ -442,29 +408,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { } ], "returnType" : "MyAtom", - "documentation" : " PRIVATE\n\n A key-value store. This type assumes all keys are pairwise comparable,\n using the `<`, `>` and `==` operators.\n\n Arguments:\n - one: The first.\n - two_three: The *second*.\n\n ? Info\n Here is a thing.", - "documentationSections" : [ - { - "type" : "tag", - "name" : "PRIVATE", - "body" : "" - }, - { - "type" : "paragraph", - "body" : "A key-value store. This type assumes all keys are pairwise comparable, using the <, > and == operators. " - }, - { - "type" : "keyed", - "key" : "Arguments", - "body" : "
  • one: The first.
  • two_three: The second.
" - }, - { - "type" : "marked", - "mark" : "Info", - "header" : "Info", - "body" : " Here is a thing." - } - ] + "documentation" : " PRIVATE\n\n A key-value store. This type assumes all keys are pairwise comparable,\n using the `<`, `>` and `==` operators.\n\n Arguments:\n - one: The first.\n - two_three: The *second*.\n\n ? Info\n Here is a thing." } }, { @@ -511,13 +455,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { "character" : 22 } }, - "documentation" : "My Function", - "documentationSections" : [ - { - "type" : "paragraph", - "body" : "My Function" - } - ] + "documentation" : "My Function" } }, { @@ -548,13 +486,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { "selfType" : "MyType", "returnType" : "Number", "isStatic" : false, - "documentation" : "Lovely", - "documentationSections" : [ - { - "type" : "paragraph", - "body" : "Lovely" - } - ] + "documentation" : "Lovely" } }, { diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala index 42b0cf13ac77..aa47d780c0df 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala @@ -171,16 +171,12 @@ object Suggestion { * * @param module the fully qualified module name * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections * @param reexport the module re-exporting this module */ case class Module( module: String, documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None, - reexport: Option[String] = None + reexport: Option[String] = None ) extends Suggestion with ToLogString { @@ -209,8 +205,6 @@ object Suggestion { * @param returnType the type of an atom * @param parentType qualified name of the parent type * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections * @param reexport the module re-exporting this atom */ case class Type( @@ -221,9 +215,7 @@ object Suggestion { returnType: String, parentType: Option[String], documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None, - reexport: Option[String] = None + reexport: Option[String] = None ) extends Suggestion with ToLogString { @@ -249,8 +241,6 @@ object Suggestion { * @param arguments the list of arguments * @param returnType the type of an atom * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections * @param reexport the module re-exporting this atom */ case class Constructor( @@ -260,9 +250,7 @@ object Suggestion { arguments: Seq[Argument], returnType: String, documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None, - reexport: Option[String] = None + reexport: Option[String] = None ) extends Suggestion with ToLogString { @@ -289,8 +277,6 @@ object Suggestion { * @param returnType the return type of a method * @param isStatic the flag indicating whether a method is static or instance * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections * @param reexport the module re-exporting this method */ case class Method( @@ -302,9 +288,7 @@ object Suggestion { returnType: String, isStatic: Boolean, documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None, - reexport: Option[String] = None + reexport: Option[String] = None ) extends Suggestion with ToLogString { @@ -330,8 +314,6 @@ object Suggestion { * @param sourceType the source type of a conversion * @param returnType the return type of a conversion * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections * @param reexport the module re-exporting this conversion */ case class Conversion( @@ -341,9 +323,7 @@ object Suggestion { sourceType: String, returnType: String, documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None, - reexport: Option[String] = None + reexport: Option[String] = None ) extends Suggestion { /** @inheritdoc */ @@ -371,8 +351,6 @@ object Suggestion { * @param returnType the return type of a function * @param scope the scope where the function is defined * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections */ case class Function( externalId: Option[ExternalId], @@ -381,9 +359,7 @@ object Suggestion { arguments: Seq[Argument], returnType: String, scope: Scope, - documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None + documentation: Option[String] ) extends Suggestion with ToLogString { @@ -408,8 +384,6 @@ object Suggestion { * @param returnType the type of a local value * @param scope the scope where the value is defined * @param documentation the documentation string - * @param documentationHtml the documentation rendered as HTML - * @param documentationSections the documentation parsed into sections */ case class Local( externalId: Option[ExternalId], @@ -417,9 +391,7 @@ object Suggestion { name: String, returnType: String, scope: Scope, - documentation: Option[String], - documentationHtml: Option[String] = None, - documentationSections: Option[List[DocSection]] = None + documentation: Option[String] ) extends Suggestion { /** @inheritdoc */ diff --git a/engine/runner/src/main/resources/META-INF/native-image/org/enso/runner/serialization-config.json b/engine/runner/src/main/resources/META-INF/native-image/org/enso/runner/serialization-config.json index 4fce99722907..cafbbaf9d5ea 100644 --- a/engine/runner/src/main/resources/META-INF/native-image/org/enso/runner/serialization-config.json +++ b/engine/runner/src/main/resources/META-INF/native-image/org/enso/runner/serialization-config.json @@ -284,27 +284,9 @@ { "name":"org.enso.pkg.QualifiedName" }, - { - "name":"org.enso.syntax.text.AST$ASTOf" - }, { "name":"org.enso.syntax.text.Location" }, - { - "name":"org.enso.syntax.text.Shape$Cons" - }, - { - "name":"org.enso.syntax.text.Shape$Opr" - }, - { - "name":"org.enso.syntax.text.Shape$Unexpected" - }, - { - "name":"org.enso.syntax.text.Shape$Var" - }, - { - "name":"org.enso.syntax.text.ast.opr.Assoc$Right$" - }, { "name":"scala.None$" }, diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala index 0f01a89ef3f4..981b57f7e6f7 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala @@ -2,7 +2,6 @@ package org.enso.interpreter.test.instrument import org.enso.distribution.FileSystem import org.enso.distribution.locking.ThreadSafeFileLockManager -import org.enso.docs.generator.DocsGenerator import org.enso.interpreter.test.Metadata import org.enso.pkg.{Package, PackageManager, QualifiedName} import org.enso.polyglot._ @@ -42,8 +41,6 @@ class RuntimeStdlibTest class TestContext(packageName: String) { - val docsGenerator = new DocsGenerator - val messageQueue: LinkedBlockingQueue[Api.Response] = new LinkedBlockingQueue() @@ -312,30 +309,6 @@ class RuntimeStdlibTest } contentRootNotifications should contain theSameElementsAs expectedLibraries - // check documentation generation - failAfter(30.seconds) { - responses.collect { - case Api.Response( - None, - Api.SuggestionsDatabaseModuleUpdateNotification( - _, - _, - _, - _, - updates - ) - ) => - updates.toVector.foreach { update => - update.suggestion.documentation.foreach { documentation => - context.docsGenerator.generate( - documentation, - update.suggestion.name - ) - } - } - } - } - context.consumeOut shouldEqual List("Hello World!") } diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala index 28133fbda313..290ecad15cea 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala @@ -154,8 +154,6 @@ class RuntimeSuggestionUpdatesTest Api.SuggestionUpdate( Suggestion.Module( moduleName, - None, - None, None ), Api.SuggestionAction.Add() @@ -172,8 +170,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Add() @@ -232,8 +228,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Modify() @@ -315,8 +309,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Modify() @@ -419,8 +411,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Modify() @@ -533,8 +523,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Modify() @@ -611,8 +599,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Add() @@ -686,8 +672,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Modify( @@ -788,8 +772,6 @@ class RuntimeSuggestionUpdatesTest Api.SuggestionUpdate( Suggestion.Module( moduleName, - None, - None, None ), Api.SuggestionAction.Add() @@ -806,8 +788,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Add() @@ -852,8 +832,6 @@ class RuntimeSuggestionUpdatesTest ConstantsGen.TEXT, ConstantsGen.ANY, false, - None, - None, None ), Api.SuggestionAction.Add() @@ -880,8 +858,6 @@ class RuntimeSuggestionUpdatesTest ConstantsGen.NUMBER, ConstantsGen.ANY, false, - None, - None, None ), Api.SuggestionAction.Add() @@ -974,7 +950,7 @@ class RuntimeSuggestionUpdatesTest Vector( Tree.Node( Api.SuggestionUpdate( - Suggestion.Module("Enso_Test.Test.A", None, None, None), + Suggestion.Module("Enso_Test.Test.A", None), Api.SuggestionAction.Add() ), Vector() @@ -988,8 +964,6 @@ class RuntimeSuggestionUpdatesTest List(), "Enso_Test.Test.A.MyType", Some(ConstantsGen.ANY), - None, - None, None ), Api.SuggestionAction.Add() @@ -1007,8 +981,6 @@ class RuntimeSuggestionUpdatesTest .Argument("a", ConstantsGen.ANY, false, false, None) ), "Enso_Test.Test.A.MyType", - None, - None, None ), Api.SuggestionAction.Add() @@ -1034,8 +1006,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.A.MyType", ConstantsGen.ANY, false, - None, - None, None ), Api.SuggestionAction.Add() @@ -1060,8 +1030,6 @@ class RuntimeSuggestionUpdatesTest ConstantsGen.INTEGER, ConstantsGen.ANY, false, - None, - None, None ), Api.SuggestionAction.Add() @@ -1086,8 +1054,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.A", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Add() @@ -1121,8 +1087,6 @@ class RuntimeSuggestionUpdatesTest Api.SuggestionUpdate( Suggestion.Module( moduleName, - None, - None, None ), Api.SuggestionAction.Add() @@ -1139,8 +1103,6 @@ class RuntimeSuggestionUpdatesTest "Enso_Test.Test.Main", ConstantsGen.ANY, true, - None, - None, None ), Api.SuggestionAction.Add() diff --git a/engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java b/engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java index 81f6f7c31fa0..4f67c9c75c47 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java +++ b/engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java @@ -14,7 +14,7 @@ public EnsoCompiler() { p = Parser.create(); } catch (LinkageError err) { err.printStackTrace(); - p = null; + throw err; } this.parser = p; } @@ -31,19 +31,19 @@ public IR.Module compile(CharSequence src) { return generateIR(tree); } - boolean isReady() { - return parser != null; - } - IR.Module compile(Source src) { var tree = parse(src); return generateIR(tree); } - Tree parse(Source src) { + public Tree parse(Source src) { return parser.parse(src.getCharacters()); } + public Tree parse(CharSequence src) { + return parser.parse(src); + } + public IR.Module generateIR(Tree t) { return TreeToIr.MODULE.translate(t); } diff --git a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java index e97a7ee3420f..ee28a5d39f36 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java +++ b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java @@ -272,7 +272,7 @@ IR.Module translateModule(Tree module) { var error = translateSyntaxError(inputAst, new IR$Error$Syntax$InvalidForeignDefinition(message)); yield cons(error, appendTo); } - var text = buildTextConstant(body, body.getElements(), true); + var text = buildTextConstant(body, body.getElements()); var def = new IR$Foreign$Definition(language, text, getIdentifiedLocation(fn.getBody()), meta(), diag()); var binding = new IR$Module$Scope$Definition$Method$Binding( methodRef, args, def, getIdentifiedLocation(inputAst), meta(), diag() @@ -417,7 +417,7 @@ private List translateTypeBodyExpressionImpl(Tree exp, List appendTo) th var error = translateSyntaxError(inputAst, new IR$Error$Syntax$InvalidForeignDefinition(message)); yield cons(error, appendTo); } - var text = buildTextConstant(body, body.getElements(), true); + var text = buildTextConstant(body, body.getElements()); var def = new IR$Foreign$Definition(language, text, getIdentifiedLocation(fn.getBody()), meta(), diag()); var binding = new IR$Function$Binding(name, args, def, getIdentifiedLocation(fn), true, meta(), diag()); yield cons(binding, appendTo); @@ -746,8 +746,7 @@ yield switch (op.codeRepr()) { ); var loc = getIdentifiedLocation(app); if (lhs == null && rhs == null) { - // AstToIr doesn't emit a location in this case. - yield new IR$Application$Operator$Section$Sides(name, Option.empty(), meta(), diag()); + yield new IR$Application$Operator$Section$Sides(name, loc, meta(), diag()); } else if (lhs == null) { yield new IR$Application$Operator$Section$Right(name, rhs, loc, meta(), diag()); } else if (rhs == null) { @@ -1206,37 +1205,16 @@ IR.Expression translateNumber(Tree.Number ast) { } IR.Literal translateLiteral(Tree.TextLiteral txt) throws SyntaxException { - var stripComments = txt.getOpen().codeRepr().length() > 1; // Splices are not yet supported in the IR. - var value = buildTextConstant(txt, txt.getElements(), stripComments); + var value = buildTextConstant(txt, txt.getElements()); return new IR$Literal$Text(value, getIdentifiedLocation(txt), meta(), diag()); } - String buildTextConstant(Tree at, Iterable elements, boolean stripComments) throws SyntaxException { + String buildTextConstant(Tree at, Iterable elements) throws SyntaxException { var sb = new StringBuilder(); TextElement error = null; for (var t : elements) { switch (t) { - case TextElement.Section s -> { - var text = s.getText().codeRepr(); - if (stripComments) { - // Reproduce an AstToIr bug for testing. - var quotedSegments = text.split("\"", -1); - for (int i = 0; i < quotedSegments.length; i++) { - var seg = quotedSegments[i]; - if (i % 2 == 0) { - sb.append(seg.replaceAll("#[^\n\r]*", "")); - } else { - sb.append('"'); - sb.append(seg); - if (i + 1 < quotedSegments.length) { - sb.append('"'); - } - } - } - } else { - sb.append(text); - } - } + case TextElement.Section s -> sb.append(s.getText().codeRepr()); case TextElement.Escape e -> { var val = e.getToken().getValue(); if (val == -1) { @@ -1592,21 +1570,17 @@ hidingNames, getIdentifiedLocation(exp), false, * @return the [[IR]] representation of `comment` */ IR$Comment$Documentation translateComment(Tree where, DocComment doc) throws SyntaxException { - var text = buildTextConstant(where, doc.getElements(), true); + var text = buildTextConstant(where, doc.getElements()); return new IR$Comment$Documentation(text, getIdentifiedLocation(where), meta(), diag()); } IR$Error$Syntax translateSyntaxError(Tree where, IR$Error$Syntax$Reason reason) { - var at = getIdentifiedLocation(where); - if (at.isEmpty()) { - return new IR$Error$Syntax(where, reason, meta(), diag()); - } else { - return new IR$Error$Syntax(at.get(), reason, meta(), diag()).setLocation(at); - } + var at = getIdentifiedLocation(where).get(); + return new IR$Error$Syntax(at, reason, meta(), diag()); } IR$Error$Syntax translateSyntaxError(IdentifiedLocation where, IR$Error$Syntax$Reason reason) { - return new IR$Error$Syntax(where, reason, meta(), diag()); + return new IR$Error$Syntax(where, reason, meta(), diag()); } SyntaxException translateEntity(Tree where, String msg) throws SyntaxException { diff --git a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala index 7a1450da0e04..d8596055ca7f 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala @@ -2,10 +2,9 @@ package org.enso.compiler import com.oracle.truffle.api.TruffleLogger import com.oracle.truffle.api.source.Source -import org.enso.compiler.codegen.{AstToIr, IrToTruffle, RuntimeStubsGenerator} +import org.enso.compiler.codegen.{IrToTruffle, RuntimeStubsGenerator} import org.enso.compiler.context.{FreshNameSupply, InlineContext, ModuleContext} import org.enso.compiler.core.IR -import org.enso.compiler.core.IR.Expression import org.enso.compiler.data.{BindingsMap, CompilerConfig} import org.enso.compiler.exception.{CompilationAbortedException, CompilerError} @@ -24,7 +23,7 @@ import org.enso.interpreter.runtime.{EnsoContext, Module} import org.enso.pkg.QualifiedName import org.enso.polyglot.{LanguageInfo, RuntimeOptions} import org.enso.syntax.text.Parser.IDMap -import org.enso.syntax.text.{AST, Parser} +import org.enso.syntax.text.Parser import org.enso.syntax2.Tree import java.io.{PrintStream, StringReader} @@ -541,21 +540,9 @@ class Compiler( isGeneratingDocs = isGenDocs ) - val src = module.getSource - def oldParser() = { - System.err.println("Using old parser to process " + src.getURI()) - val tree = parse(src) - generateIR(tree) - } - val expr = - if ( - !"scala".equals(System.getenv("ENSO_PARSER")) && ensoCompiler.isReady() - ) { - val tree = ensoCompiler.parse(src) - ensoCompiler.generateIR(tree) - } else { - oldParser() - } + val src = module.getSource + val tree = ensoCompiler.parse(src) + val expr = ensoCompiler.generateIR(tree) val exprWithModuleExports = if (module.isSynthetic) @@ -671,14 +658,6 @@ class Compiler( module.getScope } - /** Parses the provided language sources. - * - * @param source the code to parse - * @return an AST representation of `source` - */ - def parse(source: Source): AST = - Parser().runWithIds(source.getCharacters.toString) - /** Parses the given source with the new Rust parser. * * @param source The inline code to parse @@ -694,15 +673,6 @@ class Compiler( def parseMeta(source: CharSequence): IDMap = Parser().splitMeta(source.toString)._2 - /** Lowers the input AST to the compiler's high-level intermediate - * representation. - * - * @param sourceAST the parser AST input - * @return an IR representation of the program represented by `sourceAST` - */ - def generateIR(sourceAST: AST): IR.Module = - AstToIr.translate(sourceAST) - /** Enhances the provided IR with import/export statements for the provided list * of fully qualified names of modules. The statements are considered to be "synthetic" i.e. compiler-generated. * That way one can access modules using fully qualified names. @@ -785,15 +755,6 @@ class Compiler( ) } - /** Lowers the input AST to the compiler's high-level intermediate - * representation. - * - * @param sourceAST the parser AST representing the program source - * @return an IR representation of the program represented by `sourceAST` - */ - def generateIRInline(sourceAST: AST): Option[Expression] = - AstToIr.translateInline(sourceAST) - /** Runs the various compiler passes. * * @param ir the compiler intermediate representation to transform diff --git a/engine/runtime/src/main/scala/org/enso/compiler/Passes.scala b/engine/runtime/src/main/scala/org/enso/compiler/Passes.scala index ba6303b44b87..06124e672ec3 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/Passes.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/Passes.scala @@ -36,7 +36,6 @@ class Passes( FunctionBinding, GenerateMethodBodies, BindingAnalysis, - GenerateDocumentation, ModuleNameConflicts ) ) diff --git a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala index 5a52fb828bb9..ad0dd80376e3 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala @@ -5,7 +5,6 @@ import com.oracle.truffle.api.source.Source import org.enso.compiler.context.SuggestionBuilder import org.enso.compiler.core.IR import org.enso.compiler.pass.analyse.BindingAnalysis -import org.enso.docs.sections.DocSectionsBuilder import org.enso.editions.LibraryName import org.enso.interpreter.runtime.Module import org.enso.pkg.QualifiedName @@ -28,8 +27,7 @@ import scala.collection.mutable import scala.jdk.OptionConverters.RichOptional final class SerializationManager( - compiler: Compiler, - docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder() + compiler: Compiler ) { import SerializationManager._ @@ -238,7 +236,6 @@ final class SerializationManager( .build(module.getName, module.getIr) .toVector } - .map(generateDocumentation) .foreach(suggestions.add) val cachedSuggestions = new SuggestionsCache.CachedSuggestions( @@ -617,43 +614,6 @@ final class SerializationManager( ) .asScala } - - /** Generate the documentation for the given suggestion. - * - * @param suggestion the initial suggestion - * @return the suggestion with documentation fields set - */ - private def generateDocumentation(suggestion: Suggestion): Suggestion = - suggestion match { - case module: Suggestion.Module => - val docSections = module.documentation.map(docSectionsBuilder.build) - module.copy(documentationSections = docSections) - - case constructor: Suggestion.Constructor => - val docSections = - constructor.documentation.map(docSectionsBuilder.build) - constructor.copy(documentationSections = docSections) - - case tpe: Suggestion.Type => - val docSections = tpe.documentation.map(docSectionsBuilder.build) - tpe.copy(documentationSections = docSections) - - case method: Suggestion.Method => - val docSections = method.documentation.map(docSectionsBuilder.build) - method.copy(documentationSections = docSections) - - case conversion: Suggestion.Conversion => - val docSections = conversion.documentation.map(docSectionsBuilder.build) - conversion.copy(documentationSections = docSections) - - case function: Suggestion.Function => - val docSections = function.documentation.map(docSectionsBuilder.build) - function.copy(documentationSections = docSections) - - case local: Suggestion.Local => - val docSections = local.documentation.map(docSectionsBuilder.build) - local.copy(documentationSections = docSections) - } } object SerializationManager { diff --git a/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstToIr.scala b/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstToIr.scala deleted file mode 100644 index 1b5b23707562..000000000000 --- a/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstToIr.scala +++ /dev/null @@ -1,1298 +0,0 @@ -package org.enso.compiler.codegen - -import java.math.BigInteger -import java.nio.ByteBuffer - -import cats.Foldable -import cats.implicits._ -import org.enso.interpreter.Constants -import org.enso.compiler.core.IR -import org.enso.compiler.core.IR.Name.MethodReference -import org.enso.compiler.core.IR._ -import org.enso.compiler.exception.UnhandledEntity -import org.enso.interpreter.epb.EpbParser -import org.enso.syntax.text.AST -import org.enso.syntax.text.Shape.{ - SegmentEscape, - SegmentExpr, - SegmentPlain, - SegmentRawEscape, - TextUnclosed -} -import org.enso.syntax.text.ast.text.Escape -import org.enso.syntax.text.ast.text.Escape.Unicode - -import scala.annotation.tailrec -import scala.util.control.Breaks.{break, breakable} - -/** This file contains the functionality that translates from the parser's - * [[AST]] type to the internal representation used by the compiler. - * - * The current internal representation is [[IR]]. - */ -object AstToIr { - private def getIdentifiedLocation(ast: AST): Option[IdentifiedLocation] = - ast.location.map(IdentifiedLocation(_, ast.id)) - - /** Translates a program represented in the parser [[AST]] to the compiler's - * [[IR]]. - * - * @param inputAST the [[AST]] representing the program to translate - * @return the [[IR]] representation of `inputAST` - */ - def translate(inputAST: AST): Module = - inputAST match { - case AST.Module.any(inputAST) => translateModule(inputAST) - case _ => - throw new UnhandledEntity(inputAST, "translate") - } - - /** Translates an inline program expression represented in the parser [[AST]] - * into the compiler's [[IR]] representation. - * - * Inline expressions must _only_ be expressions, and may not contain any - * type of definition. - * - * @param inputAST the [[AST]] representing the expression to translate. - * @return the [[IR]] representation of `inputAST` if it is valid, otherwise - * [[None]] - */ - def translateInline(inputAST: AST): Option[Expression] = { - inputAST match { - case AST.Module.any(module) => - val presentBlocks = module.lines.collect { - case t if t.elem.isDefined => t.elem.get - } - - val expressions = presentBlocks.map(translateExpression(_)) - - expressions match { - case List() => None - case List(expr) => Some(expr) - case _ => - val locations = expressions.map(_.location.map(_.location)) - val locationsSum = Foldable[List].fold(locations) - Some( - Expression.Block( - expressions.dropRight(1), - expressions.last, - location = locationsSum.map(IdentifiedLocation(_)) - ) - ) - } - case _ => None - } - } - - /** Translate a top-level Enso module into [[IR]]. - * - * @param module the [[AST]] representation of the module to translate - * @return the [[IR]] representation of `module` - */ - def translateModule(module: AST.Module): Module = { - module match { - case AST.Module(blocks) => - val presentBlocks = blocks.collect { - case t if t.elem.isDefined => t.elem.get - } - - val imports = presentBlocks.collect { - case AST.Import.any(list) => translateImport(list) - case AST.JavaImport.any(imp) => - val pkg = imp.path.init.map(_.name) - val cls = imp.path.last.name - val rename = imp.rename.map(_.name) - Module.Scope.Import.Polyglot( - Module.Scope.Import.Polyglot.Java(pkg.mkString("."), cls), - rename, - getIdentifiedLocation(imp) - ) - } - - val exports = presentBlocks.collect { case AST.Export.any(export) => - translateExport(export) - } - - val nonImportBlocks = presentBlocks.filter { - case AST.Import.any(_) => false - case AST.JavaImport.any(_) => false - case AST.Export.any(_) => false - case _ => true - } - - val statements = nonImportBlocks.map(translateModuleSymbol) - Module(imports, exports, statements, getIdentifiedLocation(module)) - case _ => throw new UnhandledEntity(module, "translateModule") - } - } - - /** Translates a module-level definition from its [[AST]] representation into - * [[IR]]. - * - * @param inputAst the definition to be translated - * @return the [[IR]] representation of `inputAST` - */ - def translateModuleSymbol(inputAst: AST): Module.Scope.Definition = { - inputAst match { - case AST.Ident.Annotation.any(annotation) => - IR.Name.BuiltinAnnotation( - annotation.name, - getIdentifiedLocation(annotation) - ) - case AstView.Atom(consName, args) => - val newArgs = args.map(translateArgumentDefinition(_)) - - if (newArgs.exists(_.suspended)) { - val ast = newArgs - .zip(args) - .collect { case (arg, ast) if arg.suspended => ast } - .head - Error.Syntax(ast, Error.Syntax.SuspendedArgInAtom) - } else { - Module.Scope.Definition.SugaredType( - buildName(consName), - newArgs, - List(), - getIdentifiedLocation(inputAst) - ) - } - case AstView.TypeDef(typeName, args, body) => - val translatedBody = translateTypeBody(body) - Module.Scope.Definition.SugaredType( - buildName(typeName), - args.map(translateArgumentDefinition(_)), - translatedBody, - getIdentifiedLocation(inputAst) - ) - - case AstView.MethodDefinition(targetPath, name, args, definition) => - val nameId: AST.Ident = name match { - case AST.Ident.Var.any(name) => name - case AST.Ident.Opr.any(opr) => opr - case _ => - throw new UnhandledEntity(name, "translateModuleSymbol") - } - - val methodRef = if (targetPath.nonEmpty) { - val pathSegments = targetPath.collect { case AST.Ident.Cons.any(c) => - c - } - val pathNames = pathSegments.map(buildName(_)) - - val methodSegments = pathNames :+ buildName(nameId) - - val typeSegments = methodSegments.init - - Name.MethodReference( - Some( - IR.Name.Qualified( - typeSegments, - MethodReference.genLocation(typeSegments) - ) - ), - methodSegments.last, - MethodReference.genLocation(methodSegments) - ) - } else { - val methodName = buildName(nameId) - Name.MethodReference( - None, - methodName, - methodName.location - ) - } - - Module.Scope.Definition.Method.Binding( - methodRef, - args.map(translateArgumentDefinition(_)), - translateExpression(definition), - getIdentifiedLocation(inputAst) - ) - case AstView.FunctionSugar( - AST.Ident.Var("foreign"), - header, - body - ) => - translateForeignDefinition(header, body) match { - case Right((name, arguments, body)) => - val methodRef = - Name.MethodReference(None, name, name.location) - Module.Scope.Definition.Method.Binding( - methodRef, - arguments, - body, - getIdentifiedLocation(inputAst) - ) - case Left(reason) => - IR.Error.Syntax( - inputAst, - IR.Error.Syntax.InvalidForeignDefinition(reason) - ) - } - case AstView.FunctionSugar(name, args, body) => - val methodName = buildName(name) - - val methodReference = Name.MethodReference( - None, - methodName, - methodName.location - ) - - Module.Scope.Definition.Method.Binding( - methodReference, - args.map(translateArgumentDefinition(_)), - translateExpression(body), - getIdentifiedLocation(inputAst) - ) - case AST.Comment.any(comment) => translateComment(comment) - case AstView.TypeAscription(typed, sig) => - def buildAscription(ident: AST.Ident): IR.Type.Ascription = { - val methodName = buildName(ident) - val methodReference = Name.MethodReference( - None, - methodName, - methodName.location - ) - - IR.Type.Ascription( - methodReference, - translateExpression(sig, insideTypeSignature = true), - getIdentifiedLocation(inputAst) - ) - } - typed match { - case AST.Ident.any(ident) => buildAscription(ident) - case AST.App.Section.Sides(opr) => buildAscription(opr) - case AstView.MethodReference(_, _) => - IR.Type.Ascription( - translateMethodReference(typed), - translateExpression(sig, insideTypeSignature = true), - getIdentifiedLocation(inputAst) - ) - case _ => Error.Syntax(typed, Error.Syntax.InvalidStandaloneSignature) - } - case _ => Error.Syntax(inputAst, Error.Syntax.UnexpectedExpression) - } - } - - /** Translates the body of a type expression. - * - * @param body the body to be translated - * @return the [[IR]] representation of `body` - */ - def translateTypeBody(body: AST): List[IR] = { - body match { - case AST.Block.any(block) => - val actualLines: List[AST] = - block.firstLine.elem :: block.lines.flatMap(_.elem) - - if (actualLines.nonEmpty) { - actualLines.map(translateTypeBodyExpression) - } else { - List(Error.Syntax(body, Error.Syntax.InvalidTypeDefinition)) - } - case _ => List(Error.Syntax(body, Error.Syntax.InvalidTypeDefinition)) - } - } - - /** Translates any expression that can be found in the body of a type - * declaration from [[AST]] into [[IR]]. - * - * @param maybeParensedInput the expression to be translated - * @return the [[IR]] representation of `maybeParensedInput` - */ - def translateTypeBodyExpression(maybeParensedInput: AST): IR = { - val inputAst = AstView.MaybeManyParensed - .unapply(maybeParensedInput) - .getOrElse(maybeParensedInput) - - inputAst match { - case AST.Ident.Cons.any(cons) => - IR.Module.Scope.Definition - .Data( - buildName(cons), - List(), - List(), - getIdentifiedLocation(inputAst) - ) - case AstView.SpacedList(AST.Ident.Cons.any(cons) :: args) => - IR.Module.Scope.Definition - .Data( - buildName(cons), - args.map(translateArgumentDefinition(_)), - List(), - getIdentifiedLocation(inputAst) - ) - case AST.Ident.Annotation.any(ann) => - IR.Name.BuiltinAnnotation(ann.name, getIdentifiedLocation(ann)) - case AstView.FunctionSugar( - AST.Ident.Var("foreign"), - header, - body - ) => - translateForeignDefinition(header, body) match { - case Right((name, arguments, body)) => - IR.Function.Binding( - name, - arguments, - body, - getIdentifiedLocation(inputAst) - ) - case Left(reason) => - IR.Error.Syntax( - inputAst, - IR.Error.Syntax.InvalidForeignDefinition(reason) - ) - } - case fs @ AstView.FunctionSugar(_, _, _) => - translateExpression(fs) - case AST.Comment.any(inputAST) => translateComment(inputAST) - case AstView.Binding(AST.App.Section.Right(opr, arg), body) => - val translatedArgs = arg match { - case AstView.FunctionParamList(items) => - items.map(translateArgumentDefinition(_)) - case _ => - List(translateArgumentDefinition(arg)) - } - Function.Binding( - buildName(opr), - translatedArgs, - translateExpression(body), - getIdentifiedLocation(inputAst) - ) - case AstView.TypeAscription(typed, sig) => - val typedIdent = typed match { - case AST.App.Section.Sides(opr) => buildName(opr) - case AST.Ident.any(ident) => buildName(ident) - case other => translateExpression(other) - } - IR.Type.Ascription( - typedIdent, - translateExpression(sig, insideTypeSignature = true), - getIdentifiedLocation(inputAst) - ) - case assignment @ AstView.BasicAssignment(_, _) => - translateExpression(assignment) - case _ => - IR.Error.Syntax(inputAst, IR.Error.Syntax.UnexpectedDeclarationInType) - } - } - - private def translateForeignDefinition(header: List[AST], body: AST): Either[ - String, - (IR.Name, List[IR.DefinitionArgument], IR.Foreign.Definition) - ] = { - header match { - case AST.Ident.Var(lang) :: AST.Ident.Var.any(name) :: args => - body.shape match { - case AST.Literal.Text.Block.Raw(lines, _, _) => - val code = lines - .map(t => - t.text.collect { - case AST.Literal.Text.Segment.Plain(str) => str - case AST.Literal.Text.Segment.RawEsc(code) => code.repr - }.mkString - ) - .mkString("\n") - val methodName = buildName(name) - val arguments = args.map(translateArgumentDefinition(_)) - val language = EpbParser.ForeignLanguage.getBySyntacticTag(lang) - if (language == null) { - Left(s"Language $lang is not a supported polyglot language.") - } else { - val foreign = IR.Foreign.Definition( - language, - code, - getIdentifiedLocation(body) - ) - Right((methodName, arguments, foreign)) - } - case _ => - Left( - "The body of a foreign block must be an uninterpolated string block literal." - ) - } - case _ => Left("The method name is not specified.") - } - } - - /** Translates a method reference from [[AST]] into [[IR]]. - * - * @param inputAst the method reference to translate - * @return the [[IR]] representation of `inputAst` - */ - def translateMethodReference(inputAst: AST): IR.Name.MethodReference = { - inputAst match { - case AstView.MethodReference(path, methodName) => - val typeParts = path.map(translateExpression(_).asInstanceOf[IR.Name]) - IR.Name.MethodReference( - Some( - IR.Name.Qualified(typeParts, MethodReference.genLocation(typeParts)) - ), - translateExpression(methodName).asInstanceOf[IR.Name], - getIdentifiedLocation(inputAst) - ) - case _ => throw new UnhandledEntity(inputAst, "translateMethodReference") - } - } - - /** Translates an arbitrary program expression from [[AST]] into [[IR]]. - * - * @param maybeParensedInput the expresion to be translated - * @return the [[IR]] representation of `maybeParensedInput` - */ - def translateExpression( - maybeParensedInput: AST, - insideTypeSignature: Boolean = false - ): Expression = { - val inputAst = AstView.MaybeManyParensed - .unapply(maybeParensedInput) - .getOrElse(maybeParensedInput) - - inputAst match { - case AST.Def(consName, _, _) => - IR.Error - .Syntax(inputAst, IR.Error.Syntax.TypeDefinedInline(consName.name)) - case AstView.UnaryMinus(expression) => - expression match { - case AST.Literal.Number(base, number) => - translateExpression( - AST.Literal - .Number(base, s"-$number") - .setLocation(inputAst.location) - ) - case _ => - IR.Application.Prefix( - IR.Name - .Literal("negate", isMethod = true, None), - List( - IR.CallArgument.Specified( - None, - translateExpression(expression), - getIdentifiedLocation(expression) - ) - ), - hasDefaultsSuspended = false, - getIdentifiedLocation(inputAst) - ) - } - case AstView.FunctionSugar(name, args, body) => - Function.Binding( - translateIdent(name).asInstanceOf[IR.Name.Literal], - args.map(translateArgumentDefinition(_)), - translateExpression(body), - getIdentifiedLocation(inputAst) - ) - case AstView - .SuspendedBlock(name, block @ AstView.Block(lines, lastLine)) => - Expression.Binding( - buildName(name), - Expression.Block( - lines.map(translateExpression(_)), - translateExpression(lastLine), - getIdentifiedLocation(block), - suspended = true - ), - getIdentifiedLocation(inputAst) - ) - case AstView.BasicAssignment(name, expr) => - translateBinding(getIdentifiedLocation(inputAst), name, expr) - case AstView.TypeAscription(left, right) => - IR.Application.Operator.Binary( - translateCallArgument(left), - buildName(AST.Ident.Opr(AstView.TypeAscription.operatorName)), - translateCallArgument(right, insideTypeSignature = true), - getIdentifiedLocation(inputAst) - ) - case AstView.MethodDefinition(_, name, _, _) => - IR.Error.Syntax( - inputAst, - IR.Error.Syntax.MethodDefinedInline(name.asInstanceOf[AST.Ident].name) - ) - case AstView.MethodCall(target, name, args) => - inputAst match { - case AstView.QualifiedName(idents) if insideTypeSignature => - IR.Name.Qualified( - idents.map(x => translateIdent(x).asInstanceOf[IR.Name]), - getIdentifiedLocation(inputAst) - ) - case _ => - val (validArguments, hasDefaultsSuspended) = - calculateDefaultsSuspension(args) - - Application.Prefix( - buildName(name, isMethod = true), - (target :: validArguments).map(translateCallArgument(_)), - hasDefaultsSuspended = hasDefaultsSuspended, - getIdentifiedLocation(inputAst) - ) - } - case AstView.CaseExpression(scrutinee, branches) => - val actualScrutinee = translateExpression(scrutinee) - val allBranches = branches.map(translateCaseBranch) - - Case.Expr( - actualScrutinee, - allBranches, - getIdentifiedLocation(inputAst) - ) - case AstView.DecimalLiteral(intPart, fracPart) => - translateDecimalLiteral(inputAst, intPart, fracPart) - case AST.App.any(inputAST) => - translateApplicationLike(inputAST, insideTypeSignature) - case AST.Mixfix.any(inputAST) => translateApplicationLike(inputAST) - case AST.Literal.any(inputAST) => translateLiteral(inputAST) - case AST.Group.any(inputAST) => translateGroup(inputAST) - case AST.Ident.any(inputAST) => translateIdent(inputAST) - case AST.TypesetLiteral.any(tSet) => - IR.Application.Literal.Typeset( - tSet.expression.map(translateExpression(_)), - getIdentifiedLocation(tSet) - ) - case AST.SequenceLiteral.any(inputAST) => - translateSequenceLiteral(inputAST) - case AstView.Block(lines, retLine) => - Expression.Block( - lines.map(translateExpression(_)), - translateExpression(retLine), - location = getIdentifiedLocation(inputAst) - ) - case AST.Comment.any(inputAST) => translateComment(inputAST) - case AST.Invalid.any(inputAST) => translateInvalid(inputAST) - case AST.Foreign(_, _, _) => - Error.Syntax( - inputAst, - Error.Syntax.UnsupportedSyntax("foreign blocks") - ) - case AstView.Pattern(_) => - Error.Syntax(inputAst, Error.Syntax.InvalidPattern) - case AST.Macro.Ambiguous(_, _) => - Error.Syntax(inputAst, Error.Syntax.AmbiguousExpression) - case _ => - throw new UnhandledEntity(inputAst, "translateExpression") - } - } - - def translateDecimalLiteral( - ast: AST, - int: AST.Literal.Number, - frac: AST.Literal.Number - ): Expression = { - if (int.base.isDefined && int.base.get != "10") { - Error.Syntax( - int, - Error.Syntax.UnsupportedSyntax("non-base-10 decimal literals") - ) - } else if (frac.base.isDefined && frac.base.get != "10") { - Error.Syntax(frac, Error.Syntax.InvalidBaseInDecimalLiteral) - } else { - Literal.Number( - None, - s"${int.shape.int}.${frac.shape.int}", - getIdentifiedLocation(ast) - ) - } - } - - /** Translates a program literal from its [[AST]] representation into - * [[IR]]. - * - * @param literal the literal to translate - * @return the [[IR]] representation of `literal` - */ - def translateLiteral(literal: AST.Literal): Expression = - literal match { - case AST.Literal.Number(base, number) => - if (base.isDefined) { - val baseNum = - try { Integer.parseInt(base.get) } - catch { - case _: NumberFormatException => - return Error.Syntax( - literal, - Error.Syntax.InvalidBase(base.get) - ) - } - try { new BigInteger(number, baseNum) } - catch { - case _: NumberFormatException => - return Error.Syntax( - literal, - Error.Syntax.InvalidNumberForBase(number, base.get) - ) - } - } - Literal.Number(base, number, getIdentifiedLocation(literal)) - case AST.Literal.Text.any(literal) => - literal.shape match { - case AST.Literal.Text.Line.Raw(segments) => - val fullString = segments.collect { - case AST.Literal.Text.Segment.Plain(str) => str - case AST.Literal.Text.Segment.RawEsc(code) => code.repr - }.mkString - - Literal.Text(fullString, getIdentifiedLocation(literal)) - case AST.Literal.Text.Block.Raw(lines, _, _) => - val fullString = lines - .map(t => - t.text.collect { - case AST.Literal.Text.Segment.Plain(str) => str - case AST.Literal.Text.Segment.RawEsc(code) => code.repr - }.mkString - ) - .mkString("\n") - - Literal.Text(fullString, getIdentifiedLocation(literal)) - case AST.Literal.Text.Block.Fmt(lines, _, _) => - val ls = lines.map(l => parseFmtSegments(literal, l.text)) - val err = ls.collectFirst { case Left(e) => e } - err match { - case Some(err) => err - case None => - val str = ls.collect { case Right(str) => str }.mkString("\n") - IR.Literal.Text(str, getIdentifiedLocation(literal)) - } - case AST.Literal.Text.Line.Fmt(segments) => - parseFmtSegments(literal, segments) match { - case Left(err) => err - case Right(str) => - IR.Literal.Text(str, getIdentifiedLocation(literal)) - } - case TextUnclosed(_) => - Error.Syntax(literal, Error.Syntax.UnclosedTextLiteral) - - case _ => - throw new UnhandledEntity(literal.shape, "translateLiteral") - } - case _ => throw new UnhandledEntity(literal, "processLiteral") - } - - private def parseFmtSegments( - literal: AST, - segments: Seq[AST.Literal.Text.Segment[AST]] - ): Either[IR.Error, String] = { - val bldr = new StringBuilder - var err: Option[IR.Error] = None - breakable { - segments.foreach { - case SegmentEscape(code) => - code match { - case Escape.Number(_) => - err = Some( - Error.Syntax( - literal, - Error.Syntax.UnsupportedSyntax("escaped numbers") - ) - ) - break() - case unicode: Escape.Unicode => - unicode match { - case Unicode.InvalidUnicode(unicode) => - err = Some( - Error.Syntax( - literal, - Error.Syntax.InvalidEscapeSequence(unicode.repr) - ) - ) - break() - case Unicode._U16(digits) => - val buffer = ByteBuffer.allocate(2) - buffer.putChar( - Integer.parseInt(digits, 16).asInstanceOf[Char] - ) - val str = new String(buffer.array(), "UTF-16") - bldr.addAll(str) - case Unicode._U32(digits) => - val buffer = ByteBuffer.allocate(4) - buffer.putInt(Integer.parseInt(digits, 16)) - val str = new String(buffer.array(), "UTF-32") - bldr.addAll(str) - case Unicode._U21(digits) => - val buffer = ByteBuffer.allocate(4) - buffer.putInt(Integer.parseInt(digits, 16)) - val str = new String(buffer.array(), "UTF-32") - bldr.addAll(str) - } - case e: Escape.Character => bldr.addOne(e.code) - case e: Escape.Control => bldr.addAll(e.repr) - } - case SegmentPlain(text) => bldr.addAll(text) - case SegmentExpr(_) => - err = Some( - Error.Syntax( - literal, - Error.Syntax.UnsupportedSyntax("interpolated expressions") - ) - ) - break() - case SegmentRawEscape(e) => bldr.addAll(e.repr) - } - } - err.map(Left(_)).getOrElse(Right(bldr.toString)) - } - - /** Translates a sequence literal into its [[IR]] counterpart. - * @param literal the literal to translate - * @return the [[IR]] representation of `literal` - */ - def translateSequenceLiteral(literal: AST.SequenceLiteral): Expression = { - IR.Application.Literal.Sequence( - literal.items.map(translateExpression(_)), - getIdentifiedLocation(literal) - ) - } - - /** Translates an arbitrary expression, making sure to properly recognize - * qualified names. Qualified names should, probably, at some point be - * handled deeper in the compiler pipeline. - */ - private def translateQualifiedNameOrExpression(arg: AST): IR.Expression = - arg match { - case AstView.QualifiedName(segments) => - IR.Name.Qualified( - segments.map(buildName(_)), - getIdentifiedLocation(arg) - ) - case _ => translateExpression(arg) - } - - /** Translates an argument definition from [[AST]] into [[IR]]. - * - * @param arg the argument to translate - * @param isSuspended `true` if the argument is suspended, otherwise `false` - * @return the [[IR]] representation of `arg` - */ - @tailrec - def translateArgumentDefinition( - arg: AST, - isSuspended: Boolean = false - ): DefinitionArgument = { - arg match { - case AstView.AscribedArgument(name, ascType, mValue, isSuspended) => - translateIdent(name) match { - case name: IR.Name => - DefinitionArgument.Specified( - name, - Some(translateQualifiedNameOrExpression(ascType)), - mValue.map(translateExpression(_)), - isSuspended, - getIdentifiedLocation(arg) - ) - case _ => - throw new UnhandledEntity(arg, "translateArgumentDefinition") - } - case AstView.LazyAssignedArgumentDefinition(name, value) => - translateIdent(name) match { - case name: IR.Name => - DefinitionArgument.Specified( - name, - None, - Some(translateExpression(value)), - suspended = true, - getIdentifiedLocation(arg) - ) - case _ => - throw new UnhandledEntity(arg, "translateArgumentDefinition") - } - case AstView.LazyArgument(arg) => - translateArgumentDefinition(arg, isSuspended = true) - case AstView.DefinitionArgument(arg) => - translateIdent(arg) match { - case name: IR.Name => - DefinitionArgument.Specified( - name, - None, - None, - isSuspended, - getIdentifiedLocation(arg) - ) - case _ => - throw new UnhandledEntity(arg, "translateArgumentDefinition") - } - case AstView.AssignedArgument(name, value) => - translateIdent(name) match { - case name: IR.Name => - DefinitionArgument.Specified( - name, - None, - Some(translateExpression(value)), - isSuspended, - getIdentifiedLocation(arg) - ) - case _ => - throw new UnhandledEntity(arg, "translateArgumentDefinition") - } - case _ => - throw new UnhandledEntity(arg, "translateArgumentDefinition") - } - } - - /** Translates a call-site function argument from its [[AST]] representation - * into [[IR]]. - * - * @param arg the argument to translate - * @return the [[IR]] representation of `arg` - */ - def translateCallArgument( - arg: AST, - insideTypeSignature: Boolean = false - ): CallArgument.Specified = - arg match { - case AstView.AssignedArgument(left, right) => - CallArgument - .Specified( - Some(buildName(left)), - translateExpression(right, insideTypeSignature), - getIdentifiedLocation(arg) - ) - case _ => - CallArgument - .Specified( - None, - translateExpression(arg, insideTypeSignature), - getIdentifiedLocation(arg) - ) - } - - /** Calculates whether a set of arguments has its defaults suspended, and - * processes the argument list to remove that operator. - * - * @param args the list of arguments - * @return the list of arguments with the suspension operator removed, and - * whether or not the defaults are suspended - */ - def calculateDefaultsSuspension(args: List[AST]): (List[AST], Boolean) = { - val validArguments = args.filter { - case AstView.SuspendDefaultsOperator(_) => false - case _ => true - } - - val suspendPositions = args.zipWithIndex.collect { - case (AstView.SuspendDefaultsOperator(_), ix) => ix - } - - val hasDefaultsSuspended = suspendPositions.contains(args.length - 1) - - (validArguments, hasDefaultsSuspended) - } - - /** Translates an arbitrary expression that takes the form of a syntactic - * application from its [[AST]] representation into [[IR]]. - * - * @param callable the callable to translate - * @return the [[IR]] representation of `callable` - */ - def translateApplicationLike( - callable: AST, - insideTypeAscription: Boolean = false - ): Expression = { - callable match { - case AstView.Application(name, args) => - val (validArguments, hasDefaultsSuspended) = - calculateDefaultsSuspension(args) - - val fun = name match { - case AstView.Method(ast) => buildName(ast, isMethod = true) - case AstView.Expr(ast) => - translateExpression(ast, insideTypeAscription) - } - - Application.Prefix( - fun, - validArguments.map(translateCallArgument(_, insideTypeAscription)), - hasDefaultsSuspended, - getIdentifiedLocation(callable) - ) - case AST.App.Infix(l, AST.Ident.Opr("->"), r) if insideTypeAscription => - translateFunctionType( - List(translateExpression(l, insideTypeSignature = true)), - r - ) - case AstView.Lambda(args, body) => - if (args.length > 1) { - Error.Syntax( - args(1), - Error.Syntax.UnsupportedSyntax( - "pattern matching function arguments" - ) - ) - } else { - val realArgs = - args.map(translateArgumentDefinition(_, insideTypeAscription)) - val realBody = translateExpression(body, insideTypeAscription) - Function.Lambda(realArgs, realBody, getIdentifiedLocation(callable)) - } - case AST.App.Infix(left, fn, right) => - val leftArg = translateCallArgument(left, insideTypeAscription) - val rightArg = translateCallArgument(right, insideTypeAscription) - - fn match { - case AST.Ident.Opr.any(fn) => - if (leftArg.name.isDefined) { - IR.Error.Syntax(left, IR.Error.Syntax.NamedArgInOperator) - } else if (rightArg.name.isDefined) { - IR.Error.Syntax(right, IR.Error.Syntax.NamedArgInOperator) - } else { - Application.Operator.Binary( - leftArg, - buildName(fn), - rightArg, - getIdentifiedLocation(callable) - ) - } - case _ => IR.Error.Syntax(left, IR.Error.Syntax.InvalidOperatorName) - } - case AST.App.Prefix(_, _) => - throw new UnhandledEntity(callable, "translateCallable") - case AST.App.Section.any(sec) => translateOperatorSection(sec) - case AST.Mixfix(nameSegments, args) => - val realNameSegments = nameSegments.collect { - case AST.Ident.Var.any(v) => v.name - case AST.Ident.Cons.any(v) => v.name.toLowerCase - } - - val functionName = - AST.Ident.Var(realNameSegments.mkString("_")) - - Application.Prefix( - buildName(functionName, isMethod = true), - args.map(translateCallArgument(_, insideTypeAscription)).toList, - hasDefaultsSuspended = false, - getIdentifiedLocation(callable) - ) - case AST.Macro.Ambiguous(_, _) => - Error.Syntax(callable, Error.Syntax.AmbiguousExpression) - case _ => throw new UnhandledEntity(callable, "translateCallable") - } - } - - @tailrec - private def translateFunctionType( - argsAcc: List[Expression], - next: AST - ): Expression = - skipParens(next) match { - case AST.App.Infix(nextArg, AST.Ident.Opr("->"), next) => - translateFunctionType( - argsAcc :+ translateExpression(nextArg, insideTypeSignature = true), - next - ) - case other => - IR.Type.Function( - argsAcc, - translateExpression(other, insideTypeSignature = true), - None - ) - } - - private def skipParens(ast: AST): AST = - AstView.MaybeManyParensed.unapply(ast).getOrElse(ast) - - /** Translates an operator section from its [[AST]] representation into the - * [[IR]] representation. - * - * @param section the operator section - * @return the [[IR]] representation of `section` - */ - def translateOperatorSection( - section: AST.App.Section - ): Expression = { - section match { - case AST.App.Section.Left.any(left) => - val leftArg = translateCallArgument(left.arg) - - if (leftArg.name.isDefined) { - Error.Syntax(section, Error.Syntax.NamedArgInSection) - } else { - Application.Operator.Section.Left( - leftArg, - buildName(left.opr), - getIdentifiedLocation(left) - ) - } - case AST.App.Section.Sides.any(sides) => - Application.Operator.Section.Sides( - buildName(sides.opr), - getIdentifiedLocation(sides) - ) - case AST.App.Section - .Right(AST.Ident.Opr("."), AstView.ConsOrVar(ident)) => - buildName(ident, isMethod = true) - case AST.App.Section.Right.any(right) => - val rightArg = translateCallArgument(right.arg) - - if (rightArg.name.isDefined) { - Error.Syntax(section, Error.Syntax.NamedArgInSection) - } else { - Application.Operator.Section.Right( - buildName(right.opr), - translateCallArgument(right.arg), - getIdentifiedLocation(right) - ) - } - case _ => throw new UnhandledEntity(section, "translateOperatorSection") - } - } - - /** Translates an arbitrary program identifier from its [[AST]] representation - * into [[IR]]. - * - * @param identifier the identifier to translate - * @return the [[IR]] representation of `identifier` - */ - def translateIdent(identifier: AST.Ident): Expression = { - identifier match { - case AST.Ident.Var(name) => - if (name == Constants.Names.SELF_ARGUMENT) { - Name.Self(getIdentifiedLocation(identifier)) - } else { - buildName(identifier) - } - case AST.Ident.Annotation(name) => - Name.BuiltinAnnotation(name, getIdentifiedLocation(identifier)) - case AST.Ident.Cons(name) => - if (name == Constants.Names.SELF_TYPE_ARGUMENT) { - Name.SelfType(getIdentifiedLocation(identifier)) - } else { - buildName(identifier) - } - case AST.Ident.Blank(_) => - Name.Blank(getIdentifiedLocation(identifier)) - case AST.Ident.Opr.any(_) => - Error.Syntax( - identifier, - Error.Syntax.UnsupportedSyntax("operator sections") - ) - case AST.Ident.Mod(_) => - Error.Syntax( - identifier, - Error.Syntax.UnsupportedSyntax("module identifiers") - ) - case _ => - throw new UnhandledEntity(identifier, "translateIdent") - } - } - - /** Translates an arbitrary binding operation from its [[AST]] representation - * into [[IR]]. - * - * @param location the source location of the binding - * @param name the name of the binding being assigned to - * @param expr the expression being assigned to `name` - * @return the [[IR]] representation of `expr` being bound to `name` - */ - def translateBinding( - location: Option[IdentifiedLocation], - name: AST, - expr: AST - ): Expression.Binding = { - val irName = translateExpression(name) - - irName match { - case n: IR.Name => - Expression.Binding(n, translateExpression(expr), location) - case _ => - throw new UnhandledEntity(name, "translateBinding") - } - } - - /** Translates the branch of a case expression from its [[AST]] representation - * into [[IR]], also handling the documentation comments in between branches. - * - * The documentation comments are translated to dummy branches that contain - * an empty expression and a dummy [[IR.Pattern.Documentation]] pattern - * containing the comment. These dummy branches are removed in the - * DocumentationComments pass where the comments are attached to the actual - * branches. - * - * @param branch the case branch or comment to translate - * @return the [[IR]] representation of `branch` - */ - def translateCaseBranch(branch: AST): Case.Branch = { - branch match { - case AstView.CaseBranch(pattern, expression) => - Case.Branch( - translatePattern(pattern), - translateExpression(expression), - getIdentifiedLocation(branch) - ) - case c @ AST.Comment(lines) => - val doc = lines.mkString("\n") - val location = getIdentifiedLocation(c) - Case.Branch( - Pattern.Documentation(doc, location), - IR.Empty(None), - location - ) - case _ => throw new UnhandledEntity(branch, "translateCaseBranch") - } - } - - /** Translates a pattern in a case expression from its [[AST]] representation - * into [[IR]]. - * - * @param pattern the case pattern to translate - * @return - */ - def translatePattern(pattern: AST): Pattern = { - AstView.MaybeManyParensed.unapply(pattern).getOrElse(pattern) match { - case AstView.ConstructorPattern(conses, fields) => - val irConses = conses.map(translateIdent(_).asInstanceOf[IR.Name]) - val name = irConses match { - case List(n) => n - case _ => IR.Name.Qualified(irConses, None) - } - Pattern.Constructor( - name, - fields.map(translatePattern), - getIdentifiedLocation(pattern) - ) - case AstView.CatchAllPattern(name) => - Pattern.Name( - translateIdent(name).asInstanceOf[IR.Name], - getIdentifiedLocation(pattern) - ) - case AstView.LiteralPattern(literal) => - Pattern.Literal( - translateLiteral(literal).asInstanceOf[IR.Literal], - getIdentifiedLocation(pattern) - ) - case AstView.TypePattern(name, tpe) => - val irConses = tpe.map(translateIdent(_).asInstanceOf[IR.Name]) - val tpeName = irConses match { - case List(n) => n - case _ => IR.Name.Qualified(irConses, None) - } - Pattern.Type( - translateIdent(name).asInstanceOf[IR.Name], - tpeName, - getIdentifiedLocation(pattern) - ) - case _ => - throw new UnhandledEntity(pattern, "translatePattern") - } - } - - /** Translates an arbitrary grouped piece of syntax from its [[AST]] - * representation into [[IR]]. - * - * It is currently an error to have an empty group. - * - * @param group the group to translate - * @return the [[IR]] representation of the contents of `group` - */ - def translateGroup(group: AST.Group): Expression = { - group.body match { - case Some(ast) => translateExpression(ast) - case None => Error.Syntax(group, Error.Syntax.EmptyParentheses) - } - } - - /** Translates an import statement from its [[AST]] representation into - * [[IR]]. - * - * @param imp the import to translate - * @return the [[IR]] representation of `imp` - */ - def translateImport(imp: AST.Import): Module.Scope.Import = { - imp match { - case AST.Import(path, rename, isAll, onlyNames, hiddenNames) => - IR.Module.Scope.Import.Module( - IR.Name.Qualified(path.map(buildName(_)).toList, None), - rename.map(buildName(_)), - isAll, - onlyNames.map(_.map(buildName(_)).toList), - hiddenNames.map(_.map(buildName(_)).toList), - getIdentifiedLocation(imp) - ) - case _ => - IR.Error.Syntax(imp, IR.Error.Syntax.InvalidImport) - } - } - - /** Translates an export statement from its [[AST]] representation into - * [[IR]]. - * - * @param exp the export to translate - * @return the [[IR]] representation of `imp` - */ - def translateExport(exp: AST.Export): Module.Scope.Export.Module = { - exp match { - case AST.Export(path, rename, isAll, onlyNames, hiddenNames) => - IR.Module.Scope.Export.Module( - IR.Name.Qualified(path.map(buildName(_)).toList, None), - rename.map(buildName(_)), - isAll, - onlyNames.map(_.map(buildName(_)).toList), - hiddenNames.map(_.map(buildName(_)).toList), - getIdentifiedLocation(exp) - ) - case _ => throw new UnhandledEntity(exp, "translateExport") - } - } - - /** Translates an arbitrary invalid expression from the [[AST]] representation - * of the program into its [[IR]] representation. - * - * @param invalid the invalid entity to translate - * @return the [[IR]] representation of `invalid` - */ - def translateInvalid(invalid: AST.Invalid): Expression = { - invalid match { - case AST.Invalid.Unexpected(_, _) => - Error.Syntax( - invalid, - Error.Syntax.UnexpectedExpression - ) - case AST.Invalid.Unrecognized(_) => - Error.Syntax( - invalid, - Error.Syntax.UnrecognizedToken - ) - case AST.Ident.InvalidSuffix(_, _) => - Error.Syntax( - invalid, - Error.Syntax.InvalidSuffix - ) - case AST.Literal.Text.Unclosed(_) => - Error.Syntax( - invalid, - Error.Syntax.UnclosedTextLiteral - ) - case _ => - throw new UnhandledEntity(invalid, "translateInvalid") - } - } - - /** Translates a comment from its [[AST]] representation into its [[IR]] - * representation. - * - * Currently this only supports documentation comments, and not standarc - * types of comments as they can't currently be represented. - * - * @param comment the comment to transform - * @return the [[IR]] representation of `comment` - */ - def translateComment(comment: AST): Comment = { - comment match { - case AST.Comment(lines) => - Comment.Documentation( - lines.mkString("\n"), - getIdentifiedLocation(comment) - ) - case _ => - throw new UnhandledEntity(comment, "processComment") - } - } - - private def buildName( - ident: AST.Ident, - isMethod: Boolean = false - ): IR.Name.Literal = { - IR.Name.Literal( - ident.name, - isMethod || AST.Opr.any.unapply(ident).isDefined, - getIdentifiedLocation(ident) - ) - } -} diff --git a/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstView.scala b/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstView.scala deleted file mode 100644 index 4a3c0cf24edb..000000000000 --- a/engine/runtime/src/main/scala/org/enso/compiler/codegen/AstView.scala +++ /dev/null @@ -1,1013 +0,0 @@ -package org.enso.compiler.codegen - -import org.enso.data -import org.enso.data.List1 -import org.enso.syntax.text.AST -import org.enso.syntax.text.AST.Ident.{Opr, Var} - -/** This object contains view patterns that allow matching on the parser [[AST]] - * for more sophisticated constructs. - * - * These view patterns are implemented as custom unapply methods that only - * return [[Some]] when more complex conditions are met. These view patterns - * return the [[AST]] representations of the relevant segments in order to - * allow location information to easily be provided to the translation - * mechanism. - */ -object AstView { - - object Atom { - - /** Matches a primitive atom definition only. - * - * It will _not_ match a type definition with a body. - * - * @param ast the structure to try and match on - * @return the name of the atom, and the arguments to the atom - */ - def unapply(ast: AST): Option[(AST.Ident, List[AST])] = - ast match { - case AST.Def(name, args, body) if body.isEmpty => Some((name, args)) - case _ => None - } - } - - object TypeDef { - - /** Matches a complex type definition. - * - * @param ast the structure to try and match on - * @return the name of the type, the arguments to it, and the type body - */ - def unapply(ast: AST): Option[(AST.Ident, List[AST], AST)] = - ast match { - case AST.Def(name, args, body) if body.isDefined => - Some((name, args, body.get)) - case _ => None - } - } - - object Block { - - /** Matches an arbitrary block in the program source. - * - * @param ast the structure to try and match on - * @return a list of expressions in the block, and the final expression - * separately - */ - def unapply(ast: AST): Option[(List[AST], AST)] = - ast match { - case AST.Block(_, _, firstLine, lines) => - val actualLines = firstLine.elem :: lines.flatMap(_.elem) - if (actualLines.nonEmpty) { - Some((actualLines.dropRight(1), actualLines.last)) - } else { - None - } - case _ => None - } - } - - object SuspendedBlock { - - /** Matches an arbitrary suspended block in the program source. - * - * A suspended block is one that is bound to a name but takes no arguments. - * - * @param ast the structure to try and match on - * @return the name to which the block is assigned, and the block itself - */ - def unapply(ast: AST): Option[(AST.Ident, AST.Block)] = { - ast match { - case BasicAssignment(name, AST.Block.any(block)) => - Some((name, block)) - case _ => None - } - } - } - - object Binding { - val bindingOpSym: Opr = AST.Ident.Opr("=") - - /** Matches an arbitrary binding in the program source. - * - * A binding is any expression of the form ` = `, and this - * matcher asserts no additional properties on the structure it matches. - * - * @param ast the structure to try and match on - * @return the expression on the left of the binding operator, and the - * expression on the right side of the binding operator - */ - def unapply(ast: AST): Option[(AST, AST)] = { - ast match { - case AST.App.Infix.any(ast) => - val left = ast.larg - val op = ast.opr - val right = ast.rarg - - if (op == bindingOpSym) { - Some((left, right)) - } else { - None - } - case _ => None - } - } - } - - object BasicAssignment { - val assignmentOpSym: Opr = AST.Ident.Opr("=") - - /** Matches an assignment. - * - * An assignment is a [[Binding]] where the left-hand side is a variable - * name. - * - * @param ast the structure to try and match on - * @return the variable name assigned to, and the expression being assigned - */ - def unapply(ast: AST): Option[(AST.Ident, AST)] = { - ast match { - case Binding(AST.Ident.Var.any(left), right) => Some((left, right)) - case Binding(AST.Ident.Blank.any(left), right) => Some((left, right)) - case _ => None - } - } - } - - object MaybeBlankName { - val blankSym: String = "_" - - /** Matches an identifier that may be a blank `_`. - * - * @param ast the structure to try and match on - * @return the identifier - */ - def unapply(ast: AST): Option[AST.Ident] = { - ast match { - case AST.Ident.Var.any(variable) => Some(variable) - case AST.Ident.Cons.any(cons) => Some(cons) - case AST.Ident.Blank.any(blank) => Some(blank) - case _ => None - } - } - } - - object Lambda { - val lambdaOpSym: Opr = AST.Ident.Opr("->") - - /** Matches a lambda expression in the program source. - * - * A lambda expression is of the form ` -> ` where - * `` is a space-separated list of valid argument definitions, and - * `` is an arbitrary program expression. - * - * @param ast the structure to try and match on - * @return a list of the arguments defined for the lambda, and the body of - * the lambda - */ - def unapply(ast: AST): Option[(List[AST], AST)] = { - ast match { - case AST.App.Infix.any(ast) => - val left = ast.larg - val op = ast.opr - val right = ast.rarg - - if (op == lambdaOpSym) { - left match { - case FunctionParamList(args) => Some((args, right)) - case _ => None - } - } else { - None - } - case _ => None - } - } - } - - object LazyArgument { - - /** Matches on a lazy argument definition or usage. - * - * A lazy argument is one of the form `~t` where `t` is a valid parameter - * name. This is temporary syntax and will be removed once we have the - * ability to insert these analyticallyl - * - * @param ast the structure to try and match on - * @return the term being forced - */ - def unapply(ast: AST): Option[AST] = - ast match { - case MaybeManyParensed( - AST.App.Section.Right(AST.Ident.Opr("~"), FunctionParam(arg)) - ) => - Some(arg) - case _ => None - } - } - - object FunctionParam { - - /** Matches a definition-site function parameter. - * - * @param ast the structure to try and match on - * @return the parameter definition - */ - def unapply(ast: AST): Option[AST] = - ast match { - case AscribedArgument(_, _, _, _) => Some(ast) - case LazyAssignedArgumentDefinition(_, _) => Some(ast) - case AssignedArgument(_, _) => Some(ast) - case DefinitionArgument(_) => Some(ast) - case LazyArgument(_) => Some(ast) - case _ => None - } - } - - object FunctionParamList { - - /** Matches on the parameter list of a lambda. - * - * @param ast the structure to try and match on - * @return a list of the arguments for which the lambda is defined - */ - def unapply(ast: AST): Option[List[AST]] = { - ast match { - case SpacedList(args) => - val realArgs = args.collect { case a @ FunctionParam(_) => a } - - if (realArgs.length == args.length) { - Some(args) - } else { - None - } - case FunctionParam(p) => Some(List(p)) - case _ => None - } - } - } - - object MaybeTyped { - - /** Matches on terms that _may_ have a type signature. - * - * Such terms take the form of ` : `, where both `` and - * `` can be arbitrary program expressions. - * - * @param ast the structure to try and match on - * @return the term and the type ascribed to it - */ - def unapply(ast: AST): Option[(AST, AST)] = - ast match { - case AST.App.Infix(entity, AST.Ident.Opr(":"), signature) => - Some((entity, signature)) - case _ => None - } - } - - object FunctionSugar { - - /** Matches on terms that represent long-form function definitions. - * - * @param ast the structure to try and match on - * @return the name of the function, the list of function arguments, and - * the function body - */ - def unapply(ast: AST): Option[(AST.Ident, List[AST], AST)] = { - ast match { - case AstView.Binding(left, body) => - left match { - case FunctionParamList(items) => - if (items.length > 1) { - val fnName = items.head - val args = items.tail - - fnName match { - case AST.Ident.Var.any(name) => Some((name, args, body)) - case _ => None - } - } else { - None - } - case _ => None - } - case _ => None - } - } - } - - object Parensed { - - /** Matches on terms that is surrounded by parentheses. - * - * It 'peels off' one layer of parentheses, so the returned term may still - * be surrounded by more parentheses. - * - * @param ast the structure to try and match on - * @return the term contained in the parentheses - */ - def unapply(ast: AST): Option[AST] = { - AST.Group.unapply(ast).flatten - } - } - - object MaybeParensed { - - /** Matches on terms that _may_ be surrounded by (an arbitrary amount of) - * parentheses. - * - * It 'peels off' one layer of parentheses, so the returned term may still - * be surrounded by more parentheses. - * - * @param ast the structure to try and match on - * @return the term contained in the parentheses - */ - def unapply(ast: AST): Option[AST] = { - ast match { - case AST.Group(mExpr) => mExpr - case a => Some(a) - } - } - } - - object ManyParensed { - - /** Matches on terms that is surrounded by an arbitrary (but non-zero) - * amount of parentheses. - * - * The resulting term is not surrounded by any more parentheses. - * - * @param ast the structure to try and match on - * @return the term contained in the parentheses - */ - def unapply(ast: AST): Option[AST] = { - ast match { - case Parensed(MaybeManyParensed(p)) => Some(p) - case _ => None - } - } - } - - object MaybeManyParensed { - - /** Matches on terms that _may_ be surrounded by (an arbitrary amount of) - * parentheses. - * - * The resulting term is not surrounded by any more parentheses. - * - * @param ast the structure to try and match on - * @return the term contained in the parentheses - */ - def unapply(ast: AST): Option[AST] = { - ast match { - case AST.Group(mExpr) => mExpr.flatMap(unapply) - case a => Some(a) - } - } - } - - object AssignedArgument { - - /** Matches on the structure of an 'assigned argument'. - * - * Such an argument has the structure ` = ` where `` - * must be a valid variable name, and `` is an arbitrary Enso - * expression. - * - * @param ast the structure to try and match on - * @return the variable name and the expression being bound to it - */ - def unapply(ast: AST): Option[(AST.Ident, AST)] = - MaybeManyParensed.unapply(ast).flatMap(BasicAssignment.unapply) - } - - object AscribedArgument { - - /** Matches on the structure of an 'ascribed argument'. - * - * Such an argument has the structure - * `[~] : [= ` must be a valid - * variable name, and both ``s are arbitrary Enso expressions. - * The `~` to mark laziness is optional. - * - * @param ast the structure to try and match on - * @return the variable name, the type being ascribed to it, the value - * being bound to it, and whether or not it was marked as lazy - */ - def unapply(ast: AST): Option[(AST.Ident, AST, Option[AST], Boolean)] = { - MaybeManyParensed.unapply(ast).flatMap(matchBareArg) - } - - def matchBareArg( - ast: AST - ): Option[(AST.Ident, AST, Option[AST], Boolean)] = { - ast match { - case AST.App.Infix(maybeVarAndType, AST.Ident.Opr("="), exprVal) => - maybeVarAndType match { - case AST.App.Infix(varName, AST.Ident.Opr(":"), expr) => - varName match { - case AST.Ident.Var.any(v) => - Some((v, expr, Some(exprVal), false)) - case AST.Ident.Blank.any(v) => - Some((v, expr, Some(exprVal), false)) - case AST.App.Section - .Right(AST.Ident.Opr("~"), AST.Ident.Var.any(v)) => - Some((v, expr, Some(exprVal), true)) - case AST.App.Section - .Right(AST.Ident.Opr("~"), AST.Ident.Blank.any(v)) => - Some((v, expr, Some(exprVal), true)) - case _ => None - } - case _ => None - } - case AST.App.Infix(varName, AST.Ident.Opr(":"), expr) => - varName match { - case AST.Ident.Var.any(v) => - Some((v, expr, None, false)) - case AST.Ident.Blank.any(v) => - Some((v, expr, None, false)) - case AST.App.Section - .Right(AST.Ident.Opr("~"), AST.Ident.Var.any(v)) => - Some((v, expr, None, true)) - case AST.App.Section - .Right(AST.Ident.Opr("~"), AST.Ident.Blank.any(v)) => - Some((v, expr, None, true)) - case _ => None - } - case _ => None - } - } - - } - - object LazyAssignedArgumentDefinition { - - /** Matches on the definition of a lazy argument for a function that also - * has a default value. - * - * @param ast the structure to try and match on - * @return the name of the argument being declared and the expression of - * the default value being bound to it - */ - def unapply(ast: AST): Option[(AST.Ident, AST)] = { - ast match { - case MaybeManyParensed( - Binding( - AST.App.Section.Right(AST.Ident.Opr("~"), AST.Ident.Var.any(v)), - r - ) - ) => - Some((v, r)) - case _ => None - } - } - } - - object DefinitionArgument { - - /** Matches on a definition argument, which is a standard variable - * identifier. - * - * @param ast the structure to try and match on - * @return the name of the argument - */ - def unapply(ast: AST): Option[AST.Ident] = - ast match { - case MaybeManyParensed(MaybeBlankName(ast)) => Some(ast) - case _ => None - } - } - - /** A union type for application matchers. */ - sealed trait MethodOrExpr - - /** A wrapper for applications denoting the name as being a method name. - * @param ast the original identifier - */ - case class Method(ast: AST.Ident) extends MethodOrExpr - - /** A wrapper for applications denoting the function as not-a-method. - * @param ast the original AST - */ - case class Expr(ast: AST) extends MethodOrExpr - - object Application { - - /** Matches an arbitrary function application. This includes both method - * calls and standard function applications as they are syntactically - * unified. - * - * @param ast the structure to try and match on - * @return the name of the function, and a list of its arguments (including - * the `self` argument if using method-call syntax) - */ - def unapply(ast: AST): Option[(MethodOrExpr, List[AST])] = - SpacedList.unapply(ast).flatMap { - case fun :: args => - fun match { - case MethodCall(target, function, methodArgs) => - Some((Method(function), target :: methodArgs ++ args)) - case _ => Some((Expr(fun), args)) - } - case _ => None - } - } - - object MethodCall { - - private def consToVar(ast: AST.Ident): AST.Ident = - ast match { - case AST.Ident.Cons(c) => - AST.Ident.Var(c).setLocation(ast.location).setID(ast.id) - case _ => ast - } - - /** Matches on a method call. - * - * A method call has the form `. ` where `` is - * an arbitrary expression, `` is the name of the function being - * called, and `` are the arguments to the call. - * - * @param ast the structure to try and match on - * @return the `self` expression, the function name, and the arguments to - * the function - */ - def unapply(ast: AST): Option[(AST, AST.Ident, List[AST])] = - ast match { - case OperatorDot(target, Application(Expr(ConsOrVar(ident)), args)) => - Some((target, consToVar(ident), args)) - case AST.App.Section.Left( - MethodCall(target, ident, List()), - susp @ SuspendDefaultsOperator(_) - ) => - Some((target, ident, List(susp))) - case OperatorDot(target, ConsOrVar(ident)) => - Some((target, consToVar(ident), List())) - case _ => None - } - } - - object SuspendDefaultsOperator { - - /** Matches on a usage of the `...` 'suspend defaults' operator. - * - * @param ast the structure to try and match on - * @return the 'suspend defaults' operator - */ - def unapply(ast: AST): Option[AST] = { - ast match { - case AST.Ident.Opr("...") => Some(ast) - case _ => None - } - } - } - - object SpacedList { - - /** Matches an arbitrary space-separated list in the AST, possibly including - * a usage of the `...` operator. - * - * @param ast the structure to try and match on - * @return the elements of the list - */ - def unapply(ast: AST): Option[List[AST]] = { - matchSpacedList(ast) - } - - private[this] def matchSpacedList(ast: AST): Option[List[AST]] = { - ast match { - case MaybeManyParensed(AST.App.Prefix(fn, arg)) => - val fnRecurse = matchSpacedList(fn) - - fnRecurse match { - case Some(headItems) => Some(headItems :+ arg) - case None => Some(List(fn, arg)) - } - case MaybeManyParensed( - AST.App.Section.Left(ast, SuspendDefaultsOperator(suspend)) - ) => - ast match { - case ConsOrVar(_) => Some(List(ast, suspend)) - case _ => - val astRecurse = matchSpacedList(ast) - - astRecurse match { - case Some(items) => Some(items :+ suspend) - case None => None - } - } - case _ => None - } - } - } - - object MethodDefinition { - - /** Matches on the definition of a method. - * - * These take the form of `. = `, - * or ` = `, where `` is the name of a type, - * `` is the name of a function, and `` is an - * arbitrary program expression. - * - * @param ast the structure to try and match on - * @return the path segments of the type reference, the function name, the - * arguments to the method, and the bound expression - */ - def unapply(ast: AST): Option[(List[AST], AST, List[AST], AST)] = { - ast match { - case Binding(lhs, rhs) => - lhs match { - case MethodReference(targetPath, name) => - Some((targetPath, name, List(), rhs)) - case MethodBindingLHS(path, methodName, args) => - Some((path, methodName, args, rhs)) - case AST.App.Section.Right(opr, arg) => - Some((List(), opr, List(arg), rhs)) - case AST.Ident.Var.any(name) => Some((List(), name, List(), rhs)) - case _ => None - } - case _ => None - } - } - } - - object MethodBindingLHS { - - /** Matches on the left hand side of a sugared method definition. - * - * @param ast the structure to try and match on - * @return the path segments of the type reference, the function name, and - * the arguments - */ - def unapply(ast: AST): Option[(List[AST], AST, List[AST])] = { - ast match { - case SpacedList(MethodReference(path, methodName) :: args) => - val validArgs = args.forall { - case AstView.FunctionParam(_) => true - case _ => false - } - - if (validArgs) Some((path, methodName, args)) else None - case _ => None - } - } - } - - object ConsOrVar { - - /** Matches any expression that is either the name of a constructor or a - * variable. - * - * @param arg the structure to try and match on - * @return the identifier matched on - */ - def unapply(arg: AST): Option[AST.Ident] = - arg match { - case AST.Ident.Var.any(arg) => Some(arg) - case AST.Ident.Cons.any(arg) => Some(arg) - case _ => None - } - } - - object OperatorDot { - - /** Matches on an arbitrary usage of operator `.` with no restrictions on - * the operands. - * - * @param ast the structure to try and match on - * @return the left- and right-hand sides of the operator - */ - def unapply(ast: AST): Option[(AST, AST)] = - ast match { - case AST.App.Infix(left, AST.Ident.Opr("."), right) => - Some((left, right)) - case _ => - None - } - } - - object DotChain { - - /** Matches an arbitrary chain of [[OperatorDot]] expressions. - * - * @param ast the structure to try and match on - * @return the segments making up the chain - */ - def unapply(ast: AST): Option[List[AST]] = { - val path = matchDotChain(ast) - - if (path.isEmpty) { - None - } else { - Some(path) - } - } - - private[this] def matchDotChain(ast: AST): List[AST] = { - ast match { - case OperatorDot(left, right) => matchDotChain(left) :+ right - case AST.Ident.any(ast) => List(ast) - case _ => List() - } - } - } - - object MethodReference { - - /** Matches on a method reference. - * - * A method reference is a [[DotChain]] where all but the last element are - * the names of constructors. - * - * @param ast the structure to try and match on - * @return the constructor segments and the final segment - */ - def unapply(ast: AST): Option[(List[AST], AST)] = { - ast match { - case DotChain(segments) => - if (segments.length >= 2) { - val consPath = segments.dropRight(1) - val maybeVar = segments.last - - val isValid = consPath.collect { case a @ AST.Ident.Cons(_) => - a - }.length == consPath.length - - if (isValid) { - maybeVar match { - case AST.Ident.Var(_) => Some((consPath, maybeVar)) - case _ => None - } - } else { - None - } - } else { - None - } - case _ => None - } - } - } - - object CaseExpression { - val caseName: List1[Var] = - data.List1(AST.Ident.Var("case"), AST.Ident.Var("of")) - - /** Matches on a case expression. - * - * A case expression is of the following form: - * - * {{{ - * case of - * ## - * -> - * <...> - * }}} - * - * where: - * - `` is an arbitrary non-block program expression - * - `` is a [[Pattern]] - * - `` is an arbirary program expression - * - `` is an optional documentation comment - * - * @param ast the structure to try and match on - * @return the scrutinee and a list of the case branches and comments - */ - def unapply(ast: AST): Option[(AST, List[AST])] = { - ast match { - case AST.Mixfix(identSegments, argSegments) => - if (identSegments == caseName) { - if (argSegments.length == 2) { - val scrutinee = argSegments.head - val caseBranches = argSegments.last - - caseBranches match { - case AST.Block(_, _, firstLine, restLines) => - val blockLines = firstLine.elem :: restLines.flatMap(_.elem) - - val matchBranches = blockLines.collect { - case b @ CaseBranch(_, _) => b - case c @ AST.Comment(_) => c - } - - if (matchBranches.length == blockLines.length) { - Some((scrutinee, matchBranches)) - } else { - None - } - case _ => None - } - } else { - None - } - } else { - None - } - case _ => None - } - } - } - - object CaseBranch { - - /** Matches on an arbitrary pattern match case branch. - * - * A case branch has the form ` -> `, where - * `` is an expression that can match on the scrutinee, and - * `` is an arbitrary expression to execute on a successful - * match. - * - * @param ast the structure to try and match on - * @return the pattern, and the branch expression - */ - def unapply(ast: AST): Option[(AST, AST)] = { - ast match { - case AST.App.Infix(left, AST.Ident.Opr("->"), right) => - left match { - case Pattern(pat) => Some((pat, right)) - case _ => None - } - case AST.App.Infix( - left, - AST.Ident.Opr(":"), - AST.App.Infix(tpe, AST.Ident.Opr("->"), right) - ) => - (left, tpe) match { - case (CatchAllPattern(_), QualifiedName(_)) => - Some((AST.App.Infix(left, AST.Ident.Opr(":"), tpe), right)) - case _ => - None - } - case _ => None - } - } - } - - object Pattern { - - /** Matches on an arbitrary pattern expression. - * - * @param ast the structure to try and match on - * @return the pattern - */ - def unapply(ast: AST): Option[AST] = { - ast match { - case ConstructorPattern(_, _) => Some(ast) - case CatchAllPattern(pat) => Some(pat) - case LiteralPattern(lit) => Some(lit) - case TypePattern(_, _) => Some(ast) - case Parensed(Pattern(p)) => - println(p) - Some(p) - case _ => None - } - } - } - - object QualifiedName { - def unapply(ast: AST): Option[List[AST.Ident.Cons]] = - ast match { - case OperatorDot(l, AST.Ident.Cons.any(name)) => - unapply(l).map(_ :+ name) - case AST.Ident.Cons.any(name) => - Some(List(name)) - case _ => None - } - } - - object ConstructorPattern { - - /** Matches on a constructor pattern. - * - * Constructor patterns take the form ` `, where - * `` is a referent name, and `` is a maybe empty list of - * patterns. - * - * @param ast the structure to try and match on - * @return the pattern - */ - def unapply(ast: AST): Option[(List[AST.Ident.Cons], List[AST])] = { - MaybeManyParensed.unapply(ast).getOrElse(ast) match { - case QualifiedName(cons) => Some((cons, List())) - case SpacedList(elems) if elems.nonEmpty => - elems.head match { - case QualifiedName(refName) => - val allFieldsValid = elems.tail.forall { - case Pattern(_) => true - case _ => false - } - - if (allFieldsValid) { - Some((refName, elems.tail)) - } else { - None - } - case _ => None - } - case _ => None - } - } - } - - object LiteralPattern { - def unapply(ast: AST): Option[AST.Literal] = { - MaybeManyParensed.unapply(ast).getOrElse(ast) match { - case AST.Literal.Number.any(number) => Some(number) - case AST.Literal.Text.any(text) => Some(text) - case _ => None - } - } - } - - object CatchAllPattern { - - /** Matches on a catch all pattern. - * - * A catch all pattern takes the form of a name, which may be blank. - * - * @param ast the structure to try and match on - * @return the pattern - */ - def unapply(ast: AST): Option[AST.Ident] = { - MaybeManyParensed.unapply(ast).getOrElse(ast) match { - case AST.Ident.any(ident) => Some(ident) - case _ => None - } - } - } - - object TypePattern { - - /** Matches on a type patternn. - * - * A catch type pattern takes the form of a name, which may be blank, colon and a type name. - * - * @param ast the structure to try and match on - * @return the pattern - */ - def unapply(ast: AST): Option[(AST.Ident, List[AST.Ident])] = { - MaybeManyParensed.unapply(ast).getOrElse(ast) match { - case AST.App.Infix( - CatchAllPattern(left), - AST.Ident.Opr(":"), - QualifiedName(right) - ) => - Some((left, right)) - case _ => None - } - } - } - - object DecimalLiteral { - def unapply(ast: AST): Option[(AST.Literal.Number, AST.Literal.Number)] = - ast match { - case AST.App.Infix( - AST.Literal.Number.any(int), - AST.Ident.Opr("."), - AST.Literal.Number.any(frac) - ) => - Some((int, frac)) - case _ => None - } - } - - object UnaryMinus { - def minusSymbol: String = "-" - - /** Matches a unary minus. - * - * It should be noted that this is a hack that matches a spaced section as - * well for now. This will be improved with the new parser. - * - * @param ast the structure to try and match on - * @return the negated expression - */ - def unapply(ast: AST): Option[AST] = - ast match { - case MaybeManyParensed( - AST.App.Section.Right(AST.Ident.Opr("-"), expression) - ) => - Some(expression) - case _ => None - } - } - - object TypeAscription { - val operatorName: String = ":" - - /** Matches a usage of the type ascription operator `:`. - * - * @param ast the structure to try and match on - * @return the typed expression, and the ascribed type - */ - def unapply(ast: AST): Option[(AST, AST)] = - ast match { - case MaybeManyParensed(AST.App.Infix(typed, AST.Ident.Opr(op), sig)) - if op == operatorName => - Some((typed, sig)) - case _ => None - } - } -} diff --git a/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala b/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala index 719496504d64..5445e48fec37 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala @@ -1,9 +1,9 @@ package org.enso.compiler.context +import com.oracle.truffle.api.source.Source import java.util.UUID -import org.enso.syntax.text.Parser +import org.enso.compiler.EnsoCompiler import org.enso.compiler.core.IR -import org.enso.compiler.codegen.AstToIr import org.enso.compiler.exception.CompilerError import org.enso.compiler.pass.analyse.DataflowAnalysis import org.enso.interpreter.instrument.execution.model.PendingEdit @@ -12,6 +12,7 @@ import org.enso.text.editing.model.TextEdit import org.enso.text.editing.{IndexedSource, TextEditor} import scala.collection.mutable +import scala.util.Using /** Simple editing change description. * @@ -96,12 +97,16 @@ final class ChangesetBuilder[A: TextEditor: IndexedSource]( case pending: PendingEdit.SetExpressionValue => pending.value case other: PendingEdit.ApplyEdit => other.edit.text } - AstToIr - .translateInline(Parser().run(value)) - .flatMap(_ match { - case ir: IR.Literal => Some(ir.setLocation(oldIr.location)) - case _ => None - }) + + val source = Source.newBuilder("enso", value, null).build + Using(new EnsoCompiler) { compiler => + compiler + .generateIRInline(compiler.parse(source)) + .flatMap(_ match { + case ir: IR.Literal => Some(ir.setLocation(oldIr.location)) + case _ => None + }) + }.get } oldIr match { diff --git a/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala b/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala index 3cd97b93018f..93de37894715 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala @@ -8,7 +8,7 @@ import org.enso.compiler.data.BindingsMap import org.enso.compiler.exception.CompilerError import org.enso.compiler.pass.IRPass import org.enso.interpreter.epb.EpbParser -import org.enso.syntax.text.{AST, Debug, Location} +import org.enso.syntax.text.{Debug, Location} import java.util.UUID @@ -157,14 +157,14 @@ object IR { type Identifier = UUID /** The type of external identifiers */ - type ExternalId = AST.ID + type ExternalId = UUID /** Couples a location with a possible source identifier. * * @param location the code location. * @param id the identifier for the location. */ - case class IdentifiedLocation(location: Location, id: Option[AST.ID]) { + case class IdentifiedLocation(location: Location, id: Option[UUID]) { /** @return the character index of the start of this source location. */ @@ -7635,13 +7635,13 @@ object IR { /** A representation of an Enso syntax error. * - * @param at the erroneous AST + * @param at the error location * @param reason the cause of this error * @param passData the pass metadata associated with this node * @param diagnostics compiler diagnostics for this node */ sealed case class Syntax( - at: AnyRef, + at: IdentifiedLocation, reason: Syntax.Reason, override val passData: MetadataStorage = MetadataStorage(), override val diagnostics: DiagnosticStorage = DiagnosticStorage() @@ -7653,11 +7653,9 @@ object IR { with IRKind.Primitive { override protected var id: Identifier = randomId - def ast: AST = at.asInstanceOf[AST] - /** Creates a copy of `this`. * - * @param ast the erroneous AST + * @param ast the error location * @param reason the cause of this error * @param passData the pass metadata associated with this node * @param diagnostics compiler diagnostics for this node @@ -7665,13 +7663,13 @@ object IR { * @return a copy of `this`, updated with the specified values */ def copy( - ast: AnyRef = at, + at: IdentifiedLocation = at, reason: Syntax.Reason = reason, passData: MetadataStorage = passData, diagnostics: DiagnosticStorage = diagnostics, id: Identifier = id ): Syntax = { - val res = Syntax(ast, reason, passData, diagnostics) + val res = Syntax(at, reason, passData, diagnostics) res.id = id res } @@ -7696,13 +7694,7 @@ object IR { this /** @inheritdoc */ - @annotation.nowarn - override val location: Option[IdentifiedLocation] = - at match { - case ast: AST => ast.location.map(IdentifiedLocation(_, ast.id)) - case loc: IdentifiedLocation => Some(loc) - case _ => None - } + override val location: Option[IdentifiedLocation] = Option(at) /** @inheritdoc */ override def mapExpressions(fn: Expression => Expression): Syntax = this @@ -7711,7 +7703,7 @@ object IR { override def toString: String = s""" |IR.Error.Syntax( - |ast = $at, + |at = $at, |reason = $reason, |location = $location, |passData = ${this.showPassData}, diff --git a/engine/runtime/src/main/scala/org/enso/compiler/pass/resolve/GenerateDocumentation.scala b/engine/runtime/src/main/scala/org/enso/compiler/pass/resolve/GenerateDocumentation.scala deleted file mode 100644 index 7836761dc43a..000000000000 --- a/engine/runtime/src/main/scala/org/enso/compiler/pass/resolve/GenerateDocumentation.scala +++ /dev/null @@ -1,84 +0,0 @@ -package org.enso.compiler.pass.resolve - -import org.enso.compiler.context.{InlineContext, ModuleContext} -import org.enso.compiler.core.IR -import org.enso.compiler.core.ir.MetadataStorage.ToPair -import org.enso.compiler.pass.IRPass -import org.enso.docs.generator.DocParserWrapper - -import scala.annotation.unused - -/** Generates documentation on resolved IR. - */ -case object GenerateDocumentation extends IRPass { - - /** The type of the metadata object that the pass writes to the IR. */ - override type Metadata = DocumentationComments.Doc - - /** The type of configuration for the pass. */ - override type Config = IRPass.Configuration.Default - - /** The passes that this pass depends _directly_ on to run. */ - override val precursorPasses: Seq[IRPass] = Seq(DocumentationComments) - - /** The passes that are invalidated by running this pass. */ - override val invalidatedPasses: Seq[IRPass] = Seq() - - /** Collects comments for a module and assigns them to the commented - * entities as metadata. - * - * @param ir the Enso IR to process - * @param moduleContext a context object that contains the information needed - * to process a module - * @return `ir`, possibly having made transformations or annotations to that - * IR. - */ - override def runModule( - ir: IR.Module, - moduleContext: ModuleContext - ): IR.Module = { - if (moduleContext.isGeneratingDocs) { - resolveModule(ir) - } else { - ir - } - } - - /** Acts as an identity function to conform with the interface. - * - * @param ir the Enso IR to process. - * @param inlineContext a context object that contains the information - * needed for inline evaluation. - * @return unchanged ir. - */ - override def runExpression( - ir: IR.Expression, - inlineContext: InlineContext - ): IR.Expression = ir - - /** @inheritdoc */ - override def updateMetadataInDuplicate[T <: IR]( - @unused sourceIr: T, - copyOfIr: T - ): T = copyOfIr - - // === Pass Internals ======================================================= - - /** Resolves documentation comments in a module. - * - * @param ir the module to resolve comments in - * @return `ir`, with any doc comments associated with nodes as metadata - */ - private def resolveModule(ir: IR.Module): IR.Module = { - val newBindings = ir.bindings.map { x => - val doc = x.getMetadata(DocumentationComments) - doc match { - case Some(value) => - val genDoc = DocParserWrapper.runOnPureDoc(value.documentation) - x.updateMetadata(this -->> DocumentationComments.Doc(genDoc)) - case None => x - } - } - ir.copy(bindings = newBindings) - } -} diff --git a/engine/runtime/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala b/engine/runtime/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala index e80d142ec8ff..7682e1ab6d83 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala @@ -1,12 +1,12 @@ package org.enso.compiler.phase +import org.enso.compiler.EnsoCompiler import org.enso.compiler.Passes -import org.enso.compiler.codegen.AstToIr import org.enso.compiler.context.{FreshNameSupply, ModuleContext} import org.enso.compiler.data.CompilerConfig import org.enso.interpreter.runtime.Module import org.enso.interpreter.runtime.Module.CompilationStage -import org.enso.syntax.text.Parser +import scala.util.Using /** A phase responsible for initializing the builtins' IR from the provided * source. @@ -38,8 +38,9 @@ object BuiltinsIrBuilder { freshNameSupply = Some(freshNameSupply), compilerConfig = CompilerConfig(warningsEnabled = false) ) - val parsedAst = Parser().runWithIds(module.getSource.getCharacters.toString) - val initialIr = AstToIr.translate(parsedAst) + val initialIr = Using(new EnsoCompiler) { compiler => + compiler.compile(module.getSource.getCharacters) + }.get val irAfterModDiscovery = passManager.runPassesOnModule( initialIr, moduleContext, diff --git a/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java b/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java index 1bac4211f4ba..cf95714bc68d 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java @@ -6,11 +6,7 @@ import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.function.Function; -import org.enso.compiler.codegen.AstToIr; import org.enso.compiler.core.IR; -import org.enso.syntax.text.AST.ASTOf; -import org.enso.syntax.text.Parser; -import org.enso.syntax.text.Shape; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -1314,20 +1310,7 @@ private static void parseTest(String code) throws IOException { @SuppressWarnings("unchecked") private static void parseTest(String code, boolean noIds, boolean noLocations, boolean lessDocs) throws IOException { var ir = compile(code); - - var oldAst = new Parser().runWithIds(code); - var oldIr = AstToIr.translate((ASTOf)(Object)oldAst); - - Function filter = (f) -> simplifyIR(f, noIds, noLocations, lessDocs); - var old = filter.apply(oldIr); - var now = filter.apply(ir); - if (!old.equals(now)) { - var name = findTestMethodName(); - var home = new File(System.getProperty("user.home")).toPath(); - Files.writeString(home.resolve(name + ".old") , old, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - Files.writeString(home.resolve(name + ".now") , now, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - assertEquals("IR for " + code + " shall be equal", old, now); - } + assertNotNull(ir); } private static void equivalenceTest(String code1, String code2) throws IOException { diff --git a/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java b/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java deleted file mode 100644 index 0453b2e49019..000000000000 --- a/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java +++ /dev/null @@ -1,228 +0,0 @@ -package org.enso.compiler; - -import com.oracle.truffle.api.source.Source; -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import java.util.function.Function; -import junit.framework.TestCase; -import static junit.framework.TestCase.fail; -import junit.framework.TestSuite; -import org.enso.compiler.codegen.AstToIr; -import org.enso.compiler.core.IR; -import org.enso.syntax.text.AST.ASTOf; -import org.enso.syntax.text.Parser; -import org.enso.syntax.text.Shape; -import org.junit.runner.RunWith; -import org.junit.runners.AllTests; - -@RunWith(AllTests.class) -public final class ParseStdLibTest extends TestCase { - private static final EnsoCompiler ensoCompiler = new EnsoCompiler(); - private final File where; - private final Dump dump; - - private ParseStdLibTest(String name, File where, Dump dump) { - super(name); - this.where = where; - this.dump = dump; - } - - public static TestSuite suite() throws Exception { - TestSuite s = new TestSuite(); - var os = System.getProperty("os.name"); - if (os != null && os.contains("Window")) { - s.addTest(new ParseStdLibTest("IgnoringStdLibParsingOnWindows", null, null)); - } else { - collectDistribution(s, "Base"); - } - return s; - } - - private static File file(File dir, String... relative) { - var f = dir; - for (var ch : relative) { - f = new File(f, ch); - } - return f; - } - - private static Path locateDistribution(final String name) throws URISyntaxException { - var where = - new File(ParseStdLibTest.class.getProtectionDomain().getCodeSource().getLocation().toURI()); - var dir = where; - for (; ; ) { - dir = file(where, "distribution", "lib", "Standard", name, "0.0.0-dev", "src"); - if (dir.exists()) { - break; - } - where = where.getParentFile(); - } - return dir.toPath(); - } - - private static void collectDistribution(TestSuite s, String name) throws Exception { - var dir = locateDistribution(name); - var dump = new Dump(); - class CollectSuites implements FileVisitor { - - private final TestSuite suite; - - CollectSuites(TestSuite suite) { - this.suite = suite; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!file.getFileName().toString().endsWith(".enso")) { - return FileVisitResult.CONTINUE; - } - final String name = file.toFile().getPath().substring(dir.toFile().getPath().length() + 1); - suite.addTest(new ParseStdLibTest(name, file.toFile(), dump)); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - } - Files.walkFileTree(dir, new CollectSuites(s)); - } - - @SuppressWarnings("unchecked") - private void parseTest(Source src, boolean generate) throws IOException { - var ir = ensoCompiler.compile(src); - assertNotNull("IR was generated", ir); - - var oldAst = new Parser().runWithIds(src.getCharacters().toString()); - var oldIr = AstToIr.translate((ASTOf) (Object) oldAst); - - Function filter = (f) -> EnsoCompilerTest.simplifyIR(f, true, true, true); - - var old = filter.apply(oldIr); - var now = filter.apply(ir); - if (!old.equals(now)) { - if (generate) { - dump.dump(where, old, now); - } else { - fail("IR for " + where.getName() + " shall be equal"); - } - } - } - - @Override - public void runBare() throws Throwable { - if (where == null) { - return; - } - var code = Files.readString(where.toPath()); - var src = Source.newBuilder("enso", code, getName()).uri(where.toURI()).build(); - if (isKnownToWork(getName())) { - parseTest(src, true); - } else { - try { - parseTest(src, false); - } catch (Exception | Error e) { - // OK - return; - } - fail("This test isn't known to work!"); - } - } - - private static final Set SHOULD_FAIL; - - static { - SHOULD_FAIL = new HashSet<>(); - SHOULD_FAIL.addAll( - Arrays.asList( - // Files containing type expressions not supported by old parser. - "Data/Index_Sub_Range.enso", - "Data/Json.enso", - "Data/Json/Extensions.enso", - "Data/List.enso", - "Data/Pair.enso", - "Data/Range.enso", - "Data/Sort_Column_Selector.enso", - "Data/Text/Extensions.enso", - "Data/Text/Regex/Match_2.enso", - "Data/Text/Regex/Pattern_2.enso", - "Data/Text/Regex/Regex_Mode.enso", - "Data/Value_Type.enso", - "Data/Vector.enso", - "Data/Statistics.enso", - "Network/HTTP/HTTP_Status_Code.enso", - "Internal/Base_Generator.enso", - "System/File.enso")); - } - - private static boolean isKnownToWork(String name) { - return !SHOULD_FAIL.contains(name); - } - - private static final class Dump { - private boolean first = true; - - public void dump(File where, CharSequence old, CharSequence now) throws IOException { - var name = where.getName(); - var result = where.getParentFile().toPath(); - final Path oldPath = result.resolve(name + ".old"); - Files.writeString( - oldPath, - old, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE, - StandardOpenOption.WRITE); - final Path nowPath = result.resolve(name + ".now"); - Files.writeString( - nowPath, - now, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE, - StandardOpenOption.WRITE); - if (first) { - first = false; - fail( - "IR for " - + where.getName() - + " shall be equal:\n$ diff -u '" - + oldPath - + "' '" - + nowPath - + "'\n ===== Old =====\n" - + old - + "\n===== Now =====\n" - + now); - } - fail( - "IR for " - + where.getName() - + " shall be equal:\n$ diff -u '" - + oldPath - + "' '" - + nowPath - + "'"); - } - } -} diff --git a/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java b/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java index a775af6587e8..c6ce59260ca2 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java @@ -1,6 +1,5 @@ package org.enso.compiler; -import org.enso.docs.sections.DocSectionsBuilder; import org.enso.interpreter.runtime.EnsoContext; import org.enso.pkg.PackageManager; import org.enso.polyglot.LanguageInfo; @@ -56,8 +55,7 @@ public void testSerializationOfFQNs() throws Exception { ctx.enter(); var result = compiler.run(module); assertEquals(result.compiledModules().exists(m -> m == module), true); - var serializationManager = - new SerializationManager(ensoContext.getCompiler(), DocSectionsBuilder.apply()); + var serializationManager = new SerializationManager(ensoContext.getCompiler()); var future = serializationManager.serializeModule(module, true); var serialized = future.get(5, TimeUnit.SECONDS); assertEquals(serialized, true); diff --git a/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java b/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java index 95ae60230ae3..de24872d44db 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java @@ -20,8 +20,6 @@ public void testSerdeOfSuggestion() throws Exception { Object shape = new Suggestion.Module( "SampleModule", Option.apply("doc"), - Option.apply("html"), - Option.empty(), Option.empty() ); final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); @@ -40,8 +38,6 @@ public void testArraySerdeOfSuggestion() throws Exception { Object shape = new Suggestion[]{new Suggestion.Module( "SampleModule", Option.apply("doc"), - Option.apply("html"), - Option.empty(), Option.empty() )}; final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); @@ -65,8 +61,6 @@ public void testRecordSerdeOfSuggestion() throws Exception { Object shape = new SuggestionCache(11, List.of(new Suggestion.Module( "SampleModule", Option.apply("doc"), - Option.apply("html"), - Option.empty(), Option.empty() ))); final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/CompilerTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/CompilerTest.scala index c92a5ed7dbf2..313c8cbd7960 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/CompilerTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/CompilerTest.scala @@ -1,7 +1,6 @@ package org.enso.compiler.test import org.enso.compiler.EnsoCompiler -import org.enso.compiler.codegen.AstToIr import org.enso.compiler.context.{FreshNameSupply, InlineContext, ModuleContext} import org.enso.compiler.core.IR import org.enso.compiler.core.ir.MetadataStorage.ToPair @@ -9,7 +8,6 @@ import org.enso.compiler.data.BindingsMap.ModuleReference import org.enso.compiler.data.{BindingsMap, CompilerConfig} import org.enso.compiler.pass.analyse.BindingAnalysis import org.enso.compiler.pass.{PassConfiguration, PassManager} -import org.enso.syntax.text.{AST, Parser} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import org.enso.interpreter.runtime.Module @@ -18,40 +16,8 @@ import org.enso.pkg.QualifiedName trait CompilerTest extends AnyWordSpecLike with Matchers with CompilerRunner trait CompilerRunner { - - def useRustParser: Boolean = false - // === IR Utilities ========================================================= - /** Adds an extension method for converting a string to its AST - * representation. - * - * @param source the source code to convert - */ - implicit class ToAST(source: String) { - - /** Produces the [[AST]] representation of [[source]]. - * - * @return [[source]] as an AST - */ - def toAst: AST = { - val parser: Parser = Parser() - val unresolvedAST = parser.runWithIds(source) - - parser.dropMacroMeta(unresolvedAST) - } - - /** Produces the [[AST]] representation of [[source]] without dropping the - * macro metadata. - * - * @return [[source]] as an AST - */ - def toAstWithMeta: AST = { - val parser: Parser = Parser() - parser.runWithIds(source) - } - } - /** An extension method to allow converting string source code to IR as a * module. * @@ -64,13 +30,9 @@ trait CompilerRunner { * @return the [[IR]] representing [[source]] */ def toIrModule: IR.Module = { - if (useRustParser) { - val compiler = new EnsoCompiler() - try compiler.compile(source) - finally compiler.close() - } else { - AstToIr.translate(source.toAst) - } + val compiler = new EnsoCompiler() + try compiler.compile(source) + finally compiler.close() } } @@ -86,7 +48,9 @@ trait CompilerRunner { * @return the [[IR]] representing [[source]], if it is a valid expression */ def toIrExpression: Option[IR.Expression] = { - AstToIr.translateInline(source.toAst) + val compiler = new EnsoCompiler() + try compiler.generateIRInline(compiler.parse(source)) + finally compiler.close() } } diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/PassesTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/PassesTest.scala index c39dc2814487..7db5f1a6e4b7 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/PassesTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/PassesTest.scala @@ -52,7 +52,6 @@ class PassesTest extends CompilerTest { FunctionBinding, GenerateMethodBodies, BindingAnalysis, - GenerateDocumentation, ModuleNameConflicts, MethodDefinitions, SectionsToBinOp, diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/AstToIrTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/AstToIrTest.scala deleted file mode 100644 index 9e98e0171295..000000000000 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/AstToIrTest.scala +++ /dev/null @@ -1,1222 +0,0 @@ -package org.enso.compiler.test.codegen - -import org.enso.compiler.core.IR -import org.enso.compiler.core.IR.Error.Syntax -import org.enso.compiler.core.IR.Module.Scope.Definition.SugaredType -import org.enso.compiler.test.CompilerTest -import org.scalatest.Inside - -class AstToIrTest extends CompilerTest with Inside { - - "AST translation of operators" should { - "disallow named arguments to operators" in { - val ir = - """ - |(a = 1) + 10 - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error.Syntax] - ir.asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.NamedArgInOperator.type] - } - } - - "AST translation of operator sections" should { - "work properly for left sections" in { - val ir = - """ - |(1 +) - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Section.Left] - ir.location shouldBe defined - } - - "work properly for sides sections" in { - val ir = - """ - |(+) - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Section.Sides] - // TODO[DB] Section.Sides location is not parsed - //ir.location shouldBe defined - } - - "work properly for right sections" in { - val ir = - """ - |(+ 1) - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Section.Right] - ir.location shouldBe defined - } - - "disallow sections with named arguments" in { - val ir = - """ - |(+ (left=1)) - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error.Syntax] - ir.asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.NamedArgInSection.type] - } - } - - "AST translation of function applications" should { - "allow use of blank arguments" in { - val ir = - """ - |a b _ d - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Prefix] - - ir.arguments(1) shouldBe an[IR.CallArgument.Specified] - ir.arguments(1) - .asInstanceOf[IR.CallArgument.Specified] - .value shouldBe an[IR.Name.Blank] - } - - "allow use of named blank arguments" in { - val ir = - """ - |a b (f = _) c - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Prefix] - - ir.arguments(1) shouldBe an[IR.CallArgument.Specified] - ir.arguments(1) - .asInstanceOf[IR.CallArgument.Specified] - .value shouldBe an[IR.Name.Blank] - } - - "allow method-call syntax on a blank" in { - val ir = - """ - |_.foo a b - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Prefix] - - ir.arguments.head shouldBe an[IR.CallArgument.Specified] - ir.arguments.head - .asInstanceOf[IR.CallArgument.Specified] - .value shouldBe an[IR.Name.Blank] - } - - "allow functions in applications to be blanks" in { - val ir = - """ - |_ a b c - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Prefix] - - ir.function shouldBe an[IR.Name.Blank] - } - } - - "AST translation of case expressions" should { - "support a blank scrutinee" in { - val ir = - """ - |case _ of - | Cons a b -> a + b - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.scrutinee shouldBe an[IR.Name.Blank] - } - - "support constructor patterns" in { - val ir = - """ - |case foo of - | Cons a b -> a + b - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Constructor] - - val consPat = - ir.branches.head.pattern.asInstanceOf[IR.Pattern.Constructor] - - consPat.constructor.name shouldEqual "Cons" - consPat.fields.length shouldEqual 2 - - consPat.fields.head shouldBe an[IR.Pattern.Name] - consPat.fields(1) shouldBe an[IR.Pattern.Name] - } - - "support catch all patterns" in { - val ir = - """ - |case foo of - | _ -> 10 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Name] - ir.branches.head.pattern - .asInstanceOf[IR.Pattern.Name] - .name shouldBe an[IR.Name.Blank] - } - - "support named catch all patterns" in { - val ir = - """ - |case foo of - | a -> a - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Name] - ir.branches.head.pattern - .asInstanceOf[IR.Pattern.Name] - .name - .name shouldEqual "a" - } - - "support nested patterns" in { - val ir = - """ - |case foo of - | Cons (Cons a b) _ -> a + b - | Cons a Nil -> a - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Constructor] - ir.branches(1).pattern shouldBe an[IR.Pattern.Constructor] - - val branch1 = - ir.branches.head.pattern.asInstanceOf[IR.Pattern.Constructor] - branch1.constructor.name shouldEqual "Cons" - branch1.fields.length shouldEqual 2 - branch1.fields.head shouldBe an[IR.Pattern.Constructor] - branch1.fields(1) shouldBe an[IR.Pattern.Name] - - val branch1Field1 = - branch1.fields.head.asInstanceOf[IR.Pattern.Constructor] - branch1Field1.fields.length shouldEqual 2 - - val branch2 = - ir.branches(1).pattern.asInstanceOf[IR.Pattern.Constructor] - - branch2.constructor.name shouldEqual "Cons" - branch2.fields.length shouldEqual 2 - branch2.fields.head shouldBe an[IR.Pattern.Name] - branch2.fields(1) shouldBe an[IR.Pattern.Constructor] - } - - "support constructor-only patterns" in { - val ir = - """ - |case foo of - | Nil -> 10 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - val pattern = ir.branches.head.pattern - pattern shouldBe an[IR.Pattern.Constructor] - } - - "support literal numeric patterns" in { - val ir = - """ - |case foo of - | 1 -> 10 - | 2 -> 20 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches(0).pattern shouldBe an[IR.Pattern.Literal] - ir.branches(1).pattern shouldBe an[IR.Pattern.Literal] - } - - "support constructor patterns with nested literals" in { - val ir = - """ - |case foo of - | Cons 1 b -> a + b - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Constructor] - - val consPat = - ir.branches.head.pattern.asInstanceOf[IR.Pattern.Constructor] - - consPat.constructor.name shouldEqual "Cons" - consPat.fields.length shouldEqual 2 - - consPat.fields(0) shouldBe an[IR.Pattern.Literal] - consPat.fields(1) shouldBe an[IR.Pattern.Name] - } - - "support type patterns " in { - val ir = - """ - |case foo of - | f : Foo -> process_foo f - | b : Bar -> process_bar b - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Type] - - val tpePattern1 = - ir.branches(0).pattern.asInstanceOf[IR.Pattern.Type] - - tpePattern1.name.name shouldEqual "f" - tpePattern1.tpe.name shouldEqual "Foo" - - val tpePattern2 = - ir.branches(1).pattern.asInstanceOf[IR.Pattern.Type] - - tpePattern2.name.name shouldEqual "b" - tpePattern2.tpe.name shouldEqual "Bar" - } - - "support type patterns nested in constructor pattern " in { - val ir = - """ - |case foo of - | Cons (f : Foo) b -> process_foo f b - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Case.Expr] - - ir.branches.head.pattern shouldBe an[IR.Pattern.Constructor] - - val consPattern = - ir.branches(0).pattern.asInstanceOf[IR.Pattern.Constructor] - - consPattern.constructor.name shouldEqual "Cons" - consPattern.fields.length shouldEqual 2 - - consPattern.fields(0) shouldBe an[IR.Pattern.Type] - consPattern.fields(1) shouldBe an[IR.Pattern.Name] - } - - } - - "AST translation of function definitions" should { - "support ignored arguments" in { - val ir = - """ - |_ -> a -> a + 20 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Function.Lambda] - - ir.arguments.head shouldBe an[IR.DefinitionArgument.Specified] - val blankArg = - ir.arguments.head.asInstanceOf[IR.DefinitionArgument.Specified] - blankArg.name shouldBe an[IR.Name.Blank] - } - - "support suspended ignored arguments" in { - val ir = - """ - |~_ -> a -> a + 20 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Function.Lambda] - - ir.arguments.head shouldBe an[IR.DefinitionArgument.Specified] - val blankArg = - ir.arguments.head.asInstanceOf[IR.DefinitionArgument.Specified] - blankArg.name shouldBe an[IR.Name.Blank] - } - - "support ignored arguments with defaults" in { - val ir = - """ - |(_ = 10) -> a -> a + 20 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Function.Lambda] - - ir.arguments.head shouldBe an[IR.DefinitionArgument.Specified] - val blankArg = - ir.arguments.head.asInstanceOf[IR.DefinitionArgument.Specified] - blankArg.name shouldBe an[IR.Name.Blank] - } - - "result in a syntax error when defined with multiple arguments" in { - val ir = - """x y -> x + y - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error.Syntax] - - ir.asInstanceOf[IR.Error.Syntax].message shouldEqual - "Syntax is not supported yet: pattern matching function arguments." - } - - "support standard lambda chaining" in { - val ir = - """ - |x -> y -> z -> x - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Lambda] - ir.asInstanceOf[IR.Function.Lambda].body shouldBe an[IR.Function.Lambda] - ir.asInstanceOf[IR.Function.Lambda] - .body - .asInstanceOf[IR.Function.Lambda] - .body shouldBe an[IR.Function.Lambda] - } - - "support ascribed argument definitions" in { - val ir = - """ - |(~x : Type = Type.default) -> x.do_thing - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Lambda] - val args = ir.asInstanceOf[IR.Function.Lambda].arguments - args.length shouldEqual 1 - val firstArg = args.head - firstArg.name.name shouldEqual "x" - firstArg.ascribedType shouldBe defined - firstArg.ascribedType.get.asInstanceOf[IR.Name].name shouldEqual "Type" - firstArg.defaultValue shouldBe defined - firstArg.suspended shouldBe true - } - } - - "AST translation of bindings" should { - "allow ignored bindings" in { - val ir = - """ - |_ = foo a b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Expression.Binding] - val binding = ir.asInstanceOf[IR.Expression.Binding] - - binding.name shouldBe an[IR.Name.Blank] - } - } - - "AST translation of unary minus" should { - "work when parenthesised" in { - val ir = - """ - |(-1) - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Literal.Number] - ir.asInstanceOf[IR.Literal.Number].value shouldEqual "-1" - } - - "work when not parenthesised" in { - val ir = - """ - |-100 - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Literal.Number] - ir.asInstanceOf[IR.Literal.Number].value shouldEqual "-100" - } - - "work on non-literals" in { - val ir = - """ - |-foo - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Prefix] - - val fn = ir.asInstanceOf[IR.Application.Prefix] - fn.function shouldEqual IR.Name.Literal( - "negate", - isMethod = true, - None - ) - - val fooArg = fn.arguments.head.asInstanceOf[IR.CallArgument.Specified] - fooArg.value shouldBe an[IR.Name.Literal] - fooArg.value.asInstanceOf[IR.Name.Literal].name shouldEqual "foo" - } - } - - "AST translation of function sugar" should { - "work for function definitions" in { - val ir = - """ - |f a b = a + b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Binding] - } - - "work for method definitions" in { - val ir = - """ - |Foo.bar a b = a + b - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.Method.Binding] - } - - "work for method definitions with involved arguments" in { - val ir = - """ - |Foo.bar _ (b = 1) ~c = b + c - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.Method.Binding] - } - - "work for method definitions with ascribed arguments" in { - val ir = - """ - |T.f a:b (~c : d = 1) = a + c - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.Method.Binding] - - val method = - ir.bindings.head.asInstanceOf[IR.Module.Scope.Definition.Method.Binding] - - val args = method.arguments - args.length shouldEqual 2 - - val firstArg = args.head - firstArg.name.name shouldEqual "a" - firstArg.ascribedType shouldBe defined - firstArg.ascribedType.get.asInstanceOf[IR.Name].name shouldEqual "b" - firstArg.defaultValue should not be defined - firstArg.suspended shouldBe false - - val secondArg = args(1) - secondArg.name.name shouldEqual "c" - secondArg.ascribedType shouldBe defined - secondArg.ascribedType.get.asInstanceOf[IR.Name].name shouldEqual "d" - secondArg.defaultValue shouldBe defined - secondArg.suspended shouldBe true - } - - "work for method definitions with operator names" in { - val bindings = - """ - |My.== : My -> Boolean - |My.== that = this.a == that.a - |""".stripMargin.toIrModule.bindings - - val tpIr = bindings.head - tpIr shouldBe a[IR.Type.Ascription] - val tp = tpIr.asInstanceOf[IR.Type.Ascription] - tp.typed shouldBe a[IR.Name.MethodReference] - val methodRef = tp.typed.asInstanceOf[IR.Name.MethodReference] - methodRef.typePointer.get.name shouldEqual "My" - methodRef.methodName.name shouldEqual "==" - - val methodIr = bindings(1) - methodIr shouldBe a[IR.Module.Scope.Definition.Method.Binding] - val method = - methodIr.asInstanceOf[IR.Module.Scope.Definition.Method.Binding] - method.methodReference.methodName.name shouldEqual "==" - method.methodReference.typePointer.get.name shouldEqual "My" - } - - "not recognise pattern match bindings" in { - val ir = - """ - |F a b = a + b - |""".stripMargin.toIrExpression.get - - ir should not be an[IR.Function.Binding] - } - - "work with ignored arguments" in { - val ir = - """ - |f _ b = a + b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Binding] - } - - "work with defaulted arguments" in { - val ir = - """ - |f (a = 1) b = a + b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Binding] - } - - "work with lazy arguments" in { - val ir = - """ - |f ~a b = a + b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Binding] - } - - "work with ascribed arguments" in { - val ir = - """ - |f a:b (~c : d = 1) = a + c - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Function.Binding] - val func = ir.asInstanceOf[IR.Function.Binding] - val args = func.arguments - args.length shouldEqual 2 - - val firstArg = args.head - firstArg.name.name shouldEqual "a" - firstArg.ascribedType shouldBe defined - firstArg.ascribedType.get.asInstanceOf[IR.Name].name shouldEqual "b" - firstArg.defaultValue should not be defined - firstArg.suspended shouldBe false - - val secondArg = args(1) - secondArg.name.name shouldEqual "c" - secondArg.ascribedType shouldBe defined - secondArg.ascribedType.get.asInstanceOf[IR.Name].name shouldEqual "d" - secondArg.defaultValue shouldBe defined - secondArg.suspended shouldBe true - } - } - - "AST translation for atom definitions" should { - "work for types with no arguments" in { - val ir = - """ - |type My_Type - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.SugaredType] - val tp = - ir.bindings.head.asInstanceOf[IR.Module.Scope.Definition.SugaredType] - tp.name.name shouldEqual "My_Type" - } - - "work for types with arguments" in { - val ir = - """ - |type My_Type x - | Data a b c - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.SugaredType] - val atom = ir.bindings.head - .asInstanceOf[IR.Module.Scope.Definition.SugaredType] - .body - .head - .asInstanceOf[IR.Module.Scope.Definition.Data] - atom.name.name shouldEqual "Data" - val args = atom.arguments - args.length shouldEqual 3 - args.head.name.name shouldEqual "a" - args(1).name.name shouldEqual "b" - args(2).name.name shouldEqual "c" - } - - "raise an error for atoms with lazy arguments" in { - val ir = - """ - |type My_Type ~a - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Error.Syntax] - val error = ir.bindings.head.asInstanceOf[IR.Error.Syntax] - error.reason shouldBe an[IR.Error.Syntax.SuspendedArgInAtom.type] - } - - "work for atoms with ascribed arguments" in { - val ir = - """ - |type My_Type - | Data a:b (c : d = 1) - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.SugaredType] - val atom = ir.bindings.head - .asInstanceOf[SugaredType] - .body - .head - .asInstanceOf[IR.Module.Scope.Definition.Data] - atom.name.name shouldEqual "Data" - val args = atom.arguments - args.length shouldEqual 2 - - val firstArg = args.head - firstArg.name.name shouldEqual "a" - firstArg.ascribedType shouldBe defined - firstArg.defaultValue should not be defined - - val secondArg = args(1) - secondArg.name.name shouldEqual "c" - secondArg.ascribedType shouldBe defined - secondArg.defaultValue shouldBe defined - } - } - - "AST translation for the inline flow" should { - "disallow method definitions without exploding" in { - val ir = - """ - |Unit.foo a b = a + b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error.Syntax] - ir.asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.MethodDefinedInline] - } - - "disallow type definitions without exploding" in { - val ir = - """ - |type MyAtom a b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error.Syntax] - ir.asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.TypeDefinedInline] - } - } - - "AST translation for type definitions" should { - "translate atoms properly" in { - val ir = - """ - |type MyAtom a b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Module.Scope.Definition.SugaredType] - } - - "translate complex type defs properly" in { - val ir = - """ - |type Maybe - | Nothing - | Just a - | - | is_just = case this of - | Just _ -> true - | Nothing -> false - | - | fn a b = a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Module.Scope.Definition.SugaredType] - - val typeDef = ir.asInstanceOf[IR.Module.Scope.Definition.SugaredType] - - typeDef.name.name shouldEqual "Maybe" - typeDef.arguments.length shouldEqual 0 - - typeDef.body.head shouldBe an[IR.Module.Scope.Definition.Data] - typeDef.body(1) shouldBe an[IR.Module.Scope.Definition.Data] - typeDef.body(2) shouldBe an[IR.Expression.Binding] - typeDef.body(3) shouldBe an[IR.Function.Binding] - } - - "disallow unexpected expressions in the type body" in { - val ir = - """ - |type Maybe - | Nothing - | type Just a - | - | F a b = _ - | - | case foo of - | X -> 1 - | - | fn a b = a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Module.Scope.Definition.SugaredType] - - val typeDef = ir.asInstanceOf[IR.Module.Scope.Definition.SugaredType] - - typeDef.body(2) shouldBe an[IR.Error.Syntax] - typeDef - .body(2) - .asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.UnexpectedDeclarationInType.type] - typeDef.body(3) shouldBe an[IR.Error.Syntax] - typeDef - .body(3) - .asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[IR.Error.Syntax.UnexpectedDeclarationInType.type] - } - - "allow defining methods with operator names" in { - val body = - """ - |type My - | type My a - | - | + : My -> My - | + that = My this.a+that.a - |""".stripMargin.toIrModule.bindings.head - .asInstanceOf[IR.Module.Scope.Definition.SugaredType] - .body - - body(1) shouldBe an[IR.Type.Ascription] - body(2) shouldBe an[IR.Function.Binding] - val fun = body(2).asInstanceOf[IR.Function.Binding] - fun.name.name shouldEqual "+" - } - } - - "AST translation for documentation comments" should { - "work at the top level" in { - val ir = - """ - |## Some documentation for foo - |foo a b = a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Comment.Documentation] - ir.asInstanceOf[IR.Comment.Documentation] - .doc shouldEqual " Some documentation for foo" - } - - "work within top-level blocks" in { - val ir = - """ - |a -> - | ## Some docs for b - | b = 1 - | 10 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Function.Lambda] - - val comment = ir.body - .asInstanceOf[IR.Expression.Block] - .expressions - .head - comment shouldBe an[IR.Comment.Documentation] - comment - .asInstanceOf[IR.Comment.Documentation] - .doc shouldEqual " Some docs for b" - } - } - - "AST translation for type operators" should { - "support type ascription" in { - val ir = - """ - |a : Type - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support context ascription" in { - val ir = - """ - |a in IO - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support error ascription" in { - val ir = - """ - |IO ! FileNotFound - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the subsumption operator" in { - val ir = - """ - |IO.Read <: IO - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the equality operator" in { - val ir = - """ - |T ~ Q - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the concatenation operator" in { - val ir = - """ - |a ; b - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the union operator" in { - val ir = - """ - |A | B - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the intersection operator" in { - val ir = - """ - |A & B - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "support the minus operator" in { - val ir = - """ - |A \ B - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - } - - "AST translation for typeset literals" should { - "work properly" in { - val ir = - """ - |{ x := 0 ; y := 0 } - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Literal.Typeset] - } - } - - "AST translation for top-level type signatures" should { - "work for method signatures" in { - val ir = - """ - |MyAtom.foo : Number -> Number -> Number - |MyAtom.foo = a -> b -> a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Type.Ascription] - - val asc = ir.asInstanceOf[IR.Type.Ascription] - asc.typed shouldBe an[IR.Name.MethodReference] - } - - "work for module-method signatures" in { - val ir = - """ - |foo : Number -> Number - |foo a = a - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Type.Ascription] - val asc = ir.asInstanceOf[IR.Type.Ascription] - asc.typed shouldBe an[IR.Name.MethodReference] - } - - "work with sugared syntax" in { - val ir = - """ - |MyAtom.foo : Number -> Number -> Number - |MyAtom.foo a b = a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Type.Ascription] - - val asc = ir.asInstanceOf[IR.Type.Ascription] - asc.typed shouldBe an[IR.Name.MethodReference] - } - - "result in a syntax error if not valid" in { - val ir = - """ - |a b : Number -> Number -> Number - |MyAtom.foo a b = a + b - |""".stripMargin.toIrModule.bindings.head - - ir shouldBe an[IR.Error.Syntax] - ir.asInstanceOf[IR.Error.Syntax] - .reason shouldBe an[Syntax.InvalidStandaloneSignature.type] - } - - "properly support dotted operators in ascriptions" in { - val ir = - """with_output_stream : Vector.Vector -> Any ! File_Error - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Operator.Binary] - - ir.right.value - .asInstanceOf[IR.Type.Function] - .args(0) shouldBe an[IR.Name.Qualified] - } - - "work inside type bodies" in { - val ir = - """ - |type MyType - | type MyAtom - | - | foo : this -> integer - | foo = 0 - |""".stripMargin.toIrModule.bindings.head - .asInstanceOf[IR.Module.Scope.Definition.SugaredType] - - ir.body.length shouldEqual 3 - ir.body(1) shouldBe an[IR.Type.Ascription] - } - } - - "AST translation for expression-level type signatures" should { - "work in block contexts" in { - val ir = - """ - |x : Number - |x = 10 - |""".stripMargin.toIrExpression.get.asInstanceOf[IR.Expression.Block] - - ir.expressions.head shouldBe an[IR.Application.Operator.Binary] - } - - "work in expression contexts" in { - val ir = - """ - |(a + b) : Number - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Application.Operator.Binary] - } - - "work properly when used in assignments" in { - val ir = - """ - |x = a : Number - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Expression.Binding] - - ir.asInstanceOf[IR.Expression.Binding] - .expression shouldBe an[IR.Application.Operator.Binary] - } - - // TODO [AA] Syntax error with `f a ->` - - "properly support dotted operators in ascriptions" in { - val ir = - """with_output_stream : Vector.Vector -> Any ! File_Error - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Operator.Binary] - - ir.right.value - .asInstanceOf[IR.Type.Function] - .args(0) shouldBe an[IR.Name.Qualified] - } - - "properly support the `in` context ascription operator" in { - val ir = - """ - |x : Number in (Maybe | IO) - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Operator.Binary] - - ir.right.value shouldBe an[IR.Application.Operator.Binary] - ir.right.value - .asInstanceOf[IR.Application.Operator.Binary] - .operator - .name shouldEqual "in" - } - - "properly support the `!` error ascription operator" in { - val ir = - """ - |x : Number in IO ! OverflowError - |""".stripMargin.toIrExpression.get - .asInstanceOf[IR.Application.Operator.Binary] - .right - .value - - ir shouldBe an[IR.Application.Operator.Binary] - ir.asInstanceOf[IR.Application.Operator.Binary] - .operator - .name shouldEqual "!" - } - } - - "AST translation of top-level annotations" should { - "support annotations at the top level" in { - val ir = - """@My_Annotation - |type Foo a b - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Name.BuiltinAnnotation] - ir.bindings(1) shouldBe an[IR.Module.Scope.Definition.SugaredType] - } - - "support annotations inside complex type bodies" in { - val ir = - """type My_Type - | @My_Annotation - | type Foo - | - | @My_Annotation - | add a = this + a - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Module.Scope.Definition.SugaredType] - val complexType = - ir.bindings.head.asInstanceOf[IR.Module.Scope.Definition.SugaredType] - - complexType.body.head shouldBe an[IR.Name.BuiltinAnnotation] - complexType.body(2) shouldBe an[IR.Name.BuiltinAnnotation] - } - } - - "AST translation for imports and exports" should { - "properly support different kinds of imports" in { - val imports = List( - "import username.Foo.Bar as Baz", - "import project.Foo.Bar", - "from project import all", - "from Username.Bar.Quux import Baz", - "from Username.Bar.Test import Baz, Spam", - "from Username.Bar.Test import Baz, Spam, foo, Bar", - "from Username.Bar.Test import foo, bar", - "from username.Foo.Bar import all", - "from username.Foo.Bar as Eggs import all hiding Spam", - "from project.Foo.Bar import all hiding Spam, Eggs" - ) - imports - .mkString("\n") - .toIrModule - .imports - .map(_.showCode()) shouldEqual imports - } - - "properly support different kinds of exports" in { - val exports = List( - "export Username.Bar as Baz", - "export project.Bar", - "from Username.Bar export Baz", - "from username.Bar export baz, Spam", - "from project.Bar export all", - "from username.Bar as Eggs export all hiding Spam", - "from Username.Bar export all hiding Spam, eggs" - ) - exports - .mkString("\n") - .toIrModule - .exports - .map(_.showCode()) shouldEqual exports - } - } - - "AST translation of erroneous constructs" should { - "result in a syntax error when encountering " + - "unbalanced parentheses in application" in { - val ir = - """type MyAtom - | - |main = - | f = case _ of - | Cons (Cons MyAtom Nil) Nil -> 100 - | _ -> 50 - | f (Cons (Cons MyAtom Nil) Nil - |""".stripMargin.toIrModule - - inside(ir.bindings(1)) { - case binding: IR.Module.Scope.Definition.Method.Binding => - inside(binding.body) { case block: IR.Expression.Block => - inside(block.returnValue) { - case application: IR.Application.Prefix => - inside(application.arguments.head) { - case argument: IR.CallArgument.Specified => - inside(argument.value) { case error: IR.Error.Syntax => - error.reason shouldBe - IR.Error.Syntax.AmbiguousExpression - } - } - } - } - } - } - - "result in a syntax error when encountering " + - "unbalanced parentheses in a type definition" in { - val ir = - """type Maybe - | type Nothing - | type Just a - | (() - |""".stripMargin.toIrModule - inside(ir.bindings.head) { - case definition: IR.Module.Scope.Definition.SugaredType => - inside(definition.body(2)) { case error: IR.Error.Syntax => - error.reason shouldBe IR.Error.Syntax.UnexpectedDeclarationInType - } - } - } - - "result in a syntax error when encountering " + - "unbalanced parentheses in a pattern" in { - val ir = - """type MyAtom - | - |main = - | f = case _ of - | (Cons (Cons MyAtom Nil) Nil -> 100 - | _ -> 50 - |""".stripMargin.toIrModule - inside(ir.bindings(1)) { - case main: IR.Module.Scope.Definition.Method.Binding => - inside(main.body) { case block: IR.Expression.Block => - inside(block.returnValue) { case f: IR.Expression.Binding => - inside(f.expression) { case app: IR.Application.Prefix => - inside(app.arguments(1)) { - case arg: IR.CallArgument.Specified => - inside(arg.value) { case argBlock: IR.Expression.Block => - inside(argBlock.expressions.head) { - case error: IR.Error.Syntax => - error.reason shouldBe - IR.Error.Syntax.AmbiguousExpression - } - } - } - } - } - } - } - } - } - - "Bad macro invocations" should { - "result in syntax errors for imports" in { - val ir = - """import - | - |main = "foo" - |""".stripMargin.toIrModule - - ir.bindings.head shouldBe an[IR.Error] - } - - "result in syntax errors for lambdas with no body" in { - val ir = - """a -> - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error] - } - - "result in syntax errors for lambdas with no args" in { - val ir = - """-> a - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Error] - } - } - - "AST translation of text" should { - "not need any escapes" in { - val ir = - """"Foo bar \ \n baz" - |""".stripMargin.toIrExpression.get - - ir shouldBe an[IR.Literal.Text] - ir.asInstanceOf[IR.Literal.Text].text shouldEqual "Foo bar \\ \\n baz" - } - } -} diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/MatchersTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/MatchersTest.scala deleted file mode 100644 index e0c367405a75..000000000000 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/codegen/MatchersTest.scala +++ /dev/null @@ -1,116 +0,0 @@ -package org.enso.compiler.test.codegen - -import org.enso.compiler.codegen.AstView -import org.enso.compiler.test.CompilerTest -import org.enso.data.List1 -import org.enso.syntax.text.AST -import org.enso.syntax.text.AST.Module -import org.scalatest.Inside -import org.scalatest.matchers.{BeMatcher, MatchResult} - -class MatchersTest extends CompilerTest with Inside { - implicit class ToSimpleAST(source: String) { - - /** Produces the [[AST]] representation of [[source]] and strips the outer Module. - * - * @return [[source]] as an AST expression - */ - def toSimpleAst: AST = { - val ast = source.toAst - - inside(ast) { case Module(List1(line, _)) => - line.elem.get - } - } - } - - class GroupMatcher extends BeMatcher[AST] { - def apply(left: AST) = - MatchResult( - AST.Group.unapply(left).flatten.isDefined, - left.toString + " was not a group", - left.toString + " was a group" - ) - } - - val group = new GroupMatcher - - "Parensed" should { - "match a single paren" in { - inside("(x)".toSimpleAst) { case AstView.Parensed(_) => } - } - - "not match an expression without parentheses" in { - "x".toSimpleAst match { - case AstView.Parensed(_) => fail("should not match") - case _ => - } - } - - "peel-off exactly one paren" in { - val twoParens = "((x))".toSimpleAst - inside(twoParens) { case AstView.Parensed(oneParen) => - inside(oneParen) { case AstView.Parensed(zeroParens) => - zeroParens should not be group - } - } - } - } - - "MaybeParensed" should { - "match a single paren" in { - inside("(x)".toSimpleAst) { case AstView.MaybeParensed(_) => } - } - - "also match an expression without parens" in { - inside("x".toSimpleAst) { case AstView.MaybeParensed(_) => } - } - - "peel-off at-most one paren" in { - val twoParens = "((x))".toSimpleAst - inside(twoParens) { case AstView.MaybeParensed(oneParen) => - oneParen shouldBe group - inside(oneParen) { case AstView.MaybeParensed(zeroParens) => - zeroParens should not be group - } - } - } - } - - "ManyParensed" should { - "match a single paren" in { - inside("(x)".toSimpleAst) { case AstView.ManyParensed(_) => } - } - - "not match an expression without parentheses" in { - "x".toSimpleAst match { - case AstView.ManyParensed(_) => fail("should not match") - case _ => - } - } - - "peel-off all parens" in { - val twoParens = "((x))".toSimpleAst - inside(twoParens) { case AstView.ManyParensed(zeroParens) => - zeroParens should not be group - } - } - } - - "MaybeManyParensed" should { - "match a single paren" in { - inside("(x)".toSimpleAst) { case AstView.MaybeManyParensed(_) => } - } - - "also match an expression without parens" in { - inside("x".toSimpleAst) { case AstView.MaybeManyParensed(_) => } - } - - "peel-off all parens" in { - val twoParens = "((x))".toSimpleAst - inside(twoParens) { case AstView.MaybeManyParensed(zeroParens) => - zeroParens.shape should not be group - } - } - } -} diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala index fd99c9554bee..1a4c827c14e3 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala @@ -2185,8 +2185,6 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers { ), "Unnamed.Test.A", None, - None, - None, None ), Vector() @@ -2204,8 +2202,6 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers { "Standard.Base.Any.Any", false, None, - None, - None, None ), Vector() diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/core/ir/DiagnosticStorageTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/core/ir/DiagnosticStorageTest.scala index 630003a24086..a9a68154e8ae 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/core/ir/DiagnosticStorageTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/core/ir/DiagnosticStorageTest.scala @@ -3,7 +3,6 @@ package org.enso.compiler.test.core.ir import org.enso.compiler.core.IR import org.enso.compiler.core.ir.DiagnosticStorage import org.enso.compiler.test.CompilerTest -import org.enso.syntax.text.AST class DiagnosticStorageTest extends CompilerTest { @@ -80,8 +79,7 @@ class DiagnosticStorageTest extends CompilerTest { } "collecting across the diagnostics to produce a new sequence" in { - val err = - IR.Error.Syntax(AST.Blank(), IR.Error.Syntax.UnsupportedSyntax("aa")) + val err = IR.Error.Syntax(null, IR.Error.Syntax.UnsupportedSyntax("aa")) val diagnostics = new DiagnosticStorage( List( diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/AliasAnalysisTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/AliasAnalysisTest.scala index bb4782732ba8..2cef69da0906 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/AliasAnalysisTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/AliasAnalysisTest.scala @@ -1219,7 +1219,8 @@ class AliasAnalysisTest extends CompilerTest { } } - "Alias analysis on typeset literals" should { + "Alias analysis on typeset literals" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: ModuleContext = mkModuleContext val method = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/DataflowAnalysisTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/DataflowAnalysisTest.scala index 9c338a4b4328..7b32f4a49b2d 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/DataflowAnalysisTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/DataflowAnalysisTest.scala @@ -1103,7 +1103,8 @@ class DataflowAnalysisTest extends CompilerTest { dependencies.getDirect(argXId) shouldEqual None } - "work properly for blocks" in { + "work properly for blocks" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val inlineContext: InlineContext = mkInlineContext val ir = @@ -1285,7 +1286,8 @@ class DataflowAnalysisTest extends CompilerTest { dependencies.getDirect(vecId) shouldEqual Some(Set(xUseId, yId, litId)) } - "work properly for typeset literals" in { + "work properly for typeset literals" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val inlineContext: InlineContext = mkInlineContext val ir = @@ -1470,7 +1472,7 @@ class DataflowAnalysisTest extends CompilerTest { | a = x + 1 | b = State.read | a+b . IO.println - |""".stripMargin + |""".stripMargin.linesIterator.mkString("\n") val codeWithMeta = meta.appendToCode(code) val ir = codeWithMeta.preprocessExpression.get.analyse @@ -1483,7 +1485,8 @@ class DataflowAnalysisTest extends CompilerTest { .asInstanceOf[IR.Expression.Binding] val aBindExpr = aBind.expression - "store a mapping between internal and external identifiers" in { + "store a mapping between internal and external identifiers" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). metadata.dependents.get(asStatic(aBind)).get should contain( asStatic(ir) ) @@ -1491,7 +1494,8 @@ class DataflowAnalysisTest extends CompilerTest { asStatic(ir).externalId shouldEqual Some(lambdaId) } - "return the set of external identifiers for invalidation" in { + "return the set of external identifiers for invalidation" ignore { + // FIXME: Different result in new parser!--needs triage (#5894). metadata.dependents.getExternal(asStatic(aBindExpr)).get shouldEqual Set( lambdaId, aBindId diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/GatherDiagnosticsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/GatherDiagnosticsTest.scala index 455dbbbc3a98..946ab789dd95 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/GatherDiagnosticsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/analyse/GatherDiagnosticsTest.scala @@ -7,15 +7,11 @@ import org.enso.compiler.core.IR.CallArgument import org.enso.compiler.pass.PassManager import org.enso.compiler.pass.analyse.GatherDiagnostics import org.enso.compiler.test.CompilerTest -import org.enso.syntax.text.AST class GatherDiagnosticsTest extends CompilerTest { "Error Gathering" should { - val error1 = IR.Error.Syntax( - AST.Invalid.Unrecognized("@@"), - IR.Error.Syntax.UnrecognizedToken - ) + val error1 = IR.Error.Syntax(null, IR.Error.Syntax.UnrecognizedToken) val plusOp = IR.Name.Literal("+", isMethod = true, None) val plusApp = IR.Application.Prefix( plusOp, @@ -50,15 +46,8 @@ class GatherDiagnosticsTest extends CompilerTest { } "work with module flow" in { - val error2 = IR.Error.Syntax( - AST.Invalid.Unexpected("whoa, that was not expected", List()), - IR.Error.Syntax.UnexpectedExpression - ) - - val error3 = IR.Error.Syntax( - AST.Invalid.Unexpected("whoa, that was also not expected", List()), - IR.Error.Syntax.AmbiguousExpression - ) + val error2 = IR.Error.Syntax(null, IR.Error.Syntax.UnexpectedExpression) + val error3 = IR.Error.Syntax(null, IR.Error.Syntax.AmbiguousExpression) val typeName = IR.Name.Literal("Foo", isMethod = false, None) diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ComplexTypeTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ComplexTypeTest.scala index 426493566306..b634b9a77bf2 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ComplexTypeTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ComplexTypeTest.scala @@ -148,7 +148,8 @@ class ComplexTypeTest extends CompilerTest { } } - "Invalid complex types" should { + "Invalid complex types" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: ModuleContext = mkModuleContext val ir = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/FunctionBindingTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/FunctionBindingTest.scala index 733361f8d1fe..801edfe25e55 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/FunctionBindingTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/FunctionBindingTest.scala @@ -273,7 +273,8 @@ class FunctionBindingTest extends CompilerTest { cArg.defaultValue shouldBe defined } - "work recursively" in { + "work recursively" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |f (a = (f a = a)) = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ImportsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ImportsTest.scala index de5571c2df76..468e8f1fbceb 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ImportsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/ImportsTest.scala @@ -52,13 +52,11 @@ class ImportsTest extends CompilerTest { |import Bar.Foo |import Bar.Foo as Bar |from Bar.Foo import Bar, Baz - |from Bar.Foo as Bar import Bar, Spam |from Bar.Foo import all | |export Bar.Foo |export Bar.Foo as Bar |from Bar.Foo export Bar, Baz - |from Bar.Foo as Bar export Bar, Baz |from Bar.Foo export all | |import Foo.Bar.Baz @@ -66,21 +64,19 @@ class ImportsTest extends CompilerTest { |""".stripMargin.preprocessModule.analyse "desugar project name imports correctly" in { - ir.imports.take(5).map(_.showCode()) shouldEqual List( + ir.imports.take(4).map(_.showCode()) shouldEqual List( "import Bar.Foo.Main as Foo", "import Bar.Foo.Main as Bar", "from Bar.Foo.Main import Bar, Baz", - "from Bar.Foo.Main as Bar import Bar, Spam", "from Bar.Foo.Main import all" ) } "desugar project name exports correctly" in { - ir.exports.take(5).map(_.showCode()) shouldEqual List( + ir.exports.take(4).map(_.showCode()) shouldEqual List( "export Bar.Foo.Main as Foo", "export Bar.Foo.Main as Bar", "from Bar.Foo.Main export Bar, Baz", - "from Bar.Foo.Main as Bar export Bar, Baz", "from Bar.Foo.Main export all" ) } diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/LambdaShorthandToLambdaTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/LambdaShorthandToLambdaTest.scala index fd5cee19f3ff..8a314e74a272 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/LambdaShorthandToLambdaTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/desugar/LambdaShorthandToLambdaTest.scala @@ -534,7 +534,8 @@ class LambdaShorthandToLambdaTest extends CompilerTest { lamArg1Name shouldEqual appArg1Name } - "correctly translate the function in an application" in { + "correctly translate the function in an application" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: InlineContext = mkInlineContext val ir = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ExpressionAnnotationsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ExpressionAnnotationsTest.scala index a1296e599433..6a3a9acf9753 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ExpressionAnnotationsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ExpressionAnnotationsTest.scala @@ -56,7 +56,8 @@ class ExpressionAnnotationsTest extends CompilerTest { // === The Tests ============================================================ - "Annotations resolution" should { + "Annotations resolution" ignore { + // FIXME: New parser handles the syntax error differently--needs triage (#5894). implicit val ctx: ModuleContext = mkModuleContext val ir = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/GenerateDocumentationTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/GenerateDocumentationTest.scala deleted file mode 100644 index bb4d0dc61f93..000000000000 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/GenerateDocumentationTest.scala +++ /dev/null @@ -1,222 +0,0 @@ -package org.enso.compiler.test.pass.resolve - -import org.enso.compiler.Passes -import org.enso.compiler.context.{InlineContext, ModuleContext} -import org.enso.compiler.core.IR -import org.enso.compiler.pass.resolve.GenerateDocumentation -import org.enso.compiler.pass.{PassConfiguration, PassGroup, PassManager} -import org.enso.compiler.test.CompilerTest -import org.enso.docs.generator.DocParserWrapper -import org.scalatest.Inside -import pprint.pprintln - -class GenerateDocumentationTest extends CompilerTest with Inside { - - // === Test Setup =========================================================== - - val passes = new Passes(defaultConfig) - - val precursorPasses: PassGroup = - passes.getPrecursors(GenerateDocumentation).get - - val passConfiguration: PassConfiguration = PassConfiguration(); - - implicit val passManager: PassManager = - new PassManager(List(precursorPasses), passConfiguration) - - /** Resolves documentation comments in a module. - * - * @param ir the module - */ - implicit class ResolveModule(ir: IR.Module) { - - /** Resolves documentation comments for [[ir]]. - * - * @param moduleContext the context in which to resolve - * @return [[ir]], with documentation resolved - */ - def resolve(implicit moduleContext: ModuleContext): IR.Module = { - GenerateDocumentation.runModule(ir, moduleContext) - } - } - - /** Resolves documentation comments in an expression. - * - * @param ir the expression - */ - implicit class ResolveExpression(ir: IR.Expression) { - - /** Resolves documentation comments for [[ir]]. - * - * @param inlineContext the context in which to resolve - * @return [[ir]], with documentation resolved - */ - def resolve(implicit inlineContext: InlineContext): IR.Expression = { - GenerateDocumentation.runExpression(ir, inlineContext) - } - } - - /** Creates a defaulted module context. - * - * @return a defaulted module context - */ - def mkModuleContext: ModuleContext = { - buildModuleContext(isGeneratingDocs = true) - } - - def mkModuleContext2: ModuleContext = { - buildModuleContext(isGeneratingDocs = false) - } - - /** Gets documentation metadata from a node. - * Throws an exception if missing. - * - * @param ir the ir to get the doc from. - * @return the doc assigned to `ir`. - */ - def getDoc(ir: IR): String = { - val meta = ir.getMetadata(GenerateDocumentation) - meta shouldBe defined - meta.get.documentation - } - - // === The Tests ============================================================ - - "Documentation comments in the top scope" should { - "be associated with atoms and methods" in { - implicit val moduleContext: ModuleContext = mkModuleContext - val ir = - """## Module Docs - | - |## This is doc for My_Atom - |type My_Atom a b c - | - |## This is doc for my_method - |MyAtom.my_method x = x + this - | - |""".stripMargin.preprocessModule.resolve - - ir.bindings.length shouldEqual 2 - ir.bindings(0) shouldBe an[IR.Module.Scope.Definition.Type] - ir.bindings(1) shouldBe an[IR.Module.Scope.Definition.Method] - - getDoc(ir.bindings(0)) shouldEqual DocParserWrapper.runOnPureDoc( - " This is doc for My_Atom" - ) - getDoc(ir.bindings(1)) shouldEqual DocParserWrapper.runOnPureDoc( - " This is doc for my_method" - ) - } - } - - "Documentation comments in blocks" should { - pending - "be associated with the documented expression in module flow" in { - implicit val moduleContext: ModuleContext = mkModuleContext - val ir = - """ - |method x = - | ## Do thing - | x + y - | ## Do another thing - | z = x * y - |""".stripMargin.preprocessModule.resolve - val body = ir - .bindings(0) - .asInstanceOf[IR.Module.Scope.Definition.Method.Explicit] - .body - .asInstanceOf[IR.Function.Lambda] - .body - .asInstanceOf[IR.Expression.Block] - - pprintln(body) - body.expressions.length shouldEqual 1 - getDoc(body.expressions(0)) shouldEqual DocParserWrapper.runOnPureDoc( - " Do thing" - ) - getDoc(body.returnValue) shouldEqual DocParserWrapper.runOnPureDoc( - " Do another thing" - ) - } - - "be associated with the type ascriptions" in { - implicit val moduleContext: ModuleContext = mkModuleContext - val ir = - """ - |method x = - | ## Id - | f : Any -> Any - | f x = x - | - | ## Return thing - | f 1 - |""".stripMargin.preprocessModule.resolve - val body = ir - .bindings(0) - .asInstanceOf[IR.Module.Scope.Definition.Method.Explicit] - .body - .asInstanceOf[IR.Function.Lambda] - .body - .asInstanceOf[IR.Expression.Block] - - body.expressions.length shouldEqual 2 - body.expressions(0) shouldBe an[IR.Application.Operator.Binary] - getDoc(body.expressions(0)) shouldEqual DocParserWrapper.runOnPureDoc( - " Id" - ) - getDoc(body.returnValue) shouldEqual DocParserWrapper.runOnPureDoc( - " Return thing" - ) - } - } - - "Documentation in complex type definitions" should { - pending - implicit val moduleContext: ModuleContext = mkModuleContext2 - "assign docs to all entities" in { - val ir = - """ - |## the type Foo - |type Foo - | ## the constructor Bar - | type Bar - | - | ## the included Unit - | Unit - | - | ## a method - | foo x = - | ## a statement - | IO.println "foo" - | ## the return - | 0 - |""".stripMargin.preprocessModule.resolve - val tp = - ir.bindings(0).asInstanceOf[IR.Module.Scope.Definition.SugaredType] - getDoc(tp) shouldEqual DocParserWrapper.runOnPureDoc( - " the type Foo" - ) - val t1 = tp.body(0) - getDoc(t1) shouldEqual DocParserWrapper.runOnPureDoc( - " the constructor Bar" - ) - val t2 = tp.body(1) - getDoc(t2) shouldEqual DocParserWrapper.runOnPureDoc( - " the included Unit" - ) - val method = tp.body(2).asInstanceOf[IR.Function.Binding] - getDoc(method) shouldEqual DocParserWrapper.runOnPureDoc( - " a method" - ) - val block = method.body.asInstanceOf[IR.Expression.Block] - getDoc( - block.expressions(0) - ) shouldEqual DocParserWrapper.runOnPureDoc( - " a statement" - ) - getDoc(block.returnValue) shouldEqual DocParserWrapper.runOnPureDoc( - " the return" - ) - } - } -} diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/IgnoredBindingsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/IgnoredBindingsTest.scala index fcf57fd67109..860594a1c89c 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/IgnoredBindingsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/IgnoredBindingsTest.scala @@ -89,7 +89,8 @@ class IgnoredBindingsTest extends CompilerTest { } } - "Ignored bindings desugaring for bindings" should { + "Ignored bindings desugaring for bindings" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: InlineContext = mkInlineContext val ir = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ModuleAnnotationsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ModuleAnnotationsTest.scala index c4d63e1e0484..e9feef7c26f5 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ModuleAnnotationsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/ModuleAnnotationsTest.scala @@ -12,8 +12,6 @@ class ModuleAnnotationsTest extends CompilerTest { // === Test Setup =========================================================== - override val useRustParser = true - val passes = new Passes(defaultConfig) val precursorPasses: PassGroup = passes.getPrecursors(ModuleAnnotations).get diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SugaredTypeFunctionsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SugaredTypeFunctionsTest.scala index 8b895f8f4326..1a6d4e3f274f 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SugaredTypeFunctionsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SugaredTypeFunctionsTest.scala @@ -58,7 +58,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir shouldBe an[IR.Type.Ascription] } - "work for left sections" in { + "work for left sections" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |(a :) @@ -68,7 +69,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir.asInstanceOf[IR.Function.Lambda].body shouldBe an[IR.Type.Ascription] } - "work for centre sections" in { + "work for centre sections" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |(:) @@ -81,7 +83,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { .body shouldBe an[IR.Type.Ascription] } - "work for right sections" in { + "work for right sections" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |(: a) @@ -91,7 +94,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir.asInstanceOf[IR.Function.Lambda].body shouldBe an[IR.Type.Ascription] } - "work for underscore arguments on the left" in { + "work for underscore arguments on the left" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |_ : A @@ -124,7 +128,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir shouldBe an[IR.Type.Ascription] } - "resolve context ascription" in { + "resolve context ascription" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |a in IO @@ -151,7 +156,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir shouldBe an[IR.Type.Set.Subsumption] } - "resolve equality" in { + "resolve equality" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |T ~ P @@ -187,7 +193,8 @@ class SugaredTypeFunctionsTest extends CompilerTest { ir shouldBe an[IR.Type.Set.Intersection] } - "resolve subtraction" in { + "resolve subtraction" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). val ir = """ |T \ P diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SuspendedArgumentsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SuspendedArgumentsTest.scala index a58c3c54f358..f72d23a97cb6 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SuspendedArgumentsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/SuspendedArgumentsTest.scala @@ -205,7 +205,8 @@ class SuspendedArgumentsTest extends CompilerTest { } "Suspended arguments resolution in expressions" should { - "correctly mark arguments as suspended in blocks" in { + "correctly mark arguments as suspended in blocks" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: InlineContext = mkInlineContext val ir = @@ -223,7 +224,8 @@ class SuspendedArgumentsTest extends CompilerTest { assert(func.arguments(1).suspended, "b is not suspended") } - "correctly mark arguments as suspended using inline expressions" in { + "correctly mark arguments as suspended using inline expressions" ignore { + // FIXME: Not supported by new parser--needs triage (#5894). implicit val ctx: InlineContext = mkInlineContext val ir = diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/TypeSignaturesTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/TypeSignaturesTest.scala index c0db0b1611ec..847b754c3c56 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/TypeSignaturesTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/TypeSignaturesTest.scala @@ -255,12 +255,14 @@ class TypeSignaturesTest extends CompilerTest { |f a (b = 1 : Int) : Double |""".stripMargin.preprocessExpression.get.resolve - "associate the signature with the typed expression" in { + // FIXME: Not supported by new parser--needs triage (#5894). + "associate the signature with the typed expression" ignore { ir shouldBe an[IR.Application.Prefix] ir.getMetadata(TypeSignatures) shouldBe defined } - "work recursively" in { + // FIXME: Not supported by new parser--needs triage (#5894). + "work recursively" ignore { val arg2Value = ir.asInstanceOf[IR.Application.Prefix].arguments(1).value arg2Value shouldBe an[IR.Literal.Number] } diff --git a/lib/rust/ensogl/component/text/Cargo.toml b/lib/rust/ensogl/component/text/Cargo.toml index a218e52cab8b..d781692b8189 100644 --- a/lib/rust/ensogl/component/text/Cargo.toml +++ b/lib/rust/ensogl/component/text/Cargo.toml @@ -31,7 +31,3 @@ rustybuzz = { workspace = true } rand = { version = "0.8.5", default-features = false } rand_chacha = "0.3.1" wasm-bindgen-test = { workspace = true } - -# Stop wasm-pack from running wasm-opt, because we run it from our build scripts in order to customize options. -[package.metadata.wasm-pack.profile.release] -wasm-opt = false diff --git a/lib/rust/parser/debug/LoadParser/LoadParser.java b/lib/rust/parser/debug/LoadParser/LoadParser.java deleted file mode 100644 index 3c4b60b5da55..000000000000 --- a/lib/rust/parser/debug/LoadParser/LoadParser.java +++ /dev/null @@ -1,238 +0,0 @@ -package org.enso.checkparser; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import org.enso.compiler.EnsoCompiler; -import org.enso.compiler.codegen.AstToIr; -import org.enso.compiler.core.IR; -import org.enso.compiler.core.IR$Comment$Documentation; -import org.enso.compiler.core.IR$Module$Scope$Definition; -import org.enso.compiler.core.IR$Type$Ascription; -import org.enso.syntax.text.AST; -import org.enso.syntax.text.Shape; -import org.enso.syntax2.Parser; -import org.enso.syntax2.Tree; -import org.graalvm.polyglot.Source; -import scala.Function1; -import scala.collection.immutable.List; - -class LoadParser implements FileVisitor, AutoCloseable { - private final File root; - private final Parser parser; - private final EnsoCompiler compiler; - private final Set visited = new LinkedHashSet<>(); - private final Map failed = new LinkedHashMap<>(); - private final Set irTested = new LinkedHashSet<>(); - private final Map irFailed = new LinkedHashMap<>(); - private final Set irDiff = new LinkedHashSet<>(); - - private LoadParser(File root) { - this.parser = Parser.create(); - this.compiler = new EnsoCompiler(); - this.root = root; - } - - @Override - public void close() throws Exception { - parser.close(); - } - - public static void main(String[] args) throws Exception { - var root = new File(".").getAbsoluteFile(); - try (LoadParser checker = new LoadParser(root)) { - checker.scan("distribution"); - checker.scan("test"); - - checker.printSummary(true); - } - } - - private void scan(String path) throws IOException { - var dir = root.toPath().resolve(path); - assert Files.isDirectory(dir) : "isDirectory: " + dir; - - Files.walkFileTree(dir, this); - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return FileVisitResult.CONTINUE; - } - - private static Exception newExceptionNoStack(String msg) { - var ex = new Exception(msg); - ex.setStackTrace(new StackTraceElement[0]); - return ex; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!file.getFileName().toString().endsWith(".enso")) { - return FileVisitResult.CONTINUE; - } - visited.add(file); - - System.err.println("processing " + file); - Source src = Source.newBuilder("enso", file.toFile()).build(); - TEST: try { - Tree tree = parser.parse(src.getCharacters().toString()); - if (tree == null) { - failed.put(file, newExceptionNoStack("Rust failed")); - } else { - IR.Module ir; - try { - irTested.add(file); - IR.Module m = compiler.generateIR(tree); - if (m == null) { - throw new NullPointerException(); - } - ir = sanitize(m); - } catch (Exception ex) { - if (ex.getClass().getName().contains("UnhandledEntity")) { - if (ex.getMessage().contains("= Invalid[")) { - failed.put(file, newExceptionNoStack("Rust produces Invalid AST")); - break TEST; - } - if (ex.getMessage().contains("translateCaseBranch = Case[null, null")) { - failed.put(file, newExceptionNoStack("Rust provides null case")); - break TEST; - } - } - irFailed.put(file, ex); - break TEST; - } - - var oldAst = new org.enso.syntax.text.Parser().runWithIds(src.getCharacters().toString()); - var oldIr = sanitize(AstToIr.translate((AST.ASTOf)(Object)oldAst)); - - Function filter = (i) -> { - var txt = i.pretty().replaceAll("id = [0-9a-f\\-]*", "id = _"); - for (;;) { - final String pref = "IdentifiedLocation("; - int at = txt.indexOf(pref); - if (at == -1) { - break; - } - int to = at + pref.length(); - int depth = 1; - while (depth > 0) { - switch (txt.charAt(to)) { - case '(': depth++; break; - case ')': depth--; break; - } - to++; - } - txt = txt.substring(0, at) + "IdentifiedLocation[_]" + txt.substring(to); - } - var sb = new StringBuilder(); - for (String l : txt.split("\n")) { - final String pref = "IR.Comment.Documentation"; - if (l.contains(pref)) { - continue; - } - sb.append(l).append("\n"); - } - return sb.toString(); - }; - - var old = filter.apply(oldIr); - var now = filter.apply(ir); - if (!old.equals(now)) { - irDiff.add(file); - var oldFile = file.getParent().resolve(file.getFileName() + ".old"); - var nowFile = file.getParent().resolve(file.getFileName() + ".now"); - System.err.println("difference1: " + oldFile); - System.err.println("difference2: " + nowFile); - Files.writeString(oldFile , old, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - Files.writeString(nowFile, now, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - } - } - } catch (Exception ex) { - failed.put(file, ex); - System.err.println("failed " + file); - } - - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - visited.add(file); - failed.put(file, exc); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - - private void printSummary(boolean verbose) { - if (verbose) { - for (var en : failed.entrySet()) { - var key = en.getKey(); - var value = en.getValue(); - System.err.println("Problem " + key); - value.printStackTrace(); - } - for (var en : irFailed.entrySet()) { - var key = en.getKey(); - var value = en.getValue(); - System.err.println("File " + key); - value.printStackTrace(); - } - } - System.out.println("Found " + visited.size() + " files. " + failed.size() + " failed to parse"); - System.out.println("From " + irTested.size() + " files " + irFailed.size() + " failed to produce IR"); - System.out.println("From " + (irTested.size() - irFailed.size()) + " files " + irDiff.size() + " have different IR"); - } - - private static IR.Module sanitize(IR.Module m) { - class NoComments implements Function1 { - @Override - public IR.Expression apply(IR.Expression exp) { - if (exp == null) { - return null; - } - if (exp instanceof IR$Comment$Documentation) { - return null; - } - return exp.mapExpressions(this); - } - } - class NoCommentsInBindings implements Function1 { - @Override - public Boolean apply(IR$Module$Scope$Definition exp) { - if (exp instanceof IR$Comment$Documentation) { - return false; - } else if (exp instanceof IR$Type$Ascription) { - return false; - } else { - return true; - } - } - } - var m1 = m.mapExpressions(new NoComments()); - var m2 = m1.copy( - m1.copy$default$1(), - m1.copy$default$2(), - (List) m1.bindings().filter(new NoCommentsInBindings()), - m1.copy$default$4(), - m1.copy$default$5(), - m1.copy$default$6(), - m1.copy$default$7() - ); - return m2; - } -} diff --git a/lib/rust/parser/debug/LoadParser/LoadParser.sh b/lib/rust/parser/debug/LoadParser/LoadParser.sh deleted file mode 100755 index 2eb7dbb7f984..000000000000 --- a/lib/rust/parser/debug/LoadParser/LoadParser.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -x - -set -e - -GRAALVM=$HOME/bin/graalvm/ - -# build runtime.jar including the new parser classes -sbt --java-home $GRAALVM bootstrap -sbt --java-home $GRAALVM buildEngineDistribution - -# run test: parser all .enso files in the repository -$GRAALVM/bin/java -cp runtime.jar lib/rust/parser/debug/LoadParser/LoadParser.java diff --git a/lib/rust/parser/doc-parser/Cargo.toml b/lib/rust/parser/doc-parser/Cargo.toml new file mode 100644 index 000000000000..c10155274615 --- /dev/null +++ b/lib/rust/parser/doc-parser/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "enso-doc-parser" +version = "0.1.0" +authors = ["Enso Team "] +edition = "2021" +description = "Enso Documentation Parser." +readme = "README.md" +homepage = "https://github.com/enso-org/enso" +repository = "https://github.com/enso-org/enso" +license-file = "../../LICENSE" + +[dependencies] +enso-parser = { path = ".." } +enso-prelude = { path = "../../prelude" } +enso-profiler = { path = "../../profiler" } +enso-reflect = { path = "../../reflect" } + +[dev-dependencies] +enso-metamodel = { path = "../../metamodel", features = ["rust"] } +enso-metamodel-lexpr = { path = "../../metamodel/lexpr" } +lexpr = "0.2.6" diff --git a/lib/rust/parser/doc-parser/src/doc_sections.rs b/lib/rust/parser/doc-parser/src/doc_sections.rs new file mode 100644 index 000000000000..8b7b2dc303b8 --- /dev/null +++ b/lib/rust/parser/doc-parser/src/doc_sections.rs @@ -0,0 +1,213 @@ +//! Parses documentation text to the [`DocSection`] representation. + +use crate::*; + + + +// ============================== +// === High-level Parsing API === +// ============================== + +/// Parse the given documentation text to a collection of [`DocSection`]s. +pub fn parse(docs: &str) -> Vec { + // Although this is semantically a pure function, for efficiency we use one persistent parser + // to reuse its buffers. + thread_local! { + static PARSER: RefCell = Default::default(); + } + PARSER.with_borrow_mut(|parser| parser.parse(docs)) +} + + + +// ============================ +// === Documentation Parser === +// ============================ + +/// Parses documentation text to the [`DocSection`] representation. +/// +/// Note that this object is semantically stateless, but reusing it allows better performance by +/// avoiding the need to reallocate its working buffers. +#[derive(Default, Debug)] +pub struct DocParser { + docs: DocSectionCollector, + lexer: Lexer, +} + +impl DocParser { + /// Create a new [`DocParser`]. + pub fn new() -> Self { + Self::default() + } + + /// Parse the documentation. + #[profile(Detail)] + pub fn parse(&mut self, input: &str) -> Vec { + for (line_number, line) in input.trim_start().lines().enumerate() { + let location = Location::start_of_line(line_number); + let line = Span { location, text: line }; + self.lexer.line::(line, &mut self.docs); + } + self.lexer.finish::(&mut self.docs); + self.docs.finish() + } +} + + + +// =================== +// === Doc Section === +// =================== + +/// Text rendered as HTML (may contain HTML tags). +pub type HtmlString = String; + +/// A single section of the documentation. +#[derive(Hash, Debug, Clone, PartialEq, Eq)] +#[allow(missing_docs)] +pub enum DocSection { + /// The documentation tag. + Tag { + /// The tag name. + name: &'static str, + /// The tag text. + body: HtmlString, + }, + /// The paragraph of the text. + Paragraph { + /// The elements that make up this paragraph. + body: HtmlString, + }, + /// The section that starts with the key followed by the colon and the body. + Keyed { + /// The section key. + key: String, + /// The elements that make up the body of the section. + body: HtmlString, + }, + /// The section that starts with the mark followed by the header and the body. + Marked { + /// The section mark. + mark: Mark, + /// The section header. + header: Option, + /// The elements that make up the body of the section. + body: HtmlString, + }, +} + + + +// ============================ +// === DocSection Collector === +// ============================ + +#[derive(Default, Debug)] +struct DocSectionCollector { + sections: Vec, + in_secondary_section: bool, + current_body: String, +} + +impl DocSectionCollector { + fn finish_section(&mut self) { + let text = self.current_body.clone(); + self.current_body.clear(); + self.in_secondary_section = true; + match self.sections.last_mut() { + Some(DocSection::Paragraph { body, .. }) + | Some(DocSection::Keyed { body, .. }) + | Some(DocSection::Marked { body, .. }) => *body = text, + Some(DocSection::Tag { .. }) | None => + self.sections.push(DocSection::Paragraph { body: text }), + } + } + + fn finish(&mut self) -> Vec { + self.finish_section(); + let result = self.sections.drain(..).collect(); + let current_body = std::mem::take(&mut self.current_body); + let sections = std::mem::take(&mut self.sections); + *self = Self { + // Reuse the (empty) buffers. + current_body, + sections, + // Reset the rest of state. + in_secondary_section: Default::default(), + }; + result + } +} + +impl TokenConsumer for DocSectionCollector { + fn tag(&mut self, tag: Tag, description: Option>) { + let name = tag.to_str(); + let body = description.map(|description| description.to_string()).unwrap_or_default(); + self.sections.push(DocSection::Tag { name, body }); + } + + fn enter_marked_section(&mut self, mark: Mark, header: Option>) { + self.finish_section(); + let header = header.map(|header| header.to_string()); + let body = Default::default(); + self.sections.push(DocSection::Marked { mark, header, body }); + } + + fn enter_keyed_section(&mut self, header: Span<'_, L>) { + self.finish_section(); + let key = header.to_string(); + let body = Default::default(); + self.sections.push(DocSection::Keyed { key, body }); + } + + fn text(&mut self, text: Span<'_, L>) { + self.current_body.push_str(text.as_ref()); + } + + fn start_list(&mut self) { + self.current_body.push_str("
    "); + } + + fn start_list_item(&mut self) { + self.current_body.push_str("
  • "); + } + + fn start_paragraph(&mut self) { + let first_content = !self.in_secondary_section && self.current_body.is_empty(); + if !first_content { + self.current_body.push_str("

    "); + } + } + + fn start_raw(&mut self) { + self.current_body.push_str("

    ");
+    }
+
+    fn start_quote(&mut self) {
+        self.current_body.push_str("");
+    }
+
+    fn end_quote(&mut self) {
+        self.current_body.push_str("");
+    }
+
+    fn whitespace(&mut self) {
+        self.current_body.push(' ');
+    }
+
+    fn raw_line(&mut self, text: Span<'_, L>) {
+        if !self.current_body.is_empty() {
+            self.current_body.push('\n');
+        }
+        self.current_body.push_str(text.as_ref());
+    }
+
+    fn end(&mut self, scope: ScopeType) {
+        match scope {
+            ScopeType::List => self.current_body.push_str("
"), + ScopeType::ListItem => (), + ScopeType::Paragraph => (), + ScopeType::Raw => self.current_body.push_str(""), + } + } +} diff --git a/lib/rust/parser/doc-parser/src/lib.rs b/lib/rust/parser/doc-parser/src/lib.rs new file mode 100644 index 000000000000..2c37672a1fce --- /dev/null +++ b/lib/rust/parser/doc-parser/src/lib.rs @@ -0,0 +1,680 @@ +//! Enso documentation parser. + +#![recursion_limit = "256"] +// === Features === +#![feature(assert_matches)] +#![feature(let_chains)] +#![feature(if_let_guard)] +#![feature(local_key_cell_methods)] +// === Standard Linter Configuration === +#![deny(non_ascii_idents)] +#![warn(unsafe_code)] +#![allow(clippy::bool_to_int_with_if)] +#![allow(clippy::let_and_return)] +// === Non-Standard Linter Configuration === +#![allow(clippy::option_map_unit_fn)] +#![allow(clippy::precedence)] +#![allow(dead_code)] +#![deny(unconditional_recursion)] +#![warn(missing_copy_implementations)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] +#![warn(trivial_casts)] +#![warn(trivial_numeric_casts)] +#![warn(unused_import_braces)] +#![warn(unused_qualifications)] + +use enso_prelude::*; + +pub mod doc_sections; + +pub use doc_sections::parse; +pub use doc_sections::DocSection; + +pub(crate) use enso_profiler as profiler; +pub(crate) use enso_profiler::profile; + + + +// ============ +// === Tags === +// ============ + +/// A [`Tag`], optionally with a following description. +#[derive(Debug)] +pub struct TagWithDescription<'a, L> { + name: Tag, + description: Option>, +} + +/// Indicator placed at the beginning of a documentation section, e.g. `PRIVATE`. +#[derive(Debug, Copy, Clone)] +#[allow(missing_docs)] +pub enum Tag { + Added, + Advanced, + Alias, + Deprecated, + Modified, + Private, + Removed, + TextOnly, + Unstable, + Upcoming, +} + + +// === Lexing === + +impl<'a, L: Location> TagWithDescription<'a, L> { + /// Try to lex the given text as a [`TagWithDescription`]. + pub fn new(text: Span<'a, L>) -> Option { + let (tag, description) = text.first_word_and_rest(); + Tag::new(tag.text).map(|name| TagWithDescription { name, description }) + } +} + +impl Tag { + /// Try to lex the given text as a [`Tag`]. + pub fn new(text: &str) -> Option { + use Tag::*; + match text { + "ADDED" => Some(Added), + "ADVANCED" => Some(Advanced), + "ALIAS" => Some(Alias), + "DEPRECATED" => Some(Deprecated), + "MODIFIED" => Some(Modified), + "PRIVATE" => Some(Private), + "REMOVED" => Some(Removed), + "TEXT_ONLY" => Some(TextOnly), + "UNSTABLE" => Some(Unstable), + "UPCOMING" => Some(Upcoming), + _ => None, + } + } + + /// Return a string that lexes as the given tag. + pub fn to_str(&self) -> &'static str { + match self { + Tag::Added => "ADDED", + Tag::Advanced => "ADVANCED", + Tag::Alias => "ALIAS", + Tag::Deprecated => "DEPRECATED", + Tag::Modified => "MODIFIED", + Tag::Private => "PRIVATE", + Tag::Removed => "REMOVED", + Tag::TextOnly => "TEXT_ONLY", + Tag::Unstable => "UNSTABLE", + Tag::Upcoming => "UPCOMING", + } + } +} + + + +// ============= +// === Marks === +// ============= + +/// Header for a section introduced by a marker-character and following text. +#[derive(Debug)] +pub struct Marked<'a, L> { + mark: Mark, + header: Option>, +} + +/// Documentation section mark. +#[derive(Hash, Debug, Copy, Clone, PartialEq, Eq)] +#[allow(missing_docs)] +pub enum Mark { + Important, + Info, + Example, +} + + +// === Lexing === + +impl Mark { + /// Try to lex the given text as a header-introducing marker character. + pub fn new(text: &str) -> Option { + match text { + "!" => Some(Mark::Important), + "?" => Some(Mark::Info), + ">" => Some(Mark::Example), + _ => None, + } + } +} + +impl<'a, L: Location> Marked<'a, L> { + /// Try to lex the given text as a marked-section header. + pub fn new(text: Span<'a, L>) -> Option { + let (first_word, header) = text.first_word_and_rest(); + Mark::new(first_word.text).map(|mark| Marked { mark, header }) + } +} + + + +// ============= +// === Lines === +// ============= + +/// A line of code, separated into indent and content. +#[derive(Debug)] +pub struct Line<'a, L> { + indent: Offset, + content: Span<'a, L>, +} + + +// === Offset and Visible Offset === + +/// A length of text, representing a number of spaces. +#[derive(Copy, Clone, Debug, Default)] +pub struct Offset { + bytes: usize, + visible: VisibleOffset, +} + +/// A number of spaces. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct VisibleOffset(usize); + +impl Add for VisibleOffset { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl From for VisibleOffset { + fn from(value: Offset) -> Self { + value.visible + } +} + + + +// ============= +// === Spans === +// ============= + +/// A slice of text, with its location in the input. +#[derive(Copy, Clone, Debug, Default)] +pub struct Span<'a, L> { + /// The location of the start of the text. + pub location: L, + /// The text. + pub text: &'a str, +} + +impl<'a, L: Location> Span<'a, L> { + /// Remove all leading whitespace characters; return the initial offset and following content, + /// unless the line has no non-whitespace content. + pub fn trim_start(self) -> Option> { + let mut indent = Offset::default(); + for b in self.text.bytes() { + match b { + b' ' => indent.visible.0 += 1, + b'\t' => { + self.warn("Tab character used for indentation."); + indent.visible.0 += 4 + } + _ => { + let (whitespace, text) = self.text.split_at(indent.bytes); + let location = self.location.offset_text(whitespace); + let content = Span { location, text }; + return Some(Line { indent, content }); + } + } + indent.bytes += 1; + } + None + } + + /// Remove leading whitespace characters corresponding to the specified visible offset; return + /// the following content. + pub fn trim_start_exact(self, limit: VisibleOffset) -> Self { + let mut indent = Offset::default(); + let mut bytes = self.text.bytes(); + while indent.visible < limit { + match bytes.next() { + Some(b' ') => { + indent.visible.0 += 1; + } + Some(b'\t') => { + self.warn("Tab character used for indentation."); + indent.visible.0 += 4; + } + Some(_) => break, + None => { + let unexpected_condition = "Internal error: Expected greater indent level."; + self.warn(unexpected_condition); + warn!("{unexpected_condition}"); + break; + } + } + indent.bytes += 1; + } + let (whitespace, text) = self.text.split_at(indent.bytes); + let location = self.location.offset_text(whitespace); + Self { location, text } + } + + /// Return the first word (which may be empty), and the rest of the input, if any. + pub fn first_word_and_rest(self) -> (Self, Option) { + match self.text.split_once(' ') { + Some((first, rest)) => ( + Self { location: self.location, text: first }, + Some(Self { location: self.location.offset_text(first).offset(1), text: rest }), + ), + None => (self, None), + } + } + + /// Returns a [`Span`] with the given prefix removed, if it was present. + pub fn strip_prefix(self, prefix: &str) -> Option { + self.text + .strip_prefix(prefix) + .map(|text| Self { location: self.location.offset_text(prefix), text }) + } + + /// Returns a [`Span`] with the given suffix removed, if it was present. + pub fn strip_suffix(self, suffix: char) -> Option { + self.text.strip_suffix(suffix).map(|text| Self { location: self.location, text }) + } + + /// Return a 0-length span representing the point immediately after this span. + pub fn after(self) -> Self { + Self { location: self.location.offset_text(self.text), text: "" } + } + + /// Emit a warning for this location, if warnings are enabled. + pub fn warn(self, _warning: impl Warning) { + // TODO: self.location.warn(warning) + } + + /// Split at the given index. Panics if the index is not a character boundary. + pub fn split_at(self, i: usize) -> (Self, Self) { + let (a, b) = self.text.split_at(i); + let a_ = Self { location: self.location, text: a }; + let b_ = Self { location: self.location.offset_text(a), text: b }; + (a_, b_) + } +} + +impl<'a, L> AsRef for Span<'a, L> { + fn as_ref(&self) -> &str { + self.text + } +} + +impl<'a, L> From> for String { + fn from(value: Span<'a, L>) -> Self { + value.text.to_string() + } +} + +impl<'a, L> Display for Span<'a, L> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.text) + } +} + + +// === Location === + +/// An offset within source code, expressed as line number and character index within the line. +pub trait Location: Copy + Debug { + /// The first character of the specified line. + fn start_of_line(line_number: usize) -> Self; + /// Return the location, advanced within the line by the given text. + fn offset_text(self, text: &str) -> Self; + /// Return the location, advanced within the line by the specified number of characters. + fn offset(self, chars: usize) -> Self; +} + + +// === Ignored Location === + +/// [`Location`] type that doesn't track locations, for high-performance parsing when +/// accurately-located warnings are not needed. +#[derive(Copy, Clone, Debug, Default)] +pub struct IgnoredLocation; + +impl Location for IgnoredLocation { + fn start_of_line(_line_number: usize) -> Self { + Self + } + fn offset_text(self, _text: &str) -> Self { + Self + } + fn offset(self, _chars: usize) -> Self { + Self + } +} + + + +// =========== +// === Lex === +// =========== + +/// Breaks input state into tokens to be fed to a [`TokenConsumer`]. +#[derive(Default, Debug)] +pub struct Lexer { + scopes: Scopes, + state: State, +} + +/// Lexer state. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] +enum State { + /// No non-tag line has been read. + #[default] + Tags, + /// Within an example's description. + ExampleDescription, + /// Expecting an example's code block to start. + ExampleExpectingCode { within_indent: VisibleOffset }, + /// Within an example's code block. + ExampleCode, + /// Not in any special context. + Normal, +} + + +impl Lexer { + /// Lex a line and feed it to the given [`TokenConsumer`]. + pub fn line(&mut self, raw: Span<'_, L>, docs: &mut impl TokenConsumer) { + let line = raw.trim_start(); + match (self.state, line) { + (State::Tags, Some(line)) if let Some(tag) = TagWithDescription::new(line.content) => { + // TODO: WARN IF: indent != min. + docs.tag(tag.name, tag.description); + } + (State::Tags, Some(line)) => { + self.state = State::Normal; + self.normal_line(line, docs) + } + (State::Tags, None) => + raw.warn("Unneeded empty line before content or between tags."), + (State::ExampleDescription, None) => { + self.scopes.end_all().for_each(|scope| docs.end(scope)); + // TODO: within_indent + self.state = State::ExampleExpectingCode { within_indent: VisibleOffset(0) }; + } + (State::ExampleDescription, Some(line)) => self.normal_line(line, docs), + (State::ExampleExpectingCode { .. }, None) => + raw.warn("Extra empty line before example code."), + (State::ExampleExpectingCode { within_indent }, Some(line)) + if line.indent.visible <= within_indent => { + line.content.warn("No code found in example section."); + self.state = State::Normal; + self.normal_line(line, docs) + } + (State::ExampleExpectingCode { .. }, Some(line)) => { + self.state = State::ExampleCode; + self.scopes.start_raw(line.indent); + docs.start_raw(); + docs.raw_line(line.content); + } + (State::ExampleCode, None) if let Some(_) = self.scopes.raw() => { + docs.raw_line(raw.after()); + }, + (State::ExampleCode, None) => (), + (State::ExampleCode, Some(line)) => { + self.scopes.end_below(line.indent).for_each(|scope| docs.end(scope)); + if let Some(indent) = self.scopes.raw() { + docs.raw_line(raw.trim_start_exact(indent)); + } else { + self.state = State::Normal; + self.normal_line(line, docs) + } + } + (State::Normal, Some(line)) => self.normal_line(line, docs), + (State::Normal, None) => { + self.scopes.end_all().for_each(|scope| docs.end(scope)); + } + } + } + + /// Complete the current input, closing any open scopes for the given [`TokenConsumer`] and + /// resetting to the default state. + pub fn finish(&mut self, docs: &mut impl TokenConsumer) { + self.scopes.end_all().for_each(|scope| docs.end(scope)); + let scopes = mem::take(&mut self.scopes); + *self = Self { + // Reuse buffers. + scopes, + // Reset state. + state: Default::default(), + }; + } +} + +impl Lexer { + fn normal_line(&mut self, line: Line<'_, L>, docs: &mut impl TokenConsumer) { + let Line { indent, content } = line; + match content { + _ if let Some(marked) = Marked::new(content) => { + self.scopes.end_all().for_each(|scope| docs.end(scope)); + docs.enter_marked_section(marked.mark, marked.header); + if marked.mark == Mark::Example { + self.state = State::ExampleDescription; + } + }, + t if let Some(t) = t.strip_suffix(':') => { + self.scopes.end_all().for_each(|scope| docs.end(scope)); + docs.enter_keyed_section(t); + }, + t if let Some(content) = t.strip_prefix("- ") => { + self.scopes.end_below(indent).for_each(|scope| docs.end(scope)); + if self.scopes.start_list_if_not_started(indent) { + docs.start_list(); + } + self.scopes.start_list_item(indent); + docs.start_list_item(); + self.text(content, docs); + } + _ => { + self.scopes.end_below(indent).for_each(|scope| docs.end(scope)); + if self.scopes.is_in_text() { + docs.whitespace(); + } else { + self.scopes.start_paragraph(indent); + docs.start_paragraph(); + } + self.text(content, docs); + }, + } + } + + fn text(&mut self, text: Span, docs: &mut impl TokenConsumer) { + let mut quote_open = None; + let mut i = 0; + let mut remaining = text; + for c in text.text.bytes() { + match c { + b'`' => { + let (before_quote, including_quote) = remaining.split_at(i); + let (quote, next_remaining) = including_quote.split_at(1); + remaining = next_remaining; + i = 0; + docs.text(before_quote); + if quote_open.is_some() { + docs.end_quote(); + quote_open = None; + } else { + docs.start_quote(); + quote_open = Some(quote); + } + } + _ => i += 1, + } + } + docs.text(remaining); + if let Some(quote) = quote_open { + quote.warn("Unclosed quote."); + docs.end_quote(); + } + } +} + + +// === Scopes === + +/// A [`Lexer`] scope. +#[derive(Debug)] +struct Scope { + r#type: ScopeType, + indent: VisibleOffset, +} + +/// A [`Lexer`] scope type. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ScopeType { + /// An unordered list. + List, + /// An unordered-list item. + ListItem, + /// A paragraph of text. + Paragraph, + /// A block of preformatted text. No further scopes will occur inside. + Raw, +} + +/// A stack of [`Lexer`] scopes. +#[derive(Default, Debug)] +struct Scopes { + scopes: Vec, +} + +impl Scopes { + fn end_all(&mut self) -> impl Iterator { + self.end_including(VisibleOffset(0)) + } + + fn end_below(&mut self, indent: impl Into) -> impl Iterator { + let indent = indent.into(); + self.end_including(indent + VisibleOffset(1)) + } + + fn end_including( + &mut self, + indent: impl Into, + ) -> impl Iterator { + let indent = indent.into(); + // FIXME: Don't allocate. + let mut scopes = vec![]; + while let Some(scope) = self.scopes.pop_if(|scope| scope.indent >= indent) { + scopes.push(scope.r#type); + } + scopes.into_iter() + } + + fn start_list_if_not_started(&mut self, indent: impl Into) -> bool { + let indent = indent.into(); + match self.scopes.last() { + Some(Scope { r#type: ScopeType::List, .. }) => false, + _ => { + self.scopes.push(Scope { r#type: ScopeType::List, indent }); + true + } + } + } + + fn start_list_item(&mut self, indent: impl Into) { + let indent = indent.into(); + let min_child_indent = indent + VisibleOffset(1); + self.scopes.push(Scope { r#type: ScopeType::ListItem, indent: min_child_indent }); + } + + fn is_in_text(&self) -> bool { + let current = self.scopes.last().map(|scope| scope.r#type); + matches!(current, Some(ScopeType::ListItem | ScopeType::Paragraph)) + } + + fn start_paragraph(&mut self, indent: impl Into) { + let indent = indent.into(); + self.scopes.push(Scope { r#type: ScopeType::Paragraph, indent }); + } + + fn start_raw(&mut self, indent: impl Into) { + let indent = indent.into(); + self.scopes.push(Scope { r#type: ScopeType::Raw, indent }); + } + + fn raw(&self) -> Option { + match self.scopes.last() { + Some(Scope { r#type: ScopeType::Raw, indent }) => Some(*indent), + _ => None, + } + } +} + + +// === Warnings === + +/// A message identifying improperly-formatted documentation. +/// +/// Note that implementors of this trait should not require any work to be done (e.g. string +/// formatting or other allocations) until `to_string` is called, to prevent overhead when parsing +/// the standard library (in which case warnings are normally not used anyway). +pub trait Warning { + /// Produce the message. + fn to_string(&self) -> String; +} + +impl<'a> Warning for &'a str { + fn to_string(&self) -> String { + str::to_string(self) + } +} + +impl Warning for T +where T: Fn() -> String +{ + fn to_string(&self) -> String { + (self)() + } +} + + + +// ====================== +// === Token Consumer === +// ====================== + +/// Receives a stream of tokens. +pub trait TokenConsumer { + /// A tag. + fn tag(&mut self, tag: Tag, description: Option>); + /// Header for a section introduced by a mark character. + fn enter_marked_section(&mut self, mark: Mark, header: Option>); + /// Header for a section introduced by text ending in a colon. + fn enter_keyed_section(&mut self, header: Span<'_, L>); + /// Plain text. + fn text(&mut self, text: Span<'_, L>); + /// Start an unordered-list. + fn start_list(&mut self); + /// Start an unordered-list item. + fn start_list_item(&mut self); + /// Start a paragraph. + fn start_paragraph(&mut self); + /// Start a preformatted-text section. + fn start_raw(&mut self); + /// An opening-quote. + fn start_quote(&mut self); + /// A closing-quote. + fn end_quote(&mut self); + /// Space between two [`text`] tokens. + fn whitespace(&mut self); + /// A line of preformatted text. No newline character is included. + fn raw_line(&mut self, text: Span<'_, L>); + /// Close a scope, of specified type. As scopes form a stack, the scope closed will always be + /// the most recently-opened scope that has not been closed. + fn end(&mut self, scope: ScopeType); +} diff --git a/lib/rust/parser/doc-parser/src/main.rs b/lib/rust/parser/doc-parser/src/main.rs new file mode 100644 index 000000000000..33f8d7afc3ee --- /dev/null +++ b/lib/rust/parser/doc-parser/src/main.rs @@ -0,0 +1,189 @@ +//! Prints a debug representation of Enso documentation found in the given Enso sourec file(s). + +#![recursion_limit = "256"] +// === Features === +#![allow(incomplete_features)] +#![feature(assert_matches)] +#![feature(allocator_api)] +#![feature(exact_size_is_empty)] +#![feature(test)] +#![feature(specialization)] +#![feature(let_chains)] +#![feature(if_let_guard)] +// === Standard Linter Configuration === +#![deny(non_ascii_idents)] +#![warn(unsafe_code)] +#![allow(clippy::bool_to_int_with_if)] +#![allow(clippy::let_and_return)] +// === Non-Standard Linter Configuration === +#![allow(clippy::option_map_unit_fn)] +#![allow(clippy::precedence)] +#![allow(dead_code)] +#![deny(unconditional_recursion)] +#![warn(missing_copy_implementations)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] +#![warn(trivial_casts)] +#![warn(trivial_numeric_casts)] +#![warn(unused_import_braces)] +#![warn(unused_qualifications)] + + +use enso_doc_parser::*; +use enso_parser::prelude::*; + + + +// ==================================== +// === Debug Representation Printer === +// ==================================== + +fn main() { + init_global(); + let args = std::env::args().skip(1); + if args.is_empty() { + use std::io::Read; + let mut input = String::new(); + std::io::stdin().read_to_string(&mut input).unwrap(); + check_doc_parse("", input.as_str()); + } else { + args.for_each(|path| check_doc_parse(&path, &std::fs::read_to_string(&path).unwrap())); + } +} + +/// Print the token for the input file. +fn check_doc_parse(filename: &str, code: &str) { + println!("File: {filename}"); + let docs = extract_docs(filename, code); + for doc in &docs { + for token in parse(doc) { + println!("{token:?}"); + } + } +} + +/// Extract docs from the input file. +fn extract_docs(_filename: &str, mut code: &str) -> Vec { + if let Some((_meta, code_)) = enso_parser::metadata::parse(code) { + code = code_; + } + let ast = enso_parser::Parser::new().run(code); + let docs = RefCell::new(vec![]); + ast.map(|tree| match &*tree.variant { + enso_parser::syntax::tree::Variant::Documented(doc) => { + docs.borrow_mut().push(doc.documentation.clone()); + } + enso_parser::syntax::tree::Variant::CaseOf(case_of) => { + for case in case_of.cases.iter().filter_map(|c| c.case.as_ref()) { + docs.borrow_mut().extend(case.documentation.clone()); + } + } + _ => {} + }); + docs.take().into_iter().map(|node| node.content()).collect() +} + +/// Lex the given documentation, and return the sequence of tokens. +fn parse(input: &str) -> Vec { + let mut docs = TokenCollector::::default(); + let mut lexer = Lexer::default(); + for (line_number, line) in input.trim_start().lines().enumerate() { + let location = Location::start_of_line(line_number); + let line = Span { location, text: line }; + lexer.line::(line, &mut docs); + } + lexer.finish(&mut docs); + docs.tokens +} + + + +// ======================= +// === Token Collector === +// ======================= + +/// Token consumer that reifies the sequence of tokens for debugging and tests. +#[derive(Default, Debug)] +struct TokenCollector { + tokens: Vec, + location_type: PhantomData, +} + +#[derive(Debug)] +enum Token { + Tag { tag: Tag, description: String }, + EnterMarkedSection { mark: Mark, header: String }, + EnterKeyedSection { header: String }, + Start(ScopeType), + End(ScopeType), + StartQuote, + EndQuote, + Text(String), + RawLine(String), +} + +impl TokenConsumer for TokenCollector { + fn tag(&mut self, tag: Tag, description: Option>) { + self.tokens.push(Token::Tag { + tag, + description: description.map(String::from).unwrap_or_default(), + }) + } + + fn enter_marked_section(&mut self, mark: Mark, header: Option>) { + self.tokens.push(Token::EnterMarkedSection { + mark, + header: header.map(String::from).unwrap_or_default(), + }) + } + + fn enter_keyed_section(&mut self, header: Span) { + self.tokens.push(Token::EnterKeyedSection { header: header.into() }) + } + + fn text(&mut self, text: Span) { + match self.tokens.last_mut() { + Some(Token::Text(current)) => { + current.push(' '); + current.push_str(text.text.as_ref()) + } + _ => self.tokens.push(Token::Text(text.text.into())), + } + } + + fn start_list(&mut self) { + self.tokens.push(Token::Start(ScopeType::List)); + } + + fn start_list_item(&mut self) { + self.tokens.push(Token::Start(ScopeType::ListItem)); + } + + fn start_paragraph(&mut self) { + self.tokens.push(Token::Start(ScopeType::Paragraph)); + } + + fn start_raw(&mut self) { + self.tokens.push(Token::Start(ScopeType::Raw)); + } + + fn start_quote(&mut self) { + self.tokens.push(Token::StartQuote); + } + + fn end_quote(&mut self) { + self.tokens.push(Token::EndQuote); + } + + fn whitespace(&mut self) { + self.tokens.push(Token::Text(" ".to_owned())); + } + + fn raw_line(&mut self, text: Span) { + self.tokens.push(Token::RawLine(text.text.into())); + } + + fn end(&mut self, scope: ScopeType) { + self.tokens.push(Token::End(scope)); + } +} diff --git a/lib/rust/parser/src/syntax/tree.rs b/lib/rust/parser/src/syntax/tree.rs index 72110bf386ba..33aa1864167b 100644 --- a/lib/rust/parser/src/syntax/tree.rs +++ b/lib/rust/parser/src/syntax/tree.rs @@ -503,20 +503,15 @@ impl<'s> DocComment<'s> { /// empty lines removed; newlines will be normalized. pub fn content(&self) -> String { let mut buf = String::new(); - macro_rules! emit_token { - ($buf:expr, $token:expr) => {{ - $buf.push_str(&$token.left_offset.code.repr); - $buf.push_str(&$token.code.repr); - }}; - } for element in &self.elements { match element { TextElement::Section { text } => buf.push_str(&text.code.repr), - TextElement::Escape { token } => emit_token!(buf, token), - TextElement::Newline { newline } => { - buf.push_str(&newline.left_offset.code.repr); - buf.push('\n'); + TextElement::Newline { .. } => buf.push('\n'), + TextElement::Escape { token } if let Some(c) = token.value => { + buf.push(c); } + // Invalid escape character, ignore it. + TextElement::Escape { .. } => (), // Unreachable. TextElement::Splice { .. } => continue, } diff --git a/lib/scala/docs-generator/README.md b/lib/scala/docs-generator/README.md deleted file mode 100644 index 6345c0ada39d..000000000000 --- a/lib/scala/docs-generator/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Docs Generator - -A service generating docs from contents of library, and saving them as `.html` -files in a `docs` subdirectory. diff --git a/lib/scala/docs-generator/src/bench/java/org/enso/docs/generator/package-info.java b/lib/scala/docs-generator/src/bench/java/org/enso/docs/generator/package-info.java deleted file mode 100644 index 535727c76e8e..000000000000 --- a/lib/scala/docs-generator/src/bench/java/org/enso/docs/generator/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.enso.docs.generator; diff --git a/lib/scala/docs-generator/src/bench/scala/org/enso/docs/generator/DocsGeneratorBench.scala b/lib/scala/docs-generator/src/bench/scala/org/enso/docs/generator/DocsGeneratorBench.scala deleted file mode 100644 index 845496ebb94e..000000000000 --- a/lib/scala/docs-generator/src/bench/scala/org/enso/docs/generator/DocsGeneratorBench.scala +++ /dev/null @@ -1,59 +0,0 @@ -package org.enso.docs.generator - -import org.enso.docs.generator.DocsGenerator -import org.scalameter.api._ -import org.scalameter.execution.LocalExecutor -import org.scalameter.picklers.Implicits._ -import scala.math.pow - -object DocsGeneratorBench extends Bench.OfflineRegressionReport { - - override def executor = new LocalExecutor(warmer, aggregator, measurer) - - val range = 0 - - def exp(i: Int): Gen[Int] = - Gen.exponential("size")( - pow(2.0, (i - range).toDouble).toInt, - pow(2.0, i.toDouble).toInt, - 2 - ) - - def gen(range: Gen[Int], f: Int => List[String]): Gen[String] = - for { i <- range } yield f(i).toString() - - val tests = List( - "formatters" -> gen(exp(14), i => List.fill(i)("*foo bar*\n")), - "unclosed" -> gen(exp(14), i => List.fill(i)("*_foobar*\n")), - "combined" -> gen(exp(14), i => List.fill(i)("*_~fo0~_*\n")), - "normal" -> gen(exp(14), i => List.fill(i)("test12345\n")), - "link" -> gen(exp(14), i => List.fill(i)("[foo](bo)\n")), - "tags" -> gen(exp(14), i => List.fill(i)("ADDED\nfoo\n")), - "list" -> gen( - exp(13), - i => List.fill(i)("""foo - | - A - | - B - | - C - |""".stripMargin) - ), - "list_nested" -> gen( - exp(12), - i => List.fill(i)("""foo - | - A - | - B - | * CA - | * CB - | - D - |""".stripMargin) - ), - "sections" -> gen(exp(13), i => List.fill(i)("Foo\n\n!B\n\n?C \n\n>D \n\n")) - ) - - def run(str: String): List[String] = DocsGenerator.generate(List(str)) - performance of "Docs Generator" in { - tests.foreach { case (name, gen) => - measure method name in (using(gen) in run) - } - } -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Constants.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Constants.scala deleted file mode 100644 index a09b743c6a8d..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Constants.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.enso.docs.generator - -object Constants { - val TEMPLATE_FILES_PATH = - "./lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/" - val SOURCE_PATH = "./distribution/lib/Standard" - val JS_TEMPLATE_NAME = "template.js" - val CSS_TREE_FILE_NAME = "treeStyle.css" - val OUTPUT_DIRECTORY = "docs-js" - - val HELP_OPTION = "help" - val INPUT_PATH_OPTION = "input-path" - val OUTPUT_DIR_OPTION = "output-dir" - val DOCS_LIB_PATH_OPTION = "docs-lib-path" - val JS_TEMPLATE_OPTION = "js-template" - val CSS_TEMPLATE_OPTION = "css-template" -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocParserWrapper.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocParserWrapper.scala deleted file mode 100644 index 0b4de9298543..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocParserWrapper.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.enso.docs.generator - -import org.enso.syntax.text.{DocParser, Parser} -import org.enso.syntax.text.docparser._ - -import java.io.File - -/** Defines useful wrappers for Doc Parser. - */ -object DocParserWrapper { - - /** Generates HTML of docs from Enso program. - */ - def run(program: String): String = { - val parser = new Parser() - val module = parser.run(program) - val dropMeta = parser.dropMacroMeta(module) - val doc = DocParserRunner.createDocs(dropMeta) - val code = DocParserHTMLGenerator.generateHTMLForEveryDocumented(doc) - code - } - - /** Generates HTML from Documentation string. - */ - def runOnPureDoc(comment: String, title: String = ""): String = { - val doc = DocParser.runMatched(comment) - val html = DocParserHTMLGenerator.generateHTMLPureDoc(doc, title) - html - } - - /** Called if file doesn't contain docstrings, to let user know that they - * won't find anything at this page, and that it is not a bug. - */ - def mapIfEmpty(doc: String): String = { - if (doc.replace("
", "").replace("
", "").length == 0) { - val placeholder = - "\n\n*Enso Reference Viewer.*\n\nNo documentation available for chosen source file." - val generatedPlaceholderCode = - runOnPureDoc(placeholder).replace("style=\"font-size: 13px;\"", "") - return generatedPlaceholderCode - } - doc - } - - /** Doc Parser may output file with many nested empty divs. - * This simple function will remove all unnecessary HTML tags. - */ - def removeUnnecessaryDivs(doc: String): String = { - var tmp = doc - while (tmp.contains("
")) - tmp = tmp.replace("
", "") - tmp - } - - /** Traverses through root directory, outputs list of all accessible files. - */ - def traverse(root: File): LazyList[File] = - if (!root.exists) { LazyList.empty } - else { - LazyList.apply(root) ++ (root.listFiles match { - case null => LazyList.empty - case files => files.view.flatMap(traverse) - }) - } -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocsGenerator.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocsGenerator.scala deleted file mode 100644 index b2dbb567ed9c..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocsGenerator.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.enso.docs.generator - -import org.enso.docs.generator.DocParserWrapper.runOnPureDoc -import org.enso.syntax.text.AST -import org.enso.syntax.text.docparser.DocParserHTMLGenerator -import scalatags.Text.{all => HTML} -import HTML._ - -/** Defines methods for generating and glue-ing together doc content. - */ -class DocsGenerator { - - /** Generates list of HTML docs from given doc comments in AST. - */ - def generateFromAst(ast: List[AST.Comment]): List[String] = { - generate(ast.map(_.show())) - } - - /** Generate an HTML doc from the given doc comment. */ - def generate(comment: String, title: String): String = - runOnPureDoc(comment, title) - - /** Generates list of HTML docs from given doc comments. - */ - def generate(comments: List[String]): List[String] = { - comments.map(c => runOnPureDoc(c)) - } - - /** Connects HTML documentation with it's AST element. - */ - def connectHtmlToAst(html: String, ast: AST): String = { - val astHTML = DocParserHTMLGenerator.createHTMLFromAST(ast) - val astName = HTML.div(astHTML.header) - astHTML.body match { - case Some(body) => - HTML.div(HTML.`class` := "main ml-20")(astName, html, body).render - case None => HTML.div(astName, html).render - } - } - - /** Glues together many docs with AST elements in proper order to create one - * documentation page. - */ - def glueGeneratedContent(docs: List[String], astList: List[AST]): String = { - docs.zip(astList).map(e => connectHtmlToAst(e._1, e._2)).mkString - } -} -object DocsGenerator extends DocsGenerator diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Main.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Main.scala deleted file mode 100644 index a845e9363476..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Main.scala +++ /dev/null @@ -1,226 +0,0 @@ -package org.enso.docs.generator - -import java.io._ -import org.apache.commons.cli.{Option => CliOption, _} -import scala.util.{Try, Using} -import scala.io.Source -import scalatags.Text.{all => HTML} -import TreeOfCommonPrefixes._ -import DocParserWrapper._ -import Constants._ -import HTML._ - -/** The entry point for the documentation generator. - * - * The documentation generator is responsible for creating HTML documentation - * for the Enso standard library. - * It also generates JavaScript files containing react components for the - * [[https://enso.org/docs/reference reference website]]. - */ -object Main { - - /** Builds the [[Options]] object representing the CLI syntax. - * - * @return an [[Options]] object representing the CLI syntax - */ - private def buildOptions = { - val help = CliOption - .builder("h") - .longOpt(HELP_OPTION) - .desc("Displays this message.") - .build - val input = CliOption.builder - .longOpt(INPUT_PATH_OPTION) - .numberOfArgs(1) - .argName("path") - .desc("Specifies working path.") - .build - val output = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("directory") - .longOpt(OUTPUT_DIR_OPTION) - .desc("Specifies name of output directory.") - .build - val docsPath = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("path") - .longOpt(DOCS_LIB_PATH_OPTION) - .desc("Specifies path of Docs Generator library.") - .build - val jsTemp = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("file") - .longOpt(JS_TEMPLATE_OPTION) - .desc("Specifies name of file containing JS template.") - .build - val cssTemp = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("file") - .longOpt(CSS_TEMPLATE_OPTION) - .desc("Specifies name of file containing CSS template.") - .build - - val options = new Options - options - .addOption(help) - .addOption(input) - .addOption(output) - .addOption(docsPath) - .addOption(jsTemp) - .addOption(cssTemp) - - options - } - - /** Prints the help message to the standard output. - * - * @param options object representing the CLI syntax - */ - private def printHelp(options: Options): Unit = - new HelpFormatter().printHelp("Docs Generator", options) - - /** Terminates the process with a failure exit code. */ - private def exitFail(): Nothing = sys.exit(1) - - /** Terminates the process with a success exit code. */ - private def exitSuccess(): Nothing = sys.exit(0) - - /** Starting point. */ - def main(args: Array[String]): Unit = { - val options = buildOptions - val parser = new DefaultParser - val line = Try(parser.parse(options, args)).getOrElse { - printHelp(options) - exitFail() - } - if (line.hasOption(HELP_OPTION)) { - printHelp(options) - exitSuccess() - } - val path = - Option(line.getOptionValue(INPUT_PATH_OPTION)).getOrElse(SOURCE_PATH) - val outDir = - Option(line.getOptionValue(OUTPUT_DIR_OPTION)).getOrElse(OUTPUT_DIRECTORY) - val templateFilesPath = Option(line.getOptionValue(DOCS_LIB_PATH_OPTION)) - .getOrElse(TEMPLATE_FILES_PATH) - val jsTempFileName = Option(line.getOptionValue(JS_TEMPLATE_OPTION)) - .getOrElse(JS_TEMPLATE_NAME) - val cssFileName = Option(line.getOptionValue(CSS_TEMPLATE_OPTION)) - .getOrElse(CSS_TREE_FILE_NAME) - - generateAllDocs( - path, - templateFilesPath, - jsTempFileName, - cssFileName, - outDir - ) - } - - private val sourceStdlibVersion = "0.0.0-dev" - - /** Traverses through directory generating docs from every .enso file found. - */ - def generateAllDocs( - path: String, - templateFilesPath: String, - jsTempFileName: String, - cssFileName: String, - outDir: String - ): Unit = { - val allFiles = traverse(new File(path)) - .filter(f => f.isFile && f.getName.endsWith(".enso")) - val allFileNames = allFiles.map( - _.getPath - .replace(path + "/", "") - .replace(".enso", "") - .replace("src/", "") - .replace(s"/$sourceStdlibVersion/", "/") - ) - val allPrograms = allFiles - .map(f => Using(Source.fromFile(f, "UTF-8")) { _.mkString }) - .toList - val allDocs = allPrograms - .map(s => run(s.getOrElse(""))) - .map(mapIfEmpty) - .map(removeUnnecessaryDivs) - val treeNames = - groupByPrefix(allFileNames.toList, '/').filter(_.elems.nonEmpty) - val jsTemplate = new File(templateFilesPath + jsTempFileName) - val templateCode = Using(Source.fromFile(jsTemplate, "UTF-8")) { - _.mkString - } - val styleFile = new File(templateFilesPath + cssFileName) - val styleCode = Using(Source.fromFile(styleFile, "UTF-8")) { _.mkString } - val treeStyle = "" - val allDocJSFiles = allFiles.map { x => - val name = x.getPath - .replace(".enso", ".js") - .replace("lib/Standard/", outDir + "/") - .replace("Main.js", "index.js") - .replace(s"/$sourceStdlibVersion/", "/") - .replace("src/", "") - val ending = name.split(outDir + "/").tail.head - name.replace(ending, ending.replace('/', '-')) - } - val dir = new File(allDocJSFiles.head.split(outDir).head + outDir + "/") - dir.mkdirs() - val zippedJS = allDocJSFiles.zip(allDocs) - zippedJS.foreach(d => - createDocJSFile(d._1, d._2, outDir, treeStyle, templateCode, treeNames) - ) - } - - /** Takes a tuple of file path and documented HTML code, and generates JS doc - * file with react components for Enso website. - */ - private def createDocJSFile( - path: String, - htmlCode: String, - outDir: String, - treeStyle: String, - templateCode: Try[String], - treeNames: List[Node] - ): Unit = { - val file = new File(path) - file.createNewFile() - val bw = new BufferedWriter(new FileWriter(file)) - var treeCode = - "
" + treeStyle + HTML - .ul(treeNames.map(_.html())) - .render - .replace("class=", "className=") + "
" - if (path.contains("/index.js")) { - treeCode = treeCode.replace("a href=\"", "a href=\"reference/") - } - val partials = path - .split(outDir + "/") - .tail - .head - .replace(".js", "") - .split("-") - for (i <- 1 to partials.length) { - val id = partials.take(i).mkString("-") - val beg = " HTML} -import HTML._ - -/** An algorithm creating a tree from a list of strings separated by common - * character into list of list of words, then from that creating a tree with - * branches from same words on the same depth. - */ -object TreeOfCommonPrefixes { - - /** A single tree node of given `name` containing other nodes as list called - * `elems`. - */ - case class Node(name: String, var elems: List[Node]) { - - /** Generates an HTML tree from a tree of nodes, as this method is used to - * create the page chooser. - */ - def html(beg: String = ""): Modifier = { - val newBeg = if (beg.length > 0) beg + "-" + name else name - if (elems.isEmpty) { - HTML.li(HTML.a(HTML.href := newBeg)(name)) - } else { - HTML.li(HTML.`class` := "section")( - HTML.input(HTML.`type` := "checkbox", HTML.id := newBeg), - HTML.label(HTML.`for` := newBeg)( - HTML.a(HTML.href := newBeg)(name) - ), - elems.map(x => HTML.ul(x.html(newBeg))) - ) - } - } - } - - /** Groups a list of nodes by it's prefixes. - */ - def groupNodesByPrefix(le: List[Node], sep: Char): List[Node] = - groupByPrefix(le.map(_.name), sep) - - /** A function that groups a list of strings by it's prefixes, actually - * creating the aforementioned tree. - */ - def groupByPrefix(ls: List[String], sep: Char): List[Node] = { - var nodes = List[Node]() - for (string <- ls) { - if (string.split(sep).length <= 1) { - nodes = nodes :+ Node(string, List()) - } else { - val arr = string.split(sep) - val filtered = nodes.filter(x => x.name == arr.head) - if (filtered.nonEmpty && nodes.contains(filtered.head)) { - nodes.map(n => - if (n == filtered.head) { - n.elems = n.elems :+ Node(arr.tail.mkString(sep.toString), List()) - } - ) - } else { - nodes = nodes :+ Node( - arr.head, - List(Node(arr.tail.mkString(sep.toString), List())) - ) - } - } - } - for (node <- nodes) { - node.elems = groupNodesByPrefix(node.elems, sep) - } - nodes - } -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/template.js b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/template.js deleted file mode 100644 index 3dd9c8a39fe8..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/template.js +++ /dev/null @@ -1,53 +0,0 @@ -import { StaticNavigation } from 'components/navigation' -import { Container, ContainerOrScreenIfSmall, RootContainer } from 'components/container' -import { Header } from 'components/header' -import { Chapter } from 'components/chapter' -import { SectionCommunity } from 'components/section-community' -import { SectionFooter } from 'components/section-footer' -import { StickyButtons } from 'components/sticky-buttons' - -import AtomsIcon from '../../../public/img/icon/atoms.svg' -import MethodsIcon from '../../../public/img/icon/methods.svg' -import SubmodulesIcon from '../../../public/img/icon/submodules.svg' - -function Docs() { - return ( -
- {/*
*/} - {/* /!*BREADCRUMBS2*!/*/} - {/*
*/} - -
-
{/*BREADCRUMBS*/}
- {/*PAGE*/} -
- -
- ) -} - -export default function Main(props) { - return ( - -
- - -
- -
- -
- -
- - - - - - - - - - - ) -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/treeStyle.css b/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/treeStyle.css deleted file mode 100644 index 2d01bac8eb68..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/treeStyle.css +++ /dev/null @@ -1,46 +0,0 @@ -.section ul { - cursor: pointer; - list-style: none; - padding: 0; - margin: 0; - word-wrap: initial; -} - -.section span { - cursor: pointer; -} - -ul li { - padding: 5px 10px; -} - -.section ul { - display: none; -} -.section input:checked ~ ul { - display: block; -} -.section input[type="checkbox"] { - display: none; -} -.section { - position: relative; - padding-left: 20px !important; - list-style-type: none; -} - -.section label:after { - content: "â–¶"; - position: absolute; - top: 7px; - left: 0; - padding: 0; - text-align: center; - font-size: 14px; - color: rgba(75, 88, 100, var(--tw-text-opacity)); - transition: all 0.3s; -} - -.section input:checked ~ label:after { - transform: rotate(90deg); -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala deleted file mode 100644 index 1400d9dc04c6..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala +++ /dev/null @@ -1,71 +0,0 @@ -package org.enso.docs.sections - -import org.enso.polyglot.DocSection -import org.enso.syntax.text.DocParser -import org.enso.syntax.text.ast.Doc - -/** Module that splits the documentation into sections. - * - * @param parsedSectionsBuilder creates documenation sections from the parsed - * docstring - */ -final class DocSectionsBuilder(parsedSectionsBuilder: ParsedSectionsBuilder) { - - import DocSectionsBuilder._ - - /** Create the list of docsections from the documentation comment. - * - * @param comment the documentation comment - * @return the list of documentation sections - */ - def build(comment: String): List[DocSection] = { - val doc = DocParser.runMatched(comment) - val parsed = parsedSectionsBuilder.build(doc) - parsed.map(render) - } - - /** Converts the docsection AST into a human-readable representation. - * - * @param section the docsection AST - */ - private def render(section: Section): DocSection = - section match { - case Section.Tag(name, body) => - DocSection.Tag(name, renderElems(body)) - - case Section.Paragraph(body) => - DocSection.Paragraph(renderElems(body)) - - case Section.Keyed(key, body) => - DocSection.Keyed(key, renderElems(body)) - - case Section.Marked(mark, header, body) => - DocSection.Marked(buildMark(mark), header, renderElems(body)) - } - - /** Convert the [[Section.Mark]] into [[DocSection.Mark]] representation. - * - * @param mark the section mark AST - * @return the [[DocSection.Mark]] mark representation - */ - private def buildMark(mark: Section.Mark): DocSection.Mark = - mark match { - case Section.Mark.Important => DocSection.Mark.Important() - case Section.Mark.Info => DocSection.Mark.Info() - case Section.Mark.Example => DocSection.Mark.Example() - } -} -object DocSectionsBuilder { - - /** @return the instance of [[DocSectionsBuilder]]. */ - def apply(): DocSectionsBuilder = - new DocSectionsBuilder(new ParsedSectionsBuilder) - - private def renderElems(elems: Seq[Doc.Elem]): String = { - val builder = - elems.foldLeft(Seq.newBuilder[scalatags.Text.all.Modifier]) { (b, a) => - b ++= HtmlRepr[Doc.Elem].toHtml(a) - } - HtmlRepr.renderHtml(builder.result()) - } -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/HtmlRepr.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/HtmlRepr.scala deleted file mode 100644 index da7782940a96..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/HtmlRepr.scala +++ /dev/null @@ -1,148 +0,0 @@ -package org.enso.docs.sections - -import org.enso.syntax.text.ast.Doc -import scalatags.Text.{all => HTML} -import scalatags.Text.all._ - -/** Convertor to the [[Doc.HTML]]. */ -trait HtmlRepr[A] { - - /** Converts object to HTML representation. - * - * @param a the object to convert - * @return the [[Doc.HTML]] representation of the object - */ - def toHtml(a: A): Doc.HTML -} -object HtmlRepr { - - private val DivOpenLength = 5 - private val DivCloseLength = 6 - - /** Obtain an instance of [[HtmlRepr]] for the given type. */ - def apply[A: HtmlRepr]: HtmlRepr[A] = - implicitly[HtmlRepr[A]] - - /** Renders the [[Doc.HTML]] into string. - * - * @param elems the HTML AST - * @return the string representation of HTML AST - */ - def renderHtml(elems: Doc.HTML): String = - scalatags.Text.all - .div(elems: _*) - .toString - .drop(DivOpenLength) - .dropRight(DivCloseLength) - - val newlineRepr: String = " " - - implicit val newlineHtmlRepr: HtmlRepr[Doc.Elem.Newline.type] = { _ => - Seq(newlineRepr) - } - - implicit val textHtmlRepr: HtmlRepr[Doc.Elem.Text] = { elem => - Seq(elem.text) - } - - implicit val codeBlockHtmlRepr: HtmlRepr[Doc.Elem.CodeBlock] = { elem => - val firstIndent = elem.elems.head.indent - val elemsHtml = elem.elems.toList.map(_.htmlOffset(firstIndent)) - Seq(HTML.pre(elemsHtml)) - } - - implicit val inlineCodeBlockHtmlRepr: HtmlRepr[Doc.Elem.CodeBlock.Inline] = { - elem => Seq(HTML.code(elem.str)) - } - - implicit val codeBlockLineHtmlRepr: HtmlRepr[Doc.Elem.CodeBlock.Line] = { - elem => Seq(HTML.code(" " * elem.indent + elem.elem), HTML.br) - } - - implicit val unclosedHtmlRep: HtmlRepr[Doc.Elem.Formatter.Unclosed] = { - elem => - val elems = Doc.Elem.Text(elem.typ.marker.toString) :: elem.elems - Seq(elems.map(htmlRepr.toHtml)) - } - - implicit val formatterHtmlRepr: HtmlRepr[Doc.Elem.Formatter] = { elem => - Seq(elem.typ.htmlMarker(elem.elems.map(htmlRepr.toHtml))) - } - - implicit val headerHtmlRepr: HtmlRepr[Doc.Section.Header] = { elem => - Seq(HTML.h1(elem.elems.map(htmlRepr.toHtml))) - } - - implicit val imageHtmlRepr: HtmlRepr[Doc.Elem.Link.Image] = { elem => - Seq(HTML.img(HTML.src := elem.url), elem.name) - } - - implicit val urlHtmlRepr: HtmlRepr[Doc.Elem.Link.URL] = { elem => - Seq(HTML.a(HTML.href := elem.url)(elem.name)) - } - - implicit val invalidLinkHtmlRepr: HtmlRepr[Doc.Elem.Link.Invalid] = { elem => - Seq(elem.repr.build()) - } - - implicit val tagHtmlRepr: HtmlRepr[Doc.Tags.Tag] = { elem => - Seq(elem.details.fold(elem.name)(elem.name + " " + _)) - } - - implicit val listItemHtmlRepr: HtmlRepr[Doc.Elem.ListItem] = { elem => - Seq(elem.elems.map(htmlRepr.toHtml)) - } - - implicit val listMisalignedItemHtmlRepr: HtmlRepr[Doc.Elem.MisalignedItem] = { - elem => Seq(elem.elems.map(htmlRepr.toHtml)) - } - - implicit val listHtmlRepr: HtmlRepr[Doc.Elem.List] = { elem => - val elemsHTML = elem.elems.reverse.toList.map { - case elem: Doc.Elem.List => - listHtmlRepr.toHtml(elem) - case elem: Doc.Elem.ListItem => - Seq(HTML.li(listItemHtmlRepr.toHtml(elem))) - case elem: Doc.Elem.MisalignedItem => - Seq(HTML.li(listMisalignedItemHtmlRepr.toHtml(elem))) - } - Seq(elem.typ.HTMLMarker(elemsHTML)) - } - - implicit val htmlRepr: HtmlRepr[Doc.Elem] = { - case Doc.Elem.Newline => - newlineHtmlRepr.toHtml(Doc.Elem.Newline) - case elem: Doc.Elem.Text => - HtmlRepr[Doc.Elem.Text].toHtml(elem) - case elem: Doc.Elem.Formatter.Unclosed => - HtmlRepr[Doc.Elem.Formatter.Unclosed].toHtml(elem) - case elem: Doc.Elem.CodeBlock => - HtmlRepr[Doc.Elem.CodeBlock].toHtml(elem) - case elem: Doc.Elem.CodeBlock.Inline => - HtmlRepr[Doc.Elem.CodeBlock.Inline].toHtml(elem) - case elem: Doc.Elem.CodeBlock.Line => - HtmlRepr[Doc.Elem.CodeBlock.Line].toHtml(elem) - case elem: Doc.Elem.Formatter => - HtmlRepr[Doc.Elem.Formatter].toHtml(elem) - case elem: Doc.Section.Header => - HtmlRepr[Doc.Section.Header].toHtml(elem) - case elem: Doc.Elem.Link.Invalid => - HtmlRepr[Doc.Elem.Link.Invalid].toHtml(elem) - case elem: Doc.Tags.Tag => - HtmlRepr[Doc.Tags.Tag].toHtml(elem) - case elem: Doc.Elem.ListItem => - HtmlRepr[Doc.Elem.ListItem].toHtml(elem) - case elem: Doc.Elem.MisalignedItem => - HtmlRepr[Doc.Elem.MisalignedItem].toHtml(elem) - case elem: Doc.Elem.List => - HtmlRepr[Doc.Elem.List].toHtml(elem) - case link: Doc.Elem.Link => - link match { - case elem: Doc.Elem.Link.URL => - HtmlRepr[Doc.Elem.Link.URL].toHtml(elem) - case elem: Doc.Elem.Link.Image => - HtmlRepr[Doc.Elem.Link.Image].toHtml(elem) - } - } - -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/ParsedSectionsBuilder.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/ParsedSectionsBuilder.scala deleted file mode 100644 index 5a97bf072a31..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/ParsedSectionsBuilder.scala +++ /dev/null @@ -1,205 +0,0 @@ -package org.enso.docs.sections - -import cats.kernel.Monoid -import cats.syntax.compose._ -import org.enso.syntax.text.ast.Doc - -import scala.collection.mutable - -/** Combine the documentation into a list of [[Section]]s. */ -final class ParsedSectionsBuilder { - - import ParsedSectionsBuilder._ - - /** Build the parsed sections from the provided documentation comment. - * - * @param doc the parsed documentation comment. - * @return the list of parsed sections. - */ - def build(doc: Doc): List[Section] = { - val tagSections = doc.tags.map(buildTags) - val synopsisSections = doc.synopsis.map(buildSynopsis) - val bodySections = doc.body.map(buildBody) - Monoid.combineAll(tagSections ++ synopsisSections ++ bodySections) - } - - /** Process the tags section of the documentation comment. - * - * @param tags the tags section - * @return the list of parsed sections - */ - private def buildTags(tags: Doc.Tags): List[Section] = - tags.elems.toList.map { tag => - Section.Tag(tag.name, tag.details.map(_.trim).map(Doc.Elem.Text).toList) - } - - /** Process the synopsis section of the documentation comment. - * - * @param synopsis the synopsis section - * @return the list of parsed sections - */ - private def buildSynopsis(synopsis: Doc.Synopsis): List[Section] = - (preprocess >>> buildSections)(synopsis.elems.toList) - - /** Process the body section of the documentation comment. - * - * @param body the body section - * @return the list of parsed sections - */ - private def buildBody(body: Doc.Body): List[Section] = - (preprocess >>> buildSections)(body.elems.toList) - - /** Process the list of [[Doc.Section]] documentation sections. - * - * @param sections the list of parsed documentation sections - * @return the list of parsed sections - */ - private def buildSections(sections: List[Doc.Section]): List[Section] = - sections.flatMap { - case Doc.Section.Raw(_, elems) => - buildRaw(elems) - - case Doc.Section.Marked(_, _, typ, elems) => - elems match { - case head :: tail => - val header = head match { - case header: Doc.Section.Header => - Some(header.repr.build()) - case _ => - None - } - List(Section.Marked(buildMark(typ), header, tail)) - case Nil => - List(Section.Marked(buildMark(typ), None, elems)) - } - } - - /** Create the [[Section.Mark]] from the [[Doc.Section.Marked.Type]] - * section type. - * - * @param typ the type of documentation section - * @return the corresponding section mark - */ - private def buildMark(typ: Doc.Section.Marked.Type): Section.Mark = - typ match { - case Doc.Section.Marked.Important => Section.Mark.Important - case Doc.Section.Marked.Info => Section.Mark.Info - case Doc.Section.Marked.Example => Section.Mark.Example - } - - /** The preprocessor function that is invoked before building the - * resulting list of sections. - */ - private def preprocess: List[Doc.Section] => List[Doc.Section] = - joinSections _ >>> filterSections - - /** Preprocess the list of documentation sections and join the paragraphs of - * the same offset with the marked section. - * - * ==Example== - * In the parsed [[Doc.Section]], the "Some paragraph" is a separate section, - * while having the same indentation. This pass joins them into a single - * section. - * - * {{{ - * ? Info - * Some info. - * - * Some paragraph. - * }}} - * - * @param sections the list of documentation sections - * @return preprocessed list of documentation sections with the - * paragraphs joined into the corresponding marked sections. - */ - private def joinSections(sections: List[Doc.Section]): List[Doc.Section] = { - val init: Option[Doc.Section.Marked] = None - val stack: List[Doc.Section] = Nil - - val (result, acc) = sections.foldLeft((stack, init)) { - case ((stack, acc), section) => - (section, acc) match { - case (marked: Doc.Section.Marked, _) => - (acc.toList ::: stack, Some(marked)) - case (raw: Doc.Section.Raw, None) => - (raw :: stack, acc) - case (raw: Doc.Section.Raw, Some(marked)) => - if (raw.indent == marked.indent) { - val newElems = marked.elems ::: Doc.Elem.Newline :: raw.elems - (stack, Some(marked.copy(elems = newElems))) - } else { - (raw :: marked :: stack, None) - } - } - } - - (acc.toList ::: result).reverse - } - - /** Filter the sections before processing them. - * - * The function filters out: - * - empty sections - * - * @param sections the list of documentation sections - * @return the list of filtered sections - */ - private def filterSections(sections: List[Doc.Section]): List[Doc.Section] = - sections - .filter { - case Doc.Section.Raw(_, List(Doc.Elem.Newline)) => false - case _ => true - } - - /** Builds sections from the [[Doc.Section.Raw]] raw section. - * - * @param elems the elements of the raw section - * @return the resulting list of sections - */ - private def buildRaw(elems: List[Doc.Elem]): List[Section] = - elems match { - case Doc.Elem.Text(text) :: tail => - val (key, value) = text.span(_ != const.COLON) - if (value.nonEmpty) { - val line = value.drop(1).stripPrefix(const.SPACE) - val body = if (line.isEmpty) tail else Doc.Elem.Text(line) :: tail - val (keyBody, rest) = splitKeyed(body) - Section.Keyed(key, keyBody) :: buildRaw(rest) - } else { - List(Section.Paragraph(elems)) - } - case Nil => - Nil - case elems => - List(Section.Paragraph(elems)) - } - - private def splitKeyed( - elems: List[Doc.Elem] - ): (List[Doc.Elem], List[Doc.Elem]) = { - @scala.annotation.tailrec - def go( - elems: List[Doc.Elem], - b: mutable.Builder[Doc.Elem, List[Doc.Elem]] - ): (List[Doc.Elem], List[Doc.Elem]) = - elems match { - case Doc.Elem.Newline :: Doc.Elem.Text(text) :: tail - if text.contains(const.COLON) => - (b.result(), Doc.Elem.Text(text) :: tail) - case elem :: tail => - go(tail, b += elem) - case Nil => - (b.result(), Nil) - } - - go(elems, List.newBuilder) - } -} - -object ParsedSectionsBuilder { - - object const { - final val COLON = ':' - final val SPACE = " " - } -} diff --git a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/Section.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/Section.scala deleted file mode 100644 index 1f789792dece..000000000000 --- a/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/Section.scala +++ /dev/null @@ -1,94 +0,0 @@ -package org.enso.docs.sections -import org.enso.syntax.text.ast.Doc - -/** The base trait for the section. */ -sealed trait Section -object Section { - - /** The documentation tag. - * - * {{{ - * name text - * }}} - * - * ==Example== - * - * {{{ - * UNSTABLE - * DEPRECATED - * ALIAS Length - * }}} - * - * @param name the tag name - * @param body the tag text - */ - case class Tag(name: String, body: List[Doc.Elem]) extends Section - - /** The paragraph of the text. - * - * ==Example== - * - * {{{ - * Arbitrary text in the documentation comment. - * - * This is another paragraph. - * }}} - * - * @param body the elements that make up this paragraph - */ - case class Paragraph(body: List[Doc.Elem]) extends Section - - /** The section that starts with the key followed by the colon and the body. - * - * {{{ - * key: body - * }}} - * - * ==Example== - * - * {{{ - * Arguments: - * - one: the first - * - two: the second - * }}} - * - * {{{ - * Icon: table-from-rows - * }}} - * - * @param key the section key - * @param body the elements the make up the body of the section - */ - case class Keyed(key: String, body: List[Doc.Elem]) extends Section - - /** The section that starts with the mark followed by the header and the body. - * - * {{{ - * mark header - * body - * }}} - * - * ==Example== - * - * {{{ - * > Example - * This is how it's done. - * foo = bar baz - * }}} - * - * {{{ - * ! Notice - * This is important. - * }}} - */ - case class Marked(mark: Mark, header: Option[String], body: List[Doc.Elem]) - extends Section - - /** The base trait for the section marks. */ - sealed trait Mark - object Mark { - case object Important extends Mark - case object Info extends Mark - case object Example extends Mark - } -} diff --git a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala b/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala deleted file mode 100644 index 8dc47e2fb7f0..000000000000 --- a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala +++ /dev/null @@ -1,253 +0,0 @@ -package org.enso.docs.sections - -import org.enso.polyglot.DocSection -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class DocSectionsBuilderTest extends AnyWordSpec with Matchers { - - import DocSectionsBuilderTest._ - - "DocSectionsBuilder" should { - - "build example 1" in { - val comment = - """ ADVANCED - | UNSTABLE - | ALIAS foo, bar, baz - | - | Returns the method name of the method that could not be found. - | - | > Example - | Getting the method name from a no such method error. - | - | import Standard.Examples - | - | example_method_name = - | error = Examples.no_such_method - | error.method_name - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Tag("ADVANCED", ""), - DocSection.Tag("UNSTABLE", ""), - DocSection.Tag("ALIAS", "foo, bar, baz"), - DocSection.Paragraph( - "Returns the method name of the method that could not be found. " - ), - DocSection.Marked( - DocSection.Mark.Example(), - Some("Example"), - " Getting the method name from a no such method error. 
import Standard.Examples
example_method_name =
    error = Examples.no_such_method
    error.method_name
" - ) - ) - - build(comment) shouldEqual expected - } - - "build example 2" in { - val comment = - """ ADVANCED - | - | A function that can be used to indicate that something hasn't been - | implemented yet. - | - | Arguments: - | - message: A description of what implementation is missing. - | - | > Example - | Throwing an error to show that something is unimplemented. - | - | import Standard.Base.Error.Common as Errors - | - | example_unimplemented = Errors.unimplemented - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Tag("ADVANCED", ""), - DocSection.Paragraph( - "A function that can be used to indicate that something hasn't been implemented yet. " - ), - DocSection.Keyed( - "Arguments", - "
  • message: A description of what implementation is missing.
" - ), - DocSection.Marked( - DocSection.Mark.Example(), - Some("Example"), - " Throwing an error to show that something is unimplemented. 
import Standard.Base.Error.Common as Errors
example_unimplemented = Errors.unimplemented
" - ) - ) - - build(comment) shouldEqual expected - } - - "build example 3" in { - val comment = - """ ADVANCED - | - | Converts a polyglot value representing an array into a vector. - | - | Arguments: - | - arr: The polyglot array value to wrap into a vector. - | - | This is useful when wrapping polyglot APIs for further use in Enso. - | - | A vector allows to store an arbitrary number of elements in linear memory. It - | is the recommended data structure for most applications. - | - | ! Value Copying - | As Enso vectors implement immutable semantics, this constructor function - | makes a copy of each value in the argument array. - | - | If this didn't happen then it would be possible for the underlying array to - | be mutated under the hood, and sneak mutability into our immutable data. - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Tag("ADVANCED", ""), - DocSection.Paragraph( - "Converts a polyglot value representing an array into a vector. " - ), - DocSection.Keyed( - "Arguments", - "
  • arr: The polyglot array value to wrap into a vector.
" - ), - DocSection.Paragraph( - "This is useful when wrapping polyglot APIs for further use in Enso. " - ), - DocSection.Paragraph( - "A vector allows to store an arbitrary number of elements in linear memory. It is the recommended data structure for most applications. " - ), - DocSection.Marked( - DocSection.Mark.Important(), - Some("Value Copying"), - " As Enso vectors implement immutable semantics, this constructor function makes a copy of each value in the argument array. If this didn't happen then it would be possible for the underlying array to be mutated under the hood, and sneak mutability into our immutable data." - ) - ) - - build(comment) shouldEqual expected - } - - "build example 4" in { - val comment = - """ Returns the larger value of `this` and `that`. - | - | Arguments: - | - that: The number to compare `this` against. - | - | ? Math.max or Number.max - | While we provide the max method on `Number`, we find it more intuitive to - | write `Math.max a b` rather than `a.max b`. To that end, we recommend using - | the first style. - | - | > Example - | Find the maximum of 2 and 5. - | - | 2.max 5 - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Paragraph( - "Returns the larger value of this and that. " - ), - DocSection.Keyed( - "Arguments", - "
  • that: The number to compare this against.
" - ), - DocSection.Marked( - DocSection.Mark.Info(), - Some("Math.max or Number.max"), - " While we provide the max method on Number, we find it more intuitive to write Math.max a b rather than a.max b. To that end, we recommend using the first style. " - ), - DocSection.Marked( - DocSection.Mark.Example(), - Some("Example"), - " Find the maximum of 2 and 5. 
2.max 5
" - ) - ) - - build(comment) shouldEqual expected - } - - "build example 5" in { - val comment = - """ PRIVATE - | A key-value store. This type assumes all keys are pairwise comparable, - | using the `<`, `>` and `==` operators. - | - | Arguments: - | - s: The size of the tree at this node. - | - key: The key stored at this node. - | - value: The value stored at this node. - | - left: The left subtree. - | - right: The right subtree. - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Tag("PRIVATE", ""), - DocSection.Paragraph( - "A key-value store. This type assumes all keys are pairwise comparable, using the <, > and == operators. " - ), - DocSection.Keyed( - "Arguments", - "
  • s: The size of the tree at this node.
  • key: The key stored at this node.
  • value: The value stored at this node.
  • left: The left subtree.
  • right: The right subtree.
" - ) - ) - - build(comment) shouldEqual expected - } - - "build example 6" in { - val comment = - """ UNSTABLE - | - | Creates a new table from a vector of column names and a vector of vectors - | specifying row contents. - | - | Arguments: - | - header: A list of texts specifying the column names - | - rows: A vector of vectors, specifying the contents of each table row. The - | length of each element of `rows` must be equal in length to `header`. - | - | > Example - | Create a table with 3 columns, named `foo`, `bar`, and `baz`, containing - | `[1, 2, 3]`, `[True, False, True]`, and `['a', 'b', 'c']`, respectively. - | - | import Standard.Table - | - | example_from_rows = - | header = [ 'foo' , 'bar' , 'baz' ] - | row_1 = [ 1 , True , 'a' ] - | row_2 = [ 2 , False , 'b' ] - | row_3 = [ 3 , True , 'c' ] - | Table.from_rows header [row_1, row_2, row_3] - | - | Icon: table-from-rows - | Aliases: foo, bar baz, redshift® - |""".stripMargin.linesIterator.mkString("\n") - val expected = Seq( - DocSection.Tag("UNSTABLE", ""), - DocSection.Paragraph( - "Creates a new table from a vector of column names and a vector of vectors specifying row contents. " - ), - DocSection.Keyed( - "Arguments", - "
  • header: A list of texts specifying the column names
  • rows: A vector of vectors, specifying the contents of each table row. The length of each element of rows must be equal in length to header.
" - ), - DocSection.Marked( - DocSection.Mark.Example(), - Some("Example"), - " Create a table with 3 columns, named foo, bar, and baz, containing [1, 2, 3], [True, False, True], and ['a', 'b', 'c'], respectively. 
import Standard.Table
example_from_rows =
    header = [ 'foo' , 'bar' , 'baz' ]
    row_1 =  [ 1     , True  , 'a'   ]
    row_2 =  [ 2     , False , 'b'   ]
    row_3 =  [ 3     , True  , 'c'   ]
    Table.from_rows header [row_1, row_2, row_3]
" - ), - DocSection.Keyed("Icon", "table-from-rows"), - DocSection.Keyed("Aliases", "foo, bar baz, redshift®") - ) - - build(comment) shouldEqual expected - } - } -} -object DocSectionsBuilderTest { - - val builder: DocSectionsBuilder = - DocSectionsBuilder() - - def build(comment: String): Seq[DocSection] = - builder.build(comment) -} diff --git a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/HtmlReprTest.scala b/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/HtmlReprTest.scala deleted file mode 100644 index 322725ef2a5e..000000000000 --- a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/HtmlReprTest.scala +++ /dev/null @@ -1,147 +0,0 @@ -package org.enso.docs.sections - -import org.enso.syntax.text.ast.Doc -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class HtmlReprTest extends AnyWordSpec with Matchers { - - import HtmlReprTest._ - - "HtmlRepr" should { - - "render Unclosed italic" in { - val elem = Doc.Elem.Formatter.Unclosed(Doc.Elem.Formatter.Italic, "hello") - - render(elem) shouldEqual "_hello" - } - - "render Unclosed bold" in { - val elem = Doc.Elem.Formatter.Unclosed(Doc.Elem.Formatter.Bold, "hello") - - render(elem) shouldEqual "*hello" - } - - "render code block" in { - val elem = Doc.Elem.CodeBlock( - Doc.Elem.CodeBlock.Line(0, "main ="), - Doc.Elem.CodeBlock.Line(4, """"Hello World!"""") - ) - - render(elem) shouldEqual - "
main =
    "Hello World!"
" - } - - "render inline code block" in { - val elem = Doc.Elem.CodeBlock.Inline("foo = bar baz") - - render(elem) shouldEqual "foo = bar baz" - } - - "render code block line" in { - val elem = Doc.Elem.CodeBlock.Line(2, "foo = bar baz") - - render(elem) shouldEqual " foo = bar baz
" - } - - "render Formatter italic" in { - val elem = Doc.Elem.Formatter( - Doc.Elem.Formatter.Italic, - "hello" - ) - - render(elem) shouldEqual "hello" - } - - "render Formatter strikeout" in { - val elem = Doc.Elem.Formatter( - Doc.Elem.Formatter.Strikeout, - "world" - ) - - render(elem) shouldEqual "world" - } - - "render header section" in { - val elem = Doc.Section.Header("Section") - - render(elem) shouldEqual "

Section

" - } - - "render URL link" in { - val elem = Doc.Elem.Link.URL("hello", "https://example.com/hello") - - render(elem) shouldEqual - """hello""" - } - - "render image link" in { - val elem = Doc.Elem.Link.Image("hello", "https://example.com/hello.png") - - render(elem) shouldEqual - """hello""" - } - - "render deprecated tag" in { - val elem = Doc.Tags.Tag(2, Doc.Tags.Tag.Type.Deprecated) - - render(elem) shouldEqual "DEPRECATED" - } - - "render alias tag" in { - val elem = Doc.Tags.Tag(2, Doc.Tags.Tag.Type.Alias, "Other function") - - render(elem) shouldEqual "ALIAS Other function" - } - - "render lists" in { - val elem = Doc.Elem.List( - 2, - Doc.Elem.List.Unordered, - Doc.Elem.ListItem("first"), - Doc.Elem.List( - 4, - Doc.Elem.List.Ordered, - Doc.Elem.ListItem("a"), - Doc.Elem.ListItem("b"), - Doc.Elem.MisalignedItem(3, Doc.Elem.List.Ordered, "c") - ), - Doc.Elem.ListItem("second") - ) - - render(elem) shouldEqual - "
  • first
    1. a
    2. b
    3. c
  • second
" - } - - "render lists formatted" in { - val elem = Doc.Elem.List( - 0, - Doc.Elem.List.Unordered, - Doc.Elem.ListItem( - Doc.Elem.Formatter(Doc.Elem.Formatter.Italic, "one"), - ": the first" - ), - Doc.Elem.ListItem( - "two", - Doc.Elem.Formatter.Unclosed( - Doc.Elem.Formatter.Italic, - "three", - ": the ", - Doc.Elem.Formatter(Doc.Elem.Formatter.Bold, "second") - ) - ) - ) - - render( - elem - ) shouldEqual "
  • one: the first
  • two_three: the second
" - } - } -} -object HtmlReprTest { - - def render(elem: Doc.Elem): String = { - val html = HtmlRepr[Doc.Elem].toHtml(elem) - HtmlRepr.renderHtml(html) - } -} diff --git a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/ParsedSectionsBuilderTest.scala b/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/ParsedSectionsBuilderTest.scala deleted file mode 100644 index d18062c3f5e3..000000000000 --- a/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/ParsedSectionsBuilderTest.scala +++ /dev/null @@ -1,518 +0,0 @@ -package org.enso.docs.sections - -import org.enso.syntax.text.DocParser -import org.enso.syntax.text.ast.Doc -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class ParsedSectionsBuilderTest extends AnyWordSpec with Matchers { - - import ParsedSectionsBuilderTest._ - - "ParsedSectionsBuilder" should { - - "generate single tag" in { - val comment = - """ UNSTABLE - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Tag("UNSTABLE", List()) - ) - - parseSections(comment) shouldEqual expected - } - - "generate multiple tags" in { - val comment = - """ UNSTABLE - | DEPRECATED - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Tag("UNSTABLE", List()), - Section.Tag("DEPRECATED", List()) - ) - - parseSections(comment) shouldEqual expected - } - - "generate tag with description" in { - val comment = - """ ALIAS Check Matches - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Tag("ALIAS", List(Doc.Elem.Text("Check Matches"))) - ) - - parseSections(comment) shouldEqual expected - } - - "generate description single line" in { - val comment = - """ hello world - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph(List(Doc.Elem.Text("hello world"))) - ) - - parseSections(comment) shouldEqual expected - } - - "generate description multiline" in { - val comment = - """ hello world - | second line - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List( - Doc.Elem.Text("hello world"), - Doc.Elem.Newline, - Doc.Elem.Text("second line") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate description multiple paragraphs" in { - val comment = - """ Hello world - | second line - | - | Second paragraph - | multiline - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List( - Doc.Elem.Text("Hello world"), - Doc.Elem.Newline, - Doc.Elem.Text("second line"), - Doc.Elem.Newline - ) - ), - Section.Paragraph( - List( - Doc.Elem.Text("Second paragraph"), - Doc.Elem.Newline, - Doc.Elem.Text("multiline") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate keyed arguments" in { - val comment = - """ Description - | - | Arguments: - | - one: The first - | - two: The second - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Keyed( - "Arguments", - List( - Doc.Elem.Newline, - Doc.Elem.List( - 1, - Doc.Elem.List.Unordered, - Doc.Elem.Text("one: The first"), - Doc.Elem.Text("two: The second") - ) - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate keyed icon" in { - val comment = - """ Description - | - | Icon: my-icon - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Keyed( - "Icon", - List(Doc.Elem.Text("my-icon")) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate keyed aliases" in { - val comment = - """ Description - | - | Aliases: foo, bar baz, redshift® - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Keyed( - "Aliases", - List(Doc.Elem.Text("foo, bar baz, redshift"), Doc.Elem.Text("®")) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate multiple keyed" in { - val comment = - """ Description - | - | Icon: my-icon - | icon - | Aliases: foo, bar baz, redshift® - | and other - | - | Paragraph - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Keyed( - "Icon", - List( - Doc.Elem.Text("my-icon"), - Doc.Elem.Newline, - Doc.Elem.Text("icon") - ) - ), - Section.Keyed( - "Aliases", - List( - Doc.Elem.Text("foo, bar baz, redshift"), - Doc.Elem.Text("®"), - Doc.Elem.Newline, - Doc.Elem.Text("and other"), - Doc.Elem.Newline - ) - ), - Section.Paragraph(List(Doc.Elem.Text("Paragraph"))) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked example" in { - val comment = - """ Description - | - | > Example - | Simple program - | main = 42 - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Example, - Some("Example"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Simple program"), - Doc.Elem.Newline, - Doc.Elem.CodeBlock(Doc.Elem.CodeBlock.Line(7, "main = 42")) - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked multiple examples" in { - val comment = - """ Description - | - | > Example - | Simple program - | Multiline - | main = 42 - | - | > Example - | Another example - | - | import Foo.Bar - | - | main = - | 42 - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Example, - Some("Example"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Simple program"), - Doc.Elem.Newline, - Doc.Elem.Text("Multiline"), - Doc.Elem.Newline, - Doc.Elem.CodeBlock(Doc.Elem.CodeBlock.Line(7, "main = 42")), - Doc.Elem.Newline - ) - ), - Section.Marked( - Section.Mark.Example, - Some("Example"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Another example"), - Doc.Elem.Newline, - Doc.Elem.CodeBlock( - Doc.Elem.CodeBlock.Line(7, "import Foo.Bar"), - Doc.Elem.CodeBlock.Line(7, "main ="), - Doc.Elem.CodeBlock.Line(11, "42") - ) - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked important" in { - val comment = - """ Description - | - | ! This is important - | Beware of nulls. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Important, - Some("This is important"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Beware of nulls.") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked info" in { - val comment = - """ Description - | - | ? Out of curiosity - | FYI. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Info, - Some("Out of curiosity"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("FYI.") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked info multiple sections" in { - val comment = - """ Description - | - | ? Out of curiosity - | FYI. - | - | Another section. - | - | And another. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Info, - Some("Out of curiosity"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("FYI."), - Doc.Elem.Newline, - Doc.Elem.Newline, - Doc.Elem.Text("Another section."), - Doc.Elem.Newline, - Doc.Elem.Newline, - Doc.Elem.Text("And another.") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked info and paragraph sections" in { - val comment = - """ Description - | - | ? Out of curiosity - | FYI. - | - | Another section. - | - | This is paragraph. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Info, - Some("Out of curiosity"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("FYI."), - Doc.Elem.Newline, - Doc.Elem.Newline, - Doc.Elem.Text("Another section."), - Doc.Elem.Newline - ) - ), - Section.Paragraph( - List(Doc.Elem.Text("This is paragraph.")) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate marked info and important sections" in { - val comment = - """ Description - | - | ? Out of curiosity - | FYI. - | - | ! Warning - | Pretty important. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Paragraph( - List(Doc.Elem.Text("Description"), Doc.Elem.Newline) - ), - Section.Marked( - Section.Mark.Info, - Some("Out of curiosity"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("FYI."), - Doc.Elem.Newline - ) - ), - Section.Marked( - Section.Mark.Important, - Some("Warning"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Pretty important.") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - - "generate multiple sections" in { - val comment = - """ DEPRECATED - | - | Some paragraph - | Second line - | - | Arguments: - | - one: The first - | - two: The second - | - | ! This is important - | Boo. - | - | ? Out of curiosity - | FYI. - |""".stripMargin.linesIterator.mkString("\n") - val expected = List( - Section.Tag("DEPRECATED", List()), - Section.Paragraph( - List( - Doc.Elem.Text("Some paragraph"), - Doc.Elem.Newline, - Doc.Elem.Text("Second line"), - Doc.Elem.Newline - ) - ), - Section.Keyed( - "Arguments", - List( - Doc.Elem.Newline, - Doc.Elem.List( - 1, - Doc.Elem.List.Unordered, - Doc.Elem.Text("one: The first"), - Doc.Elem.Text("two: The second") - ), - Doc.Elem.Newline - ) - ), - Section.Marked( - Section.Mark.Important, - Some("This is important"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("Boo."), - Doc.Elem.Newline - ) - ), - Section.Marked( - Section.Mark.Info, - Some("Out of curiosity"), - List( - Doc.Elem.Newline, - Doc.Elem.Text("FYI.") - ) - ) - ) - - parseSections(comment) shouldEqual expected - } - } - -} -object ParsedSectionsBuilderTest { - - val parsedSectionsBuilder = new ParsedSectionsBuilder - - def parseSections(comment: String): List[Section] = { - val doc = DocParser.runMatched(comment) - parsedSectionsBuilder.build(doc) - } -} diff --git a/lib/scala/flexer/README.md b/lib/scala/flexer/README.md deleted file mode 100644 index edf289f7be09..000000000000 --- a/lib/scala/flexer/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Flexer - -The Flexer is an incredibly fast and flexible lexing engine for Enso source -code. diff --git a/lib/scala/flexer/src/main/java/org/enso/flexer/package-info.java b/lib/scala/flexer/src/main/java/org/enso/flexer/package-info.java deleted file mode 100644 index 7e05a3d6ff30..000000000000 --- a/lib/scala/flexer/src/main/java/org/enso/flexer/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.enso.flexer; diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/ADT.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/ADT.scala deleted file mode 100644 index 1c4f670f3370..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/ADT.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.enso.flexer - -import scala.reflect.macros.blackbox.Context - -object ADT { - def constructors[T]: Set[T] = macro constructorsImpl[T] - - def constructorsImpl[T: c.WeakTypeTag](c: Context): c.Expr[Set[T]] = { - import c.universe._ - - val subs = - c.weakTypeTag[T].tpe.typeSymbol.asClass.knownDirectSubclasses.map { - symbol => q"${c.mirror.staticModule(symbol.fullName)}" - } - c.Expr[Set[T]](q"Set(..$subs)") - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/Parser.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/Parser.scala deleted file mode 100644 index 1067c10613e6..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/Parser.scala +++ /dev/null @@ -1,167 +0,0 @@ -package org.enso.flexer - -import org.enso.Logger -import debug.Escape -import spec.Macro -import ReaderUTF.ENDOFINPUT - -import scala.collection.mutable - -trait Parser[T] { - import Parser._ - - var reader: Reader = _ - - var status = State.Status.Exit.OK - val stateDefs = new Array[Int => Int](256) - val logger = new Logger() - var currentMatch = "" - - def getResult(): Option[T] - - def run(input: Reader): Result[T] = { - reader = input - reader.rewind.matched.set() - reader.nextChar() - - while (state.runCurrent() == State.Status.Exit.OK) () - - val value: Result.Value[T] = getResult() match { - case None => Result.Failure(None) - case Some(result) => - status match { - case State.Status.Exit.FINISHED => Result.Success(result) - case State.Status.Exit.FAIL => Result.Failure(Some(result)) - case _ => Result.Partial(result) - } - } - Result(reader.offset, value) - } - - // Rewind is a mechanism that allows for arbitrary lookahead. - final def rewind(): Unit = - reader.rewind.matched.run() - - //// State management //// - - // FIXME: This is a hack. Without it sbt crashes and needs to be completely - // cleaned to compile again. - val state = _state - final object _state { - - var registry = new mutable.ArrayBuffer[State]() - - def define(label: String = "unnamed", finish: => Unit = {}): State = { - val groupIndex = registry.length - val newState = new State(label, groupIndex, () => finish) - registry.append(newState) - newState - } - - var stack: List[State] = Nil - var current: State = define("Root") - - def begin(state: State): Unit = { - logger.log(s"Begin ${state.label}") - stack +:= current - current = state - } - - def end(): Unit = stack match { - case Nil => logger.err("Trying to end root state") - case head :: tail => - logger.log(s"End ${current.label}, back to ${head.label}") - current = head - stack = tail - } - - def endTill(s: State): Unit = logger.trace { - while (s != state.current) { - state.current.finish() - state.end() - } - } - - def isInside(state: State): Boolean = - current == state || stack.contains(state) - - def runCurrent(): Int = { - val cstate = state.current - var finished = false - val nextState = stateDefs(cstate.ix) - status = State.Status.INITIAL - while (State.valid(status)) { - logger.log( - s"Step (${cstate.ix}:$status) " - + s"${Escape.str(reader.currentStr)} (${reader.charCode})" - ) - status = nextState(status) - if (finished && !reader.rewind.rewinded) - status = State.Status.Exit.FINISHED - finished = reader.charCode == ENDOFINPUT - if (State.valid(status)) { - if (reader.charCode != ENDOFINPUT) - reader.result.appendCodePoint(reader.charCode) - reader.nextChar() - } - } - status - } - - def call(rule: () => Unit): State.Status.Exit = { - currentMatch = reader.result.toString - rule() - reader.result.setLength(0) - reader.rewind.matched.set() - State.Status.Exit.OK - } - - } - - val ROOT = state.current -} - -object Parser { - - val BUFFER_SIZE = 16384 - val UTF_CHAR_SIZE = 2 - - val eofCodePoint: Int = -1 - val etxCodePoint: Int = -2 - - object State { - object Status { - val INITIAL = 0 - type Exit = Int - object Exit { - val OK = -1 - val FAIL = -2 - val FINISHED = -3 - } - } - def valid(i: Int): Boolean = - i >= 0 - } - - def compile[T, P](p: P)(implicit ev: P <:< Parser[T]): () => P = - macro Macro.compileImpl[T, P] - - case class Result[T](offset: Int, value: Result.Value[T]) { - def map[S](fn: T => S): Result[S] = copy(value = value.map(fn)) - } - object Result { - sealed trait Value[T] { - def map[S](fn: T => S): Value[S] - } - final case class Success[T](result: T) extends Value[T] { - def map[S](fn: T => S) = copy(fn(result)) - } - final case class Partial[T](result: T) extends Value[T] { - def map[S](fn: T => S) = copy(fn(result)) - } - final case class Failure[T](result: Option[T]) extends Value[T] { - def map[S](fn: T => S) = copy(result.map(fn)) - } - } - -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/Reader.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/Reader.scala deleted file mode 100644 index 208730ec6768..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/Reader.scala +++ /dev/null @@ -1,102 +0,0 @@ -package org.enso.flexer - -import java.io._ -import java.nio.charset.StandardCharsets.UTF_8 - -import org.enso.Logger - -class Reader(input: InputStream) extends ReaderUTF(input) { - - import org.enso.flexer.Reader._ - - lazy val logger = new Logger() - lazy val result = new java.lang.StringBuilder() - - def this(file: File) = this(new FileInputStream(file)) - def this(str: String) = this(new ByteArrayInputStream(str.getBytes(UTF_8))) - - final override def empty: Boolean = - copyByte == 0 && super.empty - - final override def fill(off: Int): Unit = { - if (rewind.maxRewindOffset == 0) - throw new OutOfMemoryError("Rewind is impossible. Buffer is too small.") - val keepchars = length - rewind.maxRewindOffset - rewind.decreaseOffset(length - keepchars) - for (i <- 1 to keepchars) - buffer(keepchars - i) = buffer(length - i) - super.fill(keepchars) - } - - final override def nextChar(): Int = { - rewind.rewinded = false - super.nextChar() - } - - final override def nextByte(): Int = nextByteHelper() match { - case '\r' => '\n' - case '\t' => ' ' - case byte => byte - } - - private var lastByte = 0 - private var copyByte = 0 - - final private def nextByteHelper(): Int = { - if (copyByte > 0) { - copyByte -= 1 - return lastByte - } - lastByte = (lastByte, super.nextByte()) match { - case ('\r', '\n') => nextByteHelper() - case (_, '\t') => copyByte = TABSIZE - 1; '\t' - case (_, byte) => byte - } - lastByte - } - - final def charOffset: Int = offset - charSize - - // because of bug in macroContext.eval it cannot be part of object rewind - class Rewinder(index: Int) { - import rewind._ - - final def set(): Unit = logger.trace { - rewindBy(index)(0) = charOffset - rewindBy(index)(1) = result.length - } - - final def run(): Unit = logger.trace { - result.setLength(rewindBy(index)(1)) - offset = rewindBy(index)(0) - nextChar() - rewinded = true - } - } - - final object rewind { - var rewinded = false - - lazy val rewindBy = Array(Array(0, -1), Array(0, -1)) - // Describes the currently matched sequence of characters that corresponds - // to the currently matched rule. - lazy val matched = new Rewinder(0) - // Used for the implementation of overlapping rules and also for manual - // rewinding by the user. - lazy val rule = new Rewinder(1) - - def maxRewindOffset = - if (rewindBy(0)(1) == -1) length else rewindBy(0)(0) - - def decreaseOffset(off: Int): Unit = - for (i <- rewind.rewindBy.indices) - rewind.rewindBy(i)(0) -= off - } - -} - -object Reader { - - val TABSIZE = 4 - -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/ReaderUTF.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/ReaderUTF.scala deleted file mode 100644 index 4ab81f576ef3..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/ReaderUTF.scala +++ /dev/null @@ -1,88 +0,0 @@ -package org.enso.flexer - -import java.io._ -import java.nio.charset.StandardCharsets.UTF_8 - -/** Fast UTF8 reader and preprocessor. - * It uses unboxed byte buffer under the hood, - * deals correctly with variable length UTF chars - * and replaces \r(\n) with \n and \t with 4 spaces. - */ -class ReaderUTF(val input: InputStream) { - import ReaderUTF._ - - // buffer will be unboxed as long as we don't use any fancy scala collection methods on it - val buffer = new Array[Byte](BUFFERSIZE) - var offset = 0 - var length = BUFFERSIZE - var charSize = 0 - var charCode = ENDOFINPUT - - def this(file: File) = this(new FileInputStream(file)) - def this(str: String) = this(new ByteArrayInputStream(str.getBytes(UTF_8))) - - fill(0) - - protected def fill(off: Int): Unit = { - length = off + input.read(buffer, off, BUFFERSIZE - off) - offset = off - } - - protected def nextByte(): Int = { - if (offset >= length) - if (!empty) fill(0) - else return ENDOFINPUT - val byte = buffer(offset) - offset += 1 - byte.toInt - } - - def empty: Boolean = - offset >= length && length < BUFFERSIZE - - def nextChar(): Int = { - charCode = nextByte() - charSize = charLength(charCode.toByte.toInt) - charCode = charCode & charMask(charSize) - for (_ <- 1 until charSize) - charCode = charCode << UTFBYTESIZE | nextByte() & charMask(-1) - charCode - } - - override def toString(): String = { - val builder = new java.lang.StringBuilder() - while (nextChar() != ENDOFINPUT) builder.appendCodePoint(charCode) - builder.toString - } - - final def currentStr: String = - if (charCode < 0) "" else new String(Character.toChars(charCode)) - -} - -object ReaderUTF { - - val ENDOFINPUT = -1 - val BUFFERSIZE = 32768 - val UTFBYTESIZE = 6 - - /** For more info on UTF decoding look at: https://en.wikipedia.org/wiki/UTF-8 */ - def charLength(char: Int): Int = - if (char == ENDOFINPUT) 0 - else - ~char >> 4 match { - case 0 => 4 - case 1 => 3 - case 2 | 3 => 2 - case _ => 1 - } - - def charMask(size: Int): Int = size match { - case 0 => -1 // do not mask end of input - case 1 => 127 // 0111 1111 - case 2 => 63 // 0011 1111 - case 3 => 31 // 0001 1111 - case 4 => 15 // 0000 1111 - case _ => 63 // 0011 1111 - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/Spec.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/Spec.scala deleted file mode 100644 index 9fd45a24cb73..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/Spec.scala +++ /dev/null @@ -1,143 +0,0 @@ -package org.enso.flexer - -import org.enso.flexer.automata.DFA -import org.enso.flexer.automata.State - -import scala.collection.immutable.Range -import scala.collection.mutable -import scala.reflect.macros.blackbox.Context - -/** Creates update functions for given DFA ~ nextState : state -> state. - * Each state has a pattern match on current utf code point. - * ASCII characters are explicitly matched, so that we get O(1) lookup. - * The rest of UTF characters is dispatched by tree of if-else, - * with O(log(N)) lookup. - */ -case class Spec[C <: Context](c: C, dfa: DFA) { - import c.universe._ - import Spec._ - - val stateHasOverlappingRules = mutable.Map(0 -> false) - - case class Branch(range: Range, body: Tree) - - def genBranchBody( - targetState: Int, - maybeState: Option[State.Desc], - rulesOverlap: Boolean - ): Tree = (targetState, maybeState, rulesOverlap) match { - case (State.missing, None, _) => - Literal(Constant(Parser.State.Status.Exit.FAIL)) - case (State.missing, Some(state), false) => - q"state.call(${TermName(state.rule)})" - case (State.missing, Some(state), true) => - q"reader.rewind.rule.run(); state.call(${TermName(state.rule)})" - - case _ => - val targetStateHasNoRule = maybeState match { - case Some(state) if !dfa.endStatePriorityMap.contains(targetState) => - dfa.endStatePriorityMap += targetState -> state - stateHasOverlappingRules += targetState -> true - true - case _ => false - } - val trgState = Literal(Constant(targetState)) - if (targetStateHasNoRule && !rulesOverlap) - q"reader.rewind.rule.set(); $trgState" - else - q"$trgState" - } - - def genSwitch(branchs: Seq[Branch]): Seq[CaseDef] = { - branchs.map { case Branch(range, body) => - val pattern = - Alternative(range.map(i => q"${Literal(Constant(i))}").toList) - cq"$pattern => $body" - } - } - - def genIf(branchs: Seq[Branch]): Branch = { - branchs match { - case b +: Seq() => b - case a +: b +: rest => - val range = a.range.start to b.range.end - val body = q"if (charCode <= ${a.range.end}) ${a.body} else ${b.body}" - genIf(Branch(range, body) +: rest) - } - } - - def generateCaseBody(stateIx: Int): Tree = { - val overlaps = stateHasOverlappingRules.getOrElse(stateIx, false) - val state = dfa.endStatePriorityMap.get(stateIx) - var trgState = dfa.links(stateIx)(0) - var rStart = Int.MinValue - val branches = dfa.vocabulary.toVector.flatMap { case (range, vocIx) => - val newTrgState = dfa.links(stateIx)(vocIx) - if (newTrgState != trgState) { - val rEnd = range.start - 1 - val xtrgState = trgState - val xrStart = rStart - trgState = newTrgState - rStart = range.start - Some( - Branch(xrStart to rEnd, genBranchBody(xtrgState, state, overlaps)) - ) - } else None - } - - val allBranches = branches :+ - Branch(rStart to Int.MaxValue, genBranchBody(trgState, state, overlaps)) - - val (utf1 :+ b1, rest) = allBranches.span(_.range.start < MIN_MATCH_CODE) - val (asci, utf2) = rest.span(_.range.end <= MAX_MATCH_CODE) - - utf2 match { - case b2 +: utf2 => - val b1UTF = Branch(b1.range.start until MIN_MATCH_CODE, b1.body) - val b1ASC = Branch(MIN_MATCH_CODE to b1.range.end, b1.body) - val b2ASC = Branch(b2.range.start to MAX_MATCH_CODE, b2.body) - val b2UTF = Branch(MAX_MATCH_CODE + 1 to b2.range.end, b2.body) - - val emptyB1ASC = b1ASC.range.end < MIN_MATCH_CODE - val emptyB2UTF = b2UTF.range.start <= MAX_MATCH_CODE - - val ascii = if (emptyB1ASC) asci :+ b2ASC else b1ASC +: asci :+ b2ASC - val utfMiddle = if (emptyB2UTF) Vector(b1UTF) else Vector(b1UTF, b2UTF) - val utf = utf1 ++ utfMiddle ++ utf2 - val body = genSwitch(ascii) :+ cq"charCode => ${genIf(utf).body}" - - q"${Match(q"reader.charCode", body.toList)}" - case _ => - genIf(utf1 :+ b1).body - } - } - - def generate(i: Int): Tree = { - val stateNames = - dfa.links.indices.toList - .map(st => (st, TermName(s"state${i}_${st}"))) - - val stateMatch = Match( - q"state", - stateNames.map { case (st, fun) => - cq"$st => $fun" - } - ) - val stateBodies = stateNames.map { case (st, fun) => - q"def $fun = {${generateCaseBody(st)}}" - } - q""" - stateDefs($i) = ${TermName(s"nextState$i")} - def ${TermName(s"nextState$i")}(state: Int): Int = $stateMatch - ..$stateBodies - """ - } - -} - -object Spec { - - /** Covers all ASCII characters (0 - 255) and End Of Input (-1) */ - val MIN_MATCH_CODE = -1 - val MAX_MATCH_CODE = 255 -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/State.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/State.scala deleted file mode 100644 index 78b53be7bc32..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/State.scala +++ /dev/null @@ -1,97 +0,0 @@ -package org.enso.flexer - -import org.enso.flexer.automata.NFA -import org.enso.flexer.automata.Pattern -import org.enso.flexer.state.Rule - -import scala.reflect.macros.blackbox.Context - -class State(val label: String, val ix: Int, val finish: () => Unit) { - var parent: Option[State] = None - private var revRules: List[Rule] = List() - - def parent_=(p: State): Unit = - parent = Some(p) - - def addRule(rule: Rule): Unit = - revRules = rule +: revRules - - def rule(expr: Pattern): Rule.Builder = - Rule.Builder(expr, addRule) - - def ||(expr: Pattern): Rule.Builder = - rule(expr) - - def rules: List[Rule] = { - val myRules = revRules.reverse - parent.map(myRules ++ _.rules).getOrElse(myRules) - } - - private def ruleName(ruleIx: Int): String = - s"group${ix}_rule$ruleIx" - - private def buildAutomata(): NFA = { - val nfa = new NFA - val start = nfa.addState() - val endpoints = rules.zipWithIndex.map { case (rule, ix) => - buildRuleAutomata(nfa, start, ix, rule) - } - val end = nfa.addState() - nfa.state(end).rule = Some("") - for (endpoint <- endpoints) { - nfa.link(endpoint, end) - } - nfa - } - - def buildRuleAutomata(nfa: NFA, last: Int, ruleIx: Int, rule: Rule): Int = { - val end = buildExprAutomata(nfa, last, rule.pattern) - nfa.state(end).rule = Some(ruleName(ruleIx)) - end - } - - def buildExprAutomata(nfa: NFA, last: Int, expr: Pattern): Int = { - import Pattern._ - val current = nfa.addState() - nfa.link(last, current) - expr match { - case Always => current - case Range(start, end) => - val state = nfa.addState() - nfa.link(current, state, scala.Range(start, end)) - state - case Seq(first, second) => - val s1 = buildExprAutomata(nfa, current, first) - buildExprAutomata(nfa, s1, second) - case Many(body) => - val s1 = nfa.addState() - val s2 = buildExprAutomata(nfa, s1, body) - val s3 = nfa.addState() - nfa.link(current, s1) - nfa.link(current, s3) - nfa.link(s2, s3) - nfa.link(s3, s1) - s3 - case Or(first, second) => - val s1 = buildExprAutomata(nfa, current, first) - val s2 = buildExprAutomata(nfa, current, second) - val s3 = nfa.addState() - nfa.link(s1, s3) - nfa.link(s2, s3) - s3 - } - } - - def generate[C <: Context](c: C): c.Tree = { - import c.universe._ - - val nfa = buildAutomata() - val dfa = nfa.toDFA() - val state = Spec[c.type](c, dfa).generate(ix) - val rs = rules.zipWithIndex.map { case (rule, ruleIx) => - val tree = c.parse(rule.tree) - q"def ${TermName(ruleName(ruleIx))}() = $tree" - } - q"..$state; ..$rs" - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/DFA.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/DFA.scala deleted file mode 100644 index 5c5e0e33c7b1..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/DFA.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.enso.flexer.automata - -import scala.collection.mutable - -case class DFA( - vocabulary: Dict, - links: Array[Array[Int]], - endStatePriorityMap: mutable.Map[Int, State.Desc] -) diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Dict.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Dict.scala deleted file mode 100644 index f201b2fbaf56..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Dict.scala +++ /dev/null @@ -1,23 +0,0 @@ -package org.enso.flexer.automata - -import scala.collection.immutable - -final class Dict extends Iterable[(Range, Int)] { - private var divisions = immutable.SortedSet[Int](0, Int.MaxValue) - - def insert(range: Range): Unit = { - divisions = divisions + range.start - divisions = divisions + (range.end + 1) - } - - override def size: Int = - divisions.size - 1 - - override def iterator: Iterator[(Range, Int)] = - divisions.iterator.zip(divisions.iterator.drop(1)).zipWithIndex.map { - case ((start, end), ix) => (start until end, ix) - } - - override def toString: String = - "Dict(" + divisions.toList.map(_.toString).mkString(",") + ")" -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/NFA.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/NFA.scala deleted file mode 100644 index 5b28e2cf0fd3..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/NFA.scala +++ /dev/null @@ -1,188 +0,0 @@ -package org.enso.flexer.automata - -import org.enso.Logger - -import scala.collection.mutable - -final class NFA { - val logger: Logger = new Logger() - val states: mutable.ArrayBuffer[State] = new mutable.ArrayBuffer() - val vocabulary = new Dict() - - import State.Implicits._ - - //// API //// - - def addState(): Int = { - val state = new State() - states += state - states.length - 1 - } - - def state(ix: Int): State = - states(ix) - - def link(start: Int, end: Int, charRange: Range): Unit = { - vocabulary.insert(charRange) - state(start).links.add(end, charRange) - } - - def link(start: Int, end: Int): Unit = - state(start).links.add(end) - - //// NFA -> DFA //// - - final private class EpsMatrix { - var links: Set[Int] = Set() - var computed: Boolean = false - } - - private def fillEpsMatrix(i: Int, stateToMat: Array[EpsMatrix]): Unit = { - val epsGroupIxMap: mutable.Map[Set[Int], Int] = mutable.Map() - def go(i: Int): Unit = { - var epsLinks = Set[Int](i) - if (stateToMat(i) == null) { - var circular = false - val epsMatrix = new EpsMatrix() - stateToMat(i) = epsMatrix - state(i).links.epsilon.foreach { tgt => - go(tgt) - val tgtEpsMatrix = stateToMat(tgt) - epsLinks = epsLinks + tgt ++ tgtEpsMatrix.links - if (!tgtEpsMatrix.computed) { - circular = true - } - } - epsMatrix.links = epsLinks - if (!circular) { - if (epsGroupIxMap.get(epsLinks).isEmpty) - epsGroupIxMap += (epsLinks -> epsGroupIxMap.size) - epsMatrix.computed = true - } - } - } - go(i) - } - - private def epsMatrix(): IndexedSeq[Set[Int]] = { - val arr = new Array[EpsMatrix](states.size) - states.indices.foreach(fillEpsMatrix(_, arr)) - arr.toIndexedSeq.map(_.links) - } - - private def nfaMatrix(): Array[Array[Int]] = { - logger.group("Computing NFA Matrix") { - val matrix = Array.ofDim[Int](states.length, vocabulary.size) - for (stateIx <- states.indices) { - val s = state(stateIx) - for ((range, vocIx) <- vocabulary) { - s.links.ranged.getOption(range.start) match { - case Some(tgt) => matrix(stateIx)(vocIx) = tgt - case None => matrix(stateIx)(vocIx) = State.missing - } - } - } - matrix - } - } - - def toDFA(): DFA = { - logger.group("Computing DFA Matrix") { - val epsMat = epsMatrix() - val nfaMat = nfaMatrix() - var dfaRows = 0 - var dfaMat = Array[Array[Int]]() - val dfaEpsMap = mutable.Map[Set[Int], Int]() - val dfaEpsIxs = mutable.ArrayBuffer[Set[Int]]() - - def addDFAKey(epsSet: Set[Int]): Int = { - val id = dfaEpsMap.size - dfaEpsMap += (epsSet -> id) - dfaEpsIxs += epsSet - dfaRows += 1 - dfaMat :+= Array.fill(vocabulary.size)(State.missing) - logger.log(s"DFA[$id] = $epsSet") - id - } - - logger.group(s"Preparing start points") { - val initEpsSet = epsMat(0) - addDFAKey(initEpsSet) - } - - var i = 0 - while (i < dfaRows) { - val epsIxs = dfaEpsIxs(i) - logger.group(s"Computing DFA[$i]") { - for ((voc, vocIx) <- vocabulary) { - logger.group(s"Vocabulary '$voc'") { - var epsSet = Set[Int]() - for (epsIx <- epsIxs) { - val tgt = nfaMat(epsIx)(vocIx) - if (tgt != State.missing) - epsSet = epsSet ++ epsMat(tgt) - } - if (epsSet.nonEmpty) { - dfaMat(i)(vocIx) = dfaEpsMap.get(epsSet) match { - case None => addDFAKey(epsSet) - case Some(id) => - logger.log(s"Existing DFA ID $id") - id - } - } - } - } - } - i += 1 - } - - val nfaEndStatePriorityMap = mutable.Map[Int, Int]() - for (i <- nfaMat.indices) { - if (state(i).rule.isDefined) - nfaEndStatePriorityMap += (i -> (nfaMat.length - i)) - } - - val dfaEndStatePriorityMap = mutable.Map[Int, State.Desc]() - for ((epss, dfaIx) <- dfaEpsIxs.zipWithIndex) { - val eps = epss.maxBy(nfaEndStatePriorityMap.getOrElse(_, State.missing)) - nfaEndStatePriorityMap.get(eps).foreach { priority => - val rule = state(eps).rule.getOrElse("") - dfaEndStatePriorityMap += dfaIx -> State.Desc(priority, rule) - } - } - DFA(vocabulary, dfaMat, dfaEndStatePriorityMap) - } - } - - def visualize(): String = { - import java.awt.Desktop - import java.net.URI - import java.net.URLEncoder - - val gray = "#AAAAAA" - val lines = mutable.ArrayBuffer[String]() - lines += "digraph G {" - lines += "node [shape=circle width=0.8]" - for ((state, source) <- states.zipWithIndex) { - if (state.links.ranged.isEmpty) { - lines += s"""$source [color="$gray" fontcolor="$gray"]""" - } else { - lines += s"""$source""" - } - state.links.ranged.asMapOfRanges().forEach { (range, target) => - lines += s"""$source -> $target [label="$range"]""" - } - for (target <- state.links.epsilon) { - lines += s"""$source -> $target [style="dashed" color="$gray"]""" - } - } - lines += "}" - val code = lines.mkString("\n") - var webCode = code - webCode = URLEncoder.encode(webCode, "UTF-8") - webCode = webCode.replaceAll("[+]", "%20") - val address = "https://dreampuf.github.io/GraphvizOnline/#" + webCode - Desktop.getDesktop().browse(new URI(address)) - code - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Pattern.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Pattern.scala deleted file mode 100644 index 41fa165c2f7a..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/Pattern.scala +++ /dev/null @@ -1,82 +0,0 @@ -package org.enso.flexer.automata - -import org.enso.flexer.Parser -import scala.annotation.tailrec - -trait Pattern { - import Pattern._ - - def |(that: Pattern): Pattern = Or(this, that) - def >>(that: Pattern): Pattern = Seq(this, that) - def many: Pattern = Many(this) - def many1: Pattern = this >> many - def opt: Pattern = this | always -} - -object Pattern { - case object Always extends Pattern - case class Range(start: Int, end: Int) extends Pattern - case class Or(left: Pattern, right: Pattern) extends Pattern - case class Seq(first: Pattern, second: Pattern) extends Pattern - case class Many(body: Pattern) extends Pattern - - //// API //// - - val always: Pattern = Always - def range(start: Char, end: Char): Range = Range(start.toInt, end.toInt) - def range(start: Int, end: Int): Range = Range(start, end) - def range(end: Int): Range = range(0, end) - def range(end: Char): Range = range(0, end.toInt) - def char(char: Char): Range = range(char.toInt, char.toInt) - def char(char: Int): Range = range(char, char) - - val never: Pattern = range(-1) - val any: Range = range(Int.MaxValue) - val eof: Range = char(Parser.eofCodePoint) - - def anyOf(chars: String): Pattern = anyOf(chars.map(char)) - def anyOf(alts: scala.Seq[Pattern]): Pattern = alts.fold(never)(_ | _) - def noneOf(chars: String): Pattern = { - val pointCodes = chars.map(_.toInt).sorted - val startPoints = 0 +: pointCodes.map(_ + 1) - val endPoints = pointCodes.map(_ - 1) :+ Int.MaxValue - val ranges = startPoints.zip(endPoints) - val validRanges = ranges.filter { case (s, e) => e >= s } - val patterns = validRanges.map { case (s, e) => range(s, e) } - anyOf(patterns) - } - - final def not(char: Char): Pattern = - noneOf(char.toString) - - def repeat(pat: Pattern, min: Int, max: Int): Pattern = { - @tailrec - def go(i: Int, ch: Pattern, out: Pattern): Pattern = - i match { - case 0 => out - case _ => - val ch2 = ch >> pat - go(i - 1, ch2, out | ch2) - } - val minPat = repeat(pat, min) - go(max - min, minPat, minPat) - } - - def repeat(pat: Pattern, num: Int): Pattern = - 0.until(num).foldLeft(always)((t, _) => t >> pat) - - //// Implicits //// - - implicit class ExtendedChar(_this: Char) { - final def ||(that: Char): Pattern = - Or(char(_this), char(that)) - } - - implicit def automataPtternFromChar(char: Char): Pattern = - range(char, char) - implicit def automataPatternFromString(str: String): Pattern = - str.toList match { - case Nil => always - case s :: ss => ss.foldLeft(char(s): Pattern)(_ >> _) - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/State.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/State.scala deleted file mode 100644 index 442d917371b3..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/automata/State.scala +++ /dev/null @@ -1,47 +0,0 @@ -package org.enso.flexer.automata - -import com.google.common.{collect => guava} - -import scala.collection.mutable - -class State { - val links: State.Link.Registry = new State.Link.Registry() - var rule: Option[String] = None -} - -object State { - - object Implicits { - - implicit final class RangeMapWrapper[K <: Comparable[_], V]( - underlying: guava.RangeMap[K, V] - ) { - - def getOption(key: K): Option[V] = - Option(underlying.get(key)) - - def isEmpty: Boolean = - underlying.asMapOfRanges().isEmpty() - } - } - - val missing = -1 - - case class Desc(priority: Int, rule: String) - - object Link { - - class Registry { - val epsilon: mutable.ArrayBuffer[Int] = new mutable.ArrayBuffer() - val ranged: guava.RangeMap[java.lang.Integer, Int] = - guava.TreeRangeMap.create[java.lang.Integer, Int]() - - def add(target: Int): Unit = - epsilon += target - - def add(target: Int, range: Range) = - if (range.start <= range.end) - ranged.put(guava.Range.closed(range.start, range.end), target) - } - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/debug/Escape.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/debug/Escape.scala deleted file mode 100644 index 4c6d720c3a62..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/debug/Escape.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.enso.flexer.debug - -object Escape { - def char(ch: Char): String = ch match { - case '\b' => "\\b" - case '\t' => "\\t" - case '\n' => "\\n" - case '\f' => "\\f" - case '\r' => "\\r" - case '"' => "\\\"" - case '\'' => "\\\'" - case '\\' => "\\\\" - case _ => - if (ch.isControl) "\\0" + Integer.toOctalString(ch.toInt) - else String.valueOf(ch) - } - - def char(str: String): String = str.flatMap(char(_)) - - def str(str: String): String = s"'${char(str)}'" -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/spec/Macro.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/spec/Macro.scala deleted file mode 100644 index 71725e0b3dc5..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/spec/Macro.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.enso.flexer.spec - -import org.enso.flexer.Parser - -import scala.annotation.{nowarn, unused} -import scala.reflect.macros.blackbox.Context - -// FIXME: Needs to be refactored. Contains deprecated API usage -object Macro { - - def print(c: Context, msg: String) = c.echo(c.enclosingPosition, msg) - - def runRule(c: Context)(program: c.Tree): c.Tree = { - import c.universe._ - val tree = new Transformer { - override def transform(tree: Tree): Tree = tree match { - case Select(This(TypeName(_)), name) => - super.transform(Ident(name)) - case node => super.transform(node) - } - }.transform(program) - - c.macroApplication match { - case Apply(Select(lhs, _), _) => q"$lhs.run(${showCode(tree)})" - case _ => throw new Error("Unsupported shape") - } - } - - @nowarn("msg=parameter value evidence") - def compileImpl[T: c.WeakTypeTag, P: c.WeakTypeTag]( - c: Context - )(p: c.Expr[P])(@unused ev: c.Expr[P <:< Parser[T]]): c.Expr[() => P] = { - import c.universe._ - val tree = p.tree - val expr = q"$tree" - val parser = c.eval(c.Expr[Parser[T]](c.untypecheck(expr.duplicate))) - val groups = q"..${parser.state.registry.map(_.generate(c))}" - val tree2 = tree match { - case Apply(Select(tree2 @ Select(_, _), _), _) => tree2 - case _ => - throw new Error( - s""" ERROR: Wrong shape - | Expected Apply(Select(Select(_, name), _), _), got: - | ${showRaw(tree)} - |""".stripMargin - ) - } - - val addGroupDefs = new Transformer { - override def transform(tree: Tree): Tree = tree match { - case Template(parents, self, body) => - val exprs = q"..$groups;None".asInstanceOf[Block].stats - Template(parents, self, body ++ exprs) - case node => super.transform(node) - } - } - - val clsDef = c.parse(s"final class __Parser__ extends $tree2") - val tgtDef = addGroupDefs.transform(clsDef) - - val finalCode = q"$tgtDef; () => { new __Parser__() }" - - c.Expr[() => P](finalCode) - } -} diff --git a/lib/scala/flexer/src/main/scala/org/enso/flexer/state/Rule.scala b/lib/scala/flexer/src/main/scala/org/enso/flexer/state/Rule.scala deleted file mode 100644 index b4f533e5724f..000000000000 --- a/lib/scala/flexer/src/main/scala/org/enso/flexer/state/Rule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package org.enso.flexer.state - -import org.enso.flexer.automata.Pattern -import org.enso.flexer.spec.Macro - -final case class Rule(pattern: Pattern, tree: String) -object Rule { - final case class Builder(pattern: Pattern, finalizer: Rule => Unit) { - def run(program: String): Unit = finalizer(Rule(pattern, program)) - def ||(program: => Unit): Unit = macro Macro.runRule - } -} diff --git a/lib/scala/parser-service/README.md b/lib/scala/parser-service/README.md deleted file mode 100644 index f561bd1a8348..000000000000 --- a/lib/scala/parser-service/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Parser Service - -A debugging service for the Enso parser. diff --git a/lib/scala/parser-service/src/main/scala/org/enso/ParserService.scala b/lib/scala/parser-service/src/main/scala/org/enso/ParserService.scala deleted file mode 100644 index 2926b3be40bf..000000000000 --- a/lib/scala/parser-service/src/main/scala/org/enso/ParserService.scala +++ /dev/null @@ -1,77 +0,0 @@ -package org.enso - -import io.circe.Json -import org.enso.flexer.Reader -import org.enso.parserservice.Protocol -import org.enso.parserservice.Server -import org.enso.syntax.text.{docparser, AST, DocParser, Parser, SourceFile} - -import scala.util.Try - -object ParserService { - val HOSTNAME_VAR = "ENSO_PARSER_HOSTNAME" - val PORT_VAR = "ENSO_PARSER_PORT" - - val DEFAULT_PORT = 30615 - val DEFAULT_HOSTNAME = "localhost" - - /** Obtains configuration from environment, filling missing values with - * defaults. - */ - def configFromEnv(): Server.Config = { - val hostname = sys.env.getOrElse(HOSTNAME_VAR, DEFAULT_HOSTNAME) - val port = sys.env - .get(PORT_VAR) - .flatMap(str => Try { str.toInt }.toOption) - .getOrElse(DEFAULT_PORT) - Server.Config(hostname, port) - } -} - -/** Class that allows setting up parser service with given configuration. */ -case class ParserService() extends Server with Protocol { - import parserservice._ - import Protocol._ - - def serializeAst(ast: AST.Module): String = ast.toJson().noSpaces - - def handleRequest(request: Request): Response = { - request match { - case ParseRequest(program, ids) => - val ast = new Parser().run(new Reader(program), ids) - Protocol.Success(SourceFile(ast, Json.Null)) - case ParseRequestWithMetadata(content) => - val module = new Parser().runWithMetadata(content) - Protocol.Success(module) - case DocParserGenerateHtmlSource(program) => - val parser = new Parser() - val module = parser.run(program) - val dropMeta = parser.dropMacroMeta(module) - val doc = docparser.DocParserRunner.createDocs(dropMeta) - val code = - docparser.DocParserHTMLGenerator.generateHTMLForEveryDocumented(doc) - Protocol.SuccessDoc(code) - case DocParserGenerateHtmlFromDoc(code) => - val doc = DocParser.runMatched(code) - val htmlCode = docparser.DocParserHTMLGenerator.generateHTMLPureDoc(doc) - Protocol.SuccessDoc(htmlCode) - case _ => - throw new Exception(f"unimplemented request: $request") - } - } -} - -/** Runs a simple WebSocket server that wraps Parser into a service. */ -object ParserServiceMain extends App { - import ParserService._ - println("Getting configuration from environment...") - val config = configFromEnv() - - println(s"Will serve ${config.addressString()}") - println( - s"To change configuration, restart with $HOSTNAME_VAR or " + - s"$PORT_VAR variables set to desired values" - ) - val service = ParserService() - service.start(config) -} diff --git a/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala b/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala deleted file mode 100644 index 664fc1a05395..000000000000 --- a/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala +++ /dev/null @@ -1,52 +0,0 @@ -package org.enso.parserservice - -import io.circe.parser._ -import io.circe.generic.auto._ -import io.circe.syntax._ -import org.enso.syntax.text.Parser -import org.enso.syntax.text.SourceFile - -/** Types implementing parser server protocol. - * - * The protocol always is single request -> single response. - */ -object Protocol { - sealed trait Request - final case class ParseRequest(program: String, ids: Parser.IDMap) - extends Request - final case class ParseRequestWithMetadata(content: String) extends Request - final case class DocParserGenerateHtmlSource(program: String) extends Request - final case class DocParserGenerateHtmlFromDoc(code: String) extends Request - - sealed trait Response - final case class Success(module: SourceFile) extends Response - final case class Error(message: String) extends Response - final case class SuccessDoc(code: String) extends Response -} - -/** Helper for implementing protocol over text-based transport. - * - * Requests and responses are marshaled as text using JSON - * (and default circe serialzation schema). - */ -trait Protocol { - import Protocol._ - - /** Generate [[Response]] for a given [[Request]]. - * - * Any [[Throwable]] thrown out of implementation will be translated into - * [[Protocol.Error]] message. - */ - def handleRequest(request: Request): Response - - def handleMessage(input: String): String = { - try { - decode[Request](input) match { - case Left(err) => throw err - case Right(request) => handleRequest(request).asJson.noSpaces - } - } catch { - case e: Throwable => (Error(e.toString): Response).asJson.noSpaces - } - } -} diff --git a/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala b/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala deleted file mode 100644 index 8be020086d1d..000000000000 --- a/lib/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala +++ /dev/null @@ -1,107 +0,0 @@ -package org.enso.parserservice - -import akka.NotUsed -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.HttpMethods.GET -import akka.http.scaladsl.model.HttpRequest -import akka.http.scaladsl.model.HttpResponse -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.model.ws.BinaryMessage -import akka.http.scaladsl.model.ws.Message -import akka.http.scaladsl.model.ws.TextMessage -import akka.http.scaladsl.model.ws.UpgradeToWebSocket -import akka.stream.scaladsl.Flow -import akka.stream.scaladsl.Sink -import akka.stream.scaladsl.Source - -import scala.annotation.nowarn -import scala.concurrent.ExecutionContext -import scala.util.Failure -import scala.util.Success - -object Server { - - /** Describes endpoint to which [[Server]] can bind. */ - final case class Config(interface: String, port: Int) { - def addressString(): String = s"ws://$interface:$port" - } -} - -/** WebSocket server supporting synchronous request-response protocol. - * - * Server when run binds to endpoint and accepts establishing web socket - * connection for any number of peers. - * - * Server replies to each incoming text message with a single text message. - * Server accepts a single Text Message from a peer and responds with - * another Text Message. - */ -trait Server { - implicit val system: ActorSystem = ActorSystem() - - /** Generate text reply for given request text message. */ - def handleMessage(input: String): String - - /** Akka stream defining server behavior. - * - * Incoming [[TextMessage]]s are replied to (see [[handleMessage]]). - * Incoming binary messages are ignored. - */ - val handlerFlow: Flow[Message, TextMessage.Strict, NotUsed] = - Flow[Message] - .flatMapConcat { - case tm: TextMessage => - val strict = tm.textStream.fold("")(_ + _) - strict.map(input => TextMessage(handleMessage(input))) - case bm: BinaryMessage => - bm.dataStream.runWith(Sink.ignore) - Source.empty - } - - /** Server behavior upon receiving HTTP request. - * - * As server implements websocket-based protocol, this implementation accepts - * only GET requests to set up WebSocket connection. - * - * The request's URI is not checked. - */ - @nowarn("cat=deprecation") - val handleRequest: HttpRequest => HttpResponse = { - case req @ HttpRequest(GET, _, _, _, _) => - req.header[UpgradeToWebSocket] match { - case Some(upgrade) => - println("Establishing a new connection") - upgrade.handleMessages(handlerFlow) - case None => - HttpResponse( - StatusCodes.BadRequest, - entity = "Not a valid websocket request!" - ) - } - case r: HttpRequest => - r.discardEntityBytes() - HttpResponse(StatusCodes.MethodNotAllowed) - } - - /** Starts a HTTP server listening at the given endpoint. - * - * Function is asynchronous, will return immediately. If the server fails to - * start, function will exit the process with a non-zero code. - */ - def start(config: Server.Config): Unit = { - val bindingFuture = - Http() - .newServerAt(config.interface, config.port) - .bindSync(handleRequest) - - bindingFuture.onComplete({ - case Success(_) => - println(s"Server online at ${config.addressString()}") - case Failure(exception) => - println(s"Failed to start server: $exception") - system.terminate() - System.exit(1) - })(ExecutionContext.global) - } -} diff --git a/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala b/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala index bd0bf133a913..4a6afaee2317 100644 --- a/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala +++ b/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala @@ -35,8 +35,7 @@ object SuggestionRandom { def nextSuggestionModule(): Suggestion.Module = Suggestion.Module( module = nextString(), - documentation = optional(nextString()), - documentationHtml = optional(nextString()) + documentation = optional(nextString()) ) def nextSuggestionType(): Suggestion.Type = @@ -47,8 +46,7 @@ object SuggestionRandom { params = Seq(), returnType = nextString(), parentType = optional(nextString()), - documentation = optional(nextString()), - documentationHtml = optional(nextString()) + documentation = optional(nextString()) ) def nextSuggestionConstructor(): Suggestion.Constructor = @@ -58,8 +56,7 @@ object SuggestionRandom { name = nextString(), arguments = Seq(), returnType = nextString(), - documentation = optional(nextString()), - documentationHtml = optional(nextString()) + documentation = optional(nextString()) ) def nextSuggestionMethod(): Suggestion.Method = @@ -71,8 +68,7 @@ object SuggestionRandom { selfType = nextString(), returnType = nextString(), isStatic = Random.nextBoolean(), - documentation = optional(nextString()), - documentationHtml = optional(nextString()) + documentation = optional(nextString()) ) def nextSuggestionFunction(): Suggestion.Function = diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala index aa817af6fcdf..b7713e177c36 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala @@ -979,7 +979,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit suggestion: Suggestion ): (SuggestionRow, Seq[Suggestion.Argument]) = suggestion match { - case Suggestion.Module(module, doc, _, _, reexport) => + case Suggestion.Module(module, doc, reexport) => val row = SuggestionRow( id = None, externalIdLeast = None, @@ -1007,8 +1007,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit returnType, parentType, doc, - _, - _, reexport ) => val row = SuggestionRow( @@ -1037,8 +1035,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit args, returnType, doc, - _, - _, reexport ) => val row = SuggestionRow( @@ -1069,8 +1065,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit returnType, isStatic, doc, - _, - _, reexport ) => val row = SuggestionRow( @@ -1099,8 +1093,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit sourceType, returnType, doc, - _, - _, reexport ) => val firstArg = Suggestion.Argument( @@ -1136,9 +1128,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit args, returnType, scope, - doc, - _, - _ + doc ) => val row = SuggestionRow( id = None, @@ -1159,7 +1149,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit reexport = None ) row -> args - case Suggestion.Local(expr, module, name, returnType, scope, doc, _, _) => + case Suggestion.Local(expr, module, name, returnType, scope, doc) => val row = SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), @@ -1220,66 +1210,56 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit suggestion.kind match { case SuggestionKind.MODULE => Suggestion.Module( - module = suggestion.module, - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None, - reexport = suggestion.reexport + module = suggestion.module, + documentation = suggestion.documentation, + reexport = suggestion.reexport ) case SuggestionKind.TYPE => Suggestion.Type( externalId = toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), - module = suggestion.module, - name = suggestion.name, - params = arguments.sortBy(_.index).map(toArgument), - returnType = suggestion.returnType, - parentType = suggestion.parentType, - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None, - reexport = suggestion.reexport + module = suggestion.module, + name = suggestion.name, + params = arguments.sortBy(_.index).map(toArgument), + returnType = suggestion.returnType, + parentType = suggestion.parentType, + documentation = suggestion.documentation, + reexport = suggestion.reexport ) case SuggestionKind.CONSTRUCTOR => Suggestion.Constructor( externalId = toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), - module = suggestion.module, - name = suggestion.name, - arguments = arguments.sortBy(_.index).map(toArgument), - returnType = suggestion.returnType, - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None, - reexport = suggestion.reexport + module = suggestion.module, + name = suggestion.name, + arguments = arguments.sortBy(_.index).map(toArgument), + returnType = suggestion.returnType, + documentation = suggestion.documentation, + reexport = suggestion.reexport ) case SuggestionKind.METHOD => Suggestion.Method( externalId = toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), - module = suggestion.module, - name = suggestion.name, - arguments = arguments.sortBy(_.index).map(toArgument), - selfType = suggestion.selfType, - returnType = suggestion.returnType, - isStatic = suggestion.isStatic, - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None, - reexport = suggestion.reexport + module = suggestion.module, + name = suggestion.name, + arguments = arguments.sortBy(_.index).map(toArgument), + selfType = suggestion.selfType, + returnType = suggestion.returnType, + isStatic = suggestion.isStatic, + documentation = suggestion.documentation, + reexport = suggestion.reexport ) case SuggestionKind.CONVERSION => Suggestion.Conversion( externalId = toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), - module = suggestion.module, - arguments = arguments.sortBy(_.index).tail.map(toArgument), - sourceType = arguments.minBy(_.index).tpe, - returnType = suggestion.returnType, - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None, - reexport = suggestion.reexport + module = suggestion.module, + arguments = arguments.sortBy(_.index).tail.map(toArgument), + sourceType = arguments.minBy(_.index).tpe, + returnType = suggestion.returnType, + documentation = suggestion.documentation, + reexport = suggestion.reexport ) case SuggestionKind.FUNCTION => Suggestion.Function( @@ -1299,9 +1279,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit suggestion.scopeEndOffset ) ), - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None + documentation = suggestion.documentation ) case SuggestionKind.LOCAL => Suggestion.Local( @@ -1320,9 +1298,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit suggestion.scopeEndOffset ) ), - documentation = suggestion.documentation, - documentationHtml = None, - documentationSections = None + documentation = suggestion.documentation ) case k => throw new NoSuchElementException(s"Unknown suggestion kind: $k") diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Size.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/data/Size.scala index 61c83499eefa..19a3316b7903 100644 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Size.scala +++ b/lib/scala/syntax/definition/src/main/scala/org/enso/data/Size.scala @@ -1,8 +1,5 @@ package org.enso.data -import org.enso.syntax.text.AST -import org.enso.syntax.text.ast.meta.Pattern - /** Strongly typed size for a container. */ case class Size(value: Int) extends AnyVal with Ordered[Size] { def +(offset: Size): Size = Size(value + offset.value) @@ -10,14 +7,6 @@ case class Size(value: Int) extends AnyVal with Ordered[Size] { } object Size { - val Empty = Size(0) - def apply(pat: Pattern.Match): Size = Size(pat.toStream) - def apply(ast: AST): Size = Size(ast.span) - def apply(text: String): Size = Size(text.length) - def apply(ast: Shifted[AST]): Size = Size(ast.off) + Size(ast.wrapped) - def apply[A](elems: Seq[Shifted[AST]]): Size = { - var ret = Size(0) - elems.foreach(ret += Size(_)) - ret - } + val Empty = Size(0) + def apply(text: String): Size = Size(text.length) } diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Span.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/data/Span.scala index 397f586092b9..0a7d5167350c 100644 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Span.scala +++ b/lib/scala/syntax/definition/src/main/scala/org/enso/data/Span.scala @@ -1,7 +1,5 @@ package org.enso.data -import org.enso.syntax.text.AST - /** Strongly typed span in a container. */ case class Span(index: Index, size: Size) extends Ordered[Span] { @@ -14,10 +12,6 @@ case class Span(index: Index, size: Size) extends Ordered[Span] { } object Span { - - def apply(pos: Index, ast: AST): Span = - Span(pos, Size(ast.span)) - def apply(text: String): Span = Span(Index.Start, Size(text)) diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala deleted file mode 100644 index 306e37870fbd..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala +++ /dev/null @@ -1,66 +0,0 @@ -package org.enso.data - -import io.circe.{Decoder, Encoder, Json} -import io.circe.syntax._ -import io.circe.generic.auto._ - -final case class Tree[K, V](value: Option[V], branches: Map[K, Tree[K, V]]) { - def +(item: (List[K], V)): Tree[K, V] = item._1 match { - case Nil => this.copy(value = Some(item._2)) - case p :: ps => { - val newBranch = branches.getOrElse(p, Tree[K, V]()) + (ps -> item._2) - this.copy(branches = branches + (p -> newBranch)) - } - } - - def map[S](f: V => S): Tree[K, S] = - Tree(value.map(f), branches.view.mapValues(_.map(f)).toMap) - - def dropValues(): Tree[K, Unit] = - map(_ => ()) - - def get(key: K): Option[Tree[K, V]] = - branches.get(key) - - def get(path: List[K]): Option[Tree[K, V]] = path match { - case Nil => Some(this) - case p :: ps => branches.get(p).flatMap(_.get(ps)) - } - - def getValue(path: List[K]): Option[V] = - get(path).flatMap(_.value) - - def isLeaf: Boolean = - branches.isEmpty -} - -object Tree { - def apply[K, V](): Tree[K, V] = new Tree(None, Map()) - def apply[K, V](deps: (List[K], V)*): Tree[K, V] = - deps.foldLeft(Tree[K, V]())(_ + _) - - ////////////////// - // JSON support // - ////////////////// - - /* Note [Tree Serialization] */ - implicit def jsonEncode[K: Encoder, V: Encoder]: Encoder[Tree[K, V]] = - tree => - Json.obj( - "value" -> tree.value.asJson, - "branches" -> tree.branches.toSeq.asJson - ) - - /* Note [Tree Serialization] - * We can't directly serialize Map[K,V], as circe tries to use whole K as a - * key string in the generated JSON. Thus, we serialize Map[K, V] by - * converting it to Seq[(K,V)] first. - */ - - /* Note [Tree Serialization] */ - implicit def jsonDecode[K: Decoder, V: Decoder]: Decoder[Tree[K, V]] = - Decoder.forProduct2("value", "branches")( - (value: Option[V], branches: Seq[(K, Tree[K, V])]) => - Tree(value, branches.toMap) - ) -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala index ff8633c65a37..5b5ce01b8a74 100644 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala +++ b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala @@ -1,105 +1,6 @@ package org.enso.syntax.text -import java.util.UUID - -import cats.{Foldable, Functor, Monoid} -import cats.derived._ -import cats.implicits._ -import io.circe.{Encoder, Json} -import org.enso.data.List1._ -import org.enso.data._ -import org.enso.syntax.text.HasSpan.implicits._ -import org.enso.syntax.text.ast.Repr.{R, _} -import org.enso.syntax.text.ast.meta.Pattern -import org.enso.syntax.text.ast.{opr, Doc, Repr} -import org.enso.syntax.text.ast.text.{Escape, RawEscape} - -import scala.annotation.{nowarn, tailrec, unused} -import scala.reflect.ClassTag - -/* Note [JSON Serialization] - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * Using Circe's auto-derived `asJson` on AST is extremely costly in terms - * of compile-time resource usage. It adds like 2-4 min to compile time. - * For that reason we should only have one place where it is used and other - * places where AST needs to be serialized should use this wrapper. - * - * Also, it must be placed in this package having it separate for some reason - * increases compile-time memory usage, causing CI builds to fail. Someone might - * want to reinvestigate this in future. - * - * Also, this function definition can't be just "anywhere" in the file, but - * near bottom to "properly" see other things. - * - * When working on this file, it is recommended to temporarily replace - * function body with ??? expression to radically improve compiler throughput. - * - * Also note that JSON serialization is meant to be kept synchronized with Rust - * AST implementation. Any changes made to serialization here (that includes - * changes to case classes' names and field names) need to be kept in sync with - * the Rust side. - */ - -////////////////////////////////////////////////////////////////////////////// -//// HasSpan ///////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -trait HasSpan[T] { - def span(t: T): Int -} -object HasSpan { - def apply[T: HasSpan]: HasSpan[T] = implicitly[HasSpan[T]] - - object implicits { - implicit class ToHasSpanOps[T: HasSpan](t: T) { - def span(): Int = { - implicitly[HasSpan[T]].span(t) - } - } - } - - implicit def fromShifted[T: HasSpan]: HasSpan[Shifted[T]] = { shifted => - val ev = implicitly[HasSpan[T]] - shifted.off + ev.span(shifted.wrapped) - } - - implicit def fromOption[T: HasSpan]: HasSpan[Option[T]] = - opt => opt.map(_.span()).getOrElse(0) - implicit def fromList[T: HasSpan]: HasSpan[List[T]] = - list => list.map(_.span()).sum - implicit def fromList1[T: HasSpan]: HasSpan[List1[T]] = - list => list.toList.span() - implicit def fromShiftedList1[T: HasSpan]: HasSpan[Shifted.List1[T]] = - list => list.head.span() + list.tail.span() -} - -////////////////////////////////////////////////////////////////////////////// -//// OffsetZip /////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -/** Zips every child [[A]] with offset from the left side of the parent - * node. The offset is a number of UTF-8 code points. - */ -trait OffsetZip[F[A], A] { - def zipWithOffset(t: F[A]): F[(Index, A)] -} -object OffsetZip { - def apply[F[A], A](implicit ev: OffsetZip[F, A]): OffsetZip[F, A] = ev - def apply[F[A], A](t: F[A])(implicit ev: OffsetZip[F, A]): F[(Index, A)] = - OffsetZip[F, A].zipWithOffset(t) - - //// Default Instances //// - implicit def fromStream[T: HasSpan]: OffsetZip[AST.StreamOf, T] = { stream => - val ev = implicitly[HasSpan[T]] - var off = Index.Start - stream.map { t => - off += Size(t.off) - val out = t.map((off, _)) - off += Size(ev.span(t.wrapped)) - out - } - } -} +import cats.Monoid //////////////////////////////////////////////////////////////////////////////// //// AbsolutePosition ////////////////////////////////////////////////////////// @@ -135,2443 +36,3 @@ object Location { } } } - -////////////////////////////////////////////////////////////////////////////// -//// Phantom ///////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -/** Phantom type. Use with care, as Scala cannot prove its proper usage. When - * a type is phantom, then its last type argument is not used and we can - * safely coerce it to something else. - */ -sealed trait Phantom -object Phantom { - implicit class PhantomOps[T[_] <: Phantom](ident: T[_]) { - def coerce[S]: T[S] = ident.asInstanceOf[T[S]] - } -} - -////////////////////////////////////////////////////////////////////////////// -//// Shape /////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -sealed trait Shape[T] - -@nowarn("msg=parameter value evidence") -object Shape extends ShapeImplicit { - import AST.StreamOf - import HasSpan.implicits._ - - /// Utils /// - val newline = R + '\n' - - ///////////////// - //// Invalid //// - ///////////////// - sealed trait Invalid[T] extends Shape[T] - final case class Unrecognized[T](str: String) extends Invalid[T] with Phantom - final case class Unexpected[T](msg: String, stream: StreamOf[T]) - extends Invalid[T] - final case class DanglingBase[T](base: String) extends Invalid[T] with Phantom - final case class TextUnclosed[T](line: TextLine[T]) - extends Text[T] - with Invalid[T] { - def quote = line.quote - } - final case class InvalidQuote[T](quote: Builder) - extends Invalid[T] - with Phantom - final case class InlineBlock[T](quote: Builder) - extends Invalid[T] - with Phantom - - /////////////////// - /// Identifiers /// - /////////////////// - sealed trait Ident[T] extends Shape[T] with Phantom { val name: String } - - final case class Blank[T]() extends Ident[T] { val name = "_" } - final case class Var[T](name: String) extends Ident[T] - final case class Cons[T](name: String) extends Ident[T] - final case class Mod[T](name: String) extends Ident[T] - final case class Opr[T](name: String) extends Ident[T] { - val (prec, assoc) = opr.Info.of(name) - } - final case class Annotation[T](name: String) extends Ident[T] - final case class InvalidSuffix[T](elem: AST.Ident, suffix: String) - extends Invalid[T] - with Phantom - - /////////////// - /// Literal /// - /////////////// - sealed trait Literal[T] extends Shape[T] - - ////////////// - /// Number /// - ////////////// - final case class Number[T](base: Option[String], int: String) - extends Literal[T] - with Phantom - - //////////// - /// Text /// - //////////// - sealed trait Text[T] extends Shape[T] with Literal[T] { - def quote: Repr.Builder - } - - /// Line /// - sealed trait TextLine[T] extends Text[T] - final case class TextLineRaw[T](text: List[SegmentRaw[T]]) - extends TextLine[T] - with Phantom { - val quote = '"' - } - final case class TextLineFmt[T](text: List[SegmentFmt[T]]) - extends TextLine[T] { - val quote = '\'' - } - - /// Block /// - sealed trait TextBlock[T] extends Text[T] - final case class TextBlockLine[+T](empty_lines: List[Int], text: List[T]) - final case class TextBlockRaw[T]( - text: List[TextBlockLine[SegmentRaw[T]]], - spaces: Int, - offset: Int - ) extends TextBlock[T] - with Phantom { - val quote = "\"\"\"" - } - final case class TextBlockFmt[T]( - text: List[TextBlockLine[SegmentFmt[T]]], - spaces: Int, - offset: Int - ) extends TextBlock[T] { - val quote = "'''" - } - - /// Segment /// - sealed trait Segment[T] - sealed trait SegmentFmt[T] extends Segment[T] - sealed trait SegmentRaw[T] extends SegmentFmt[T] with Phantom - - final case class SegmentPlain[T](value: String) extends SegmentRaw[T] - final case class SegmentExpr[T](value: Option[T]) extends SegmentFmt[T] - final case class SegmentEscape[T](code: Escape) - extends SegmentFmt[T] - with Phantom - final case class SegmentRawEscape[T](code: RawEscape) - extends SegmentRaw[T] - with Phantom - - /////////// - /// App /// - /////////// - sealed trait App[T] extends Shape[T] - final case class Prefix[T](func: T, off: Int, arg: T) extends App[T] - final case class Infix[T]( - larg: T, - loff: Int, - opr: T, - roff: Int, - rarg: T - ) extends App[T] - - sealed trait Section[T] extends App[T] - final case class SectionLeft[T](arg: T, off: Int, opr: AST.Opr) - extends Section[T] - final case class SectionRight[T](opr: AST.Opr, off: Int, arg: T) - extends Section[T] - final case class SectionSides[T](opr: AST.Opr) extends Section[T] with Phantom - - // Note: [Custom Encoder] - final case class Block[T]( - ty: Block.Type, - indent: Int, - emptyLines: List[Int], - firstLine: Block.Line[T], - lines: List[Block.Line[Option[T]]], - isOrphan: Boolean = false - ) extends Shape[T] { - // FIXME: Compatibility mode - def replaceType(ntyp: Block.Type): Block[T] = copy(ty = ntyp) - def replaceFirstLine(line: Block.Line[T]): Block[T] = - copy(firstLine = line) - def replaceLines(lines: List[Block.Line[Option[T]]]): Block[T] = - copy(lines = lines) - } - - /* Note [Custom encoder] - * ~~~~~~~~~~~~~~~~~~~~~ - * This type uses custom encoder in [[AstOps.toJson]]. It needs to be updated - * as well, when fields are changed. - */ - - ////////////// - /// Module /// - ////////////// - final case class Module[T](lines: List1[Block.OptLine[T]]) extends Shape[T] { - def setLines(lines: List1[Block.OptLine[T]]) = copy(lines = lines) - } - - ///////////// - /// Macro /// - ///////////// - sealed trait Macro[T] extends Shape[T] - final case class Match[T]( - pfx: Option[Pattern.Match], - segs: Shifted.List1[Match.Segment[T]], - resolved: Option[AST] - ) extends Macro[T] { - def path: List1[AST] = segs.toList1().map(_.wrapped.head) - } - final case class Ambiguous[T]( - segs: Shifted.List1[Ambiguous.Segment], - paths: Tree[AST, Unit] - ) extends Macro[T] - - ///////////////////// - /// Spaceless AST /// - ///////////////////// - sealed trait SpacelessAST[T] extends Shape[T] - final case class Comment[T](lines: List[String]) - extends SpacelessAST[T] - with Phantom - final case class Documented[T](doc: Doc, emptyLinesBetween: Int, ast: T) - extends SpacelessAST[T] - final case class Import[T]( - path: List1[AST.Ident], - rename: Option[AST.Ident.Cons], - isAll: Boolean, - onlyNames: Option[List1[AST.Ident]], - hidingNames: Option[List1[AST.Ident]] - ) extends SpacelessAST[T] - final case class Export[T]( - path: List1[AST.Ident], - rename: Option[AST.Ident.Cons], - isAll: Boolean, - onlyNames: Option[List1[AST.Ident]], - hidingNames: Option[List1[AST.Ident]] - ) extends SpacelessAST[T] - final case class JavaImport[T]( - path: List1[AST.Ident], - rename: Option[AST.Ident.Cons] - ) extends SpacelessAST[T] - final case class Mixfix[T](name: List1[AST.Ident], args: List1[T]) - extends SpacelessAST[T] - final case class Group[T](body: Option[T]) extends SpacelessAST[T] - final case class SequenceLiteral[T](items: List[T]) extends SpacelessAST[T] - final case class TypesetLiteral[T](expression: Option[T]) - extends SpacelessAST[T] - final case class Def[T](name: AST.Cons, args: List[T], body: Option[T]) - extends SpacelessAST[T] - final case class Foreign[T](indent: Int, lang: String, code: List[String]) - extends SpacelessAST[T] - final case class Modified[T](modifier: String, definition: T) - extends SpacelessAST[T] - - ////////////////////////////////////////////////////////////////////////////// - // Companion objects ///////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - // TODO: All companion objects can be generated with macros - - /** Helper to gather common implementation for traits betwen [[Shape]] and - * leaf case classes. They are implemented by delegating to [[Shape]] trait - * implementation. - */ - trait IntermediateTrait[S[U] <: Shape[U]] { - implicit def repr[T: Repr]: Repr[S[T]] = Shape.repr[T].repr(_) - // implicit def ozip[T: HasSpan]: OffsetZip[S, T] = { ident => - // Shape.ozip[T].zipWithOffset(ident).asInstanceOf - // //OffsetZip[Shape, T](ident).asInstanceOf - // } - implicit def span[T: HasSpan]: HasSpan[S[T]] = - t => (t: Shape[T]).span() - } - - object Unrecognized { - implicit def ftor: Functor[Unrecognized] = semiauto.functor - implicit def fold: Foldable[Unrecognized] = semiauto.foldable - implicit def repr[T]: Repr[Unrecognized[T]] = _.str - implicit def ozip[T]: OffsetZip[Unrecognized, T] = t => t.coerce - implicit def span[T]: HasSpan[Unrecognized[T]] = _.str.length - } - - object Unexpected { - implicit def ftor: Functor[Unexpected] = semiauto.functor - implicit def fold: Foldable[Unexpected] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Unexpected[T]] = t => Repr(t.stream) - implicit def ozip[T: HasSpan]: OffsetZip[Unexpected, T] = - t => t.copy(stream = OffsetZip(t.stream)) - implicit def span[T: HasSpan]: HasSpan[Unexpected[T]] = - t => t.stream.span() - } - object Ident { - implicit def ftor: Functor[Ident] = semiauto.functor - implicit def fold: Foldable[Ident] = semiauto.foldable - implicit def repr[T]: Repr[Ident[T]] = _.name - implicit def ozip[T: HasSpan]: OffsetZip[Ident, T] = { ident => - OffsetZip[Shape, T](ident).asInstanceOf - } - } - object Blank { - implicit def ftor: Functor[Blank] = semiauto.functor - implicit def fold: Foldable[Blank] = semiauto.foldable - implicit def repr[T]: Repr[Blank[T]] = _.name - implicit def ozip[T]: OffsetZip[Blank, T] = t => t.coerce - implicit def span[T]: HasSpan[Blank[T]] = _ => 1 - } - object Var { - implicit def ftor: Functor[Var] = semiauto.functor - implicit def fold: Foldable[Var] = semiauto.foldable - implicit def repr[T]: Repr[Var[T]] = _.name - implicit def ozip[T]: OffsetZip[Var, T] = t => t.coerce - implicit def span[T]: HasSpan[Var[T]] = t => t.name.length - } - object Cons { - implicit def ftor: Functor[Cons] = semiauto.functor - implicit def fold: Foldable[Cons] = semiauto.foldable - implicit def repr[T]: Repr[Cons[T]] = _.name - implicit def ozip[T]: OffsetZip[Cons, T] = t => t.coerce - implicit def span[T]: HasSpan[Cons[T]] = t => t.name.length - } - object Mod { - implicit def ftor: Functor[Mod] = semiauto.functor - implicit def fold: Foldable[Mod] = semiauto.foldable - implicit def repr[T]: Repr[Mod[T]] = R + _.name + "=" - implicit def ozip[T]: OffsetZip[Mod, T] = t => t.coerce - implicit def span[T]: HasSpan[Mod[T]] = t => t.name.length + 1 - } - object Opr { - implicit def ftor: Functor[Opr] = semiauto.functor - implicit def fold: Foldable[Opr] = semiauto.foldable - implicit def repr[T]: Repr[Opr[T]] = _.name - implicit def ozip[T]: OffsetZip[Opr, T] = t => t.coerce - implicit def span[T]: HasSpan[Opr[T]] = t => t.name.length - } - object Annotation { - implicit def ftor: Functor[Annotation] = semiauto.functor - implicit def fold: Foldable[Annotation] = semiauto.foldable - implicit def repr[T]: Repr[Annotation[T]] = _.name - implicit def ozip[T]: OffsetZip[Annotation, T] = t => t.coerce - implicit def span[T]: HasSpan[Annotation[T]] = t => t.name.length - } - object InvalidSuffix { - implicit def ftor: Functor[InvalidSuffix] = semiauto.functor - implicit def fold: Foldable[InvalidSuffix] = semiauto.foldable - implicit def ozip[T]: OffsetZip[InvalidSuffix, T] = t => t.coerce - implicit def repr[T]: Repr[InvalidSuffix[T]] = - t => R + t.elem.repr + t.suffix - implicit def span[T]: HasSpan[InvalidSuffix[T]] = - t => t.elem.span() + t.suffix.length - } - object Literal extends IntermediateTrait[Literal] { - implicit def ftor: Functor[Literal] = semiauto.functor - implicit def fold: Foldable[Literal] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Literal, T] = { t => - OffsetZip[Shape, T](t).asInstanceOf - } - } - object Number { - implicit def fromInt[T](int: Int): AST.Number = AST.Number(int) - implicit def ftor: Functor[Number] = semiauto.functor - implicit def fold: Foldable[Number] = semiauto.foldable - implicit def ozip[T]: OffsetZip[Number, T] = t => t.coerce - implicit def repr[T]: Repr[Number[T]] = - t => t.base.map(_ + "_").getOrElse("") + t.int - implicit def span[T]: HasSpan[Number[T]] = - t => t.base.map(_.length + 1).getOrElse(0) + t.int.length - } - object DanglingBase { - implicit def ftor: Functor[DanglingBase] = semiauto.functor - implicit def fold: Foldable[DanglingBase] = semiauto.foldable - implicit def repr[T]: Repr[DanglingBase[T]] = R + _.base + '_' - implicit def ozip[T]: OffsetZip[DanglingBase, T] = t => t.coerce - implicit def span[T]: HasSpan[DanglingBase[T]] = - t => t.base.length + 1 - } - object Text extends IntermediateTrait[Text] { - implicit def ftor: Functor[Text] = semiauto.functor - implicit def fold: Foldable[Text] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Text, T] = { t => - OffsetZip[Shape, T](t).asInstanceOf - } - } - object TextUnclosed { - implicit def ftor: Functor[TextUnclosed] = semiauto.functor - implicit def fold: Foldable[TextUnclosed] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TextUnclosed[T]] = { - case TextUnclosed(t: TextLineRaw[T]) => t.quote + t.text - case TextUnclosed(t: TextLineFmt[T]) => t.quote + t.text - } - implicit def ozip[T: HasSpan]: OffsetZip[TextUnclosed, T] = - t => t.copy(line = OffsetZip(t.line)) - implicit def span[T: HasSpan]: HasSpan[TextUnclosed[T]] = { - case TextUnclosed(t: TextLineRaw[T]) => t.quote.span + t.text.span() - case TextUnclosed(t: TextLineFmt[T]) => t.quote.span + t.text.span() - } - } - object InvalidQuote { - implicit def ftor: Functor[InvalidQuote] = semiauto.functor - implicit def fold: Foldable[InvalidQuote] = semiauto.foldable - implicit def repr[T: Repr]: Repr[InvalidQuote[T]] = _.quote - implicit def ozip[T]: OffsetZip[InvalidQuote, T] = t => t.coerce - implicit def span[T]: HasSpan[InvalidQuote[T]] = _.quote.span - } - object InlineBlock { - implicit def ftor: Functor[InlineBlock] = semiauto.functor - implicit def fold: Foldable[InlineBlock] = semiauto.foldable - implicit def repr[T]: Repr[InlineBlock[T]] = _.quote - implicit def ozip[T]: OffsetZip[InlineBlock, T] = t => t.coerce - implicit def span[T]: HasSpan[InlineBlock[T]] = _.quote.span - } - object TextLine extends IntermediateTrait[TextLine] { - implicit def ftor: Functor[TextLine] = semiauto.functor - implicit def fold: Foldable[TextLine] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[TextLine, T] = { - case t: TextLineRaw[T] => OffsetZip(t) - case t: TextLineFmt[T] => OffsetZip(t) - } - } - object TextLineRaw { - implicit def ftor: Functor[TextLineRaw] = semiauto.functor - implicit def fold: Foldable[TextLineRaw] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TextLineRaw[T]] = - t => t.quote + t.text + t.quote - implicit def ozip[T]: OffsetZip[TextLineRaw, T] = t => t.coerce - implicit def span[T: HasSpan]: HasSpan[TextLineRaw[T]] = - t => (2 * t.quote.span) + t.text.map(_.span()).sum - } - object TextLineFmt { - implicit def ftor: Functor[TextLineFmt] = semiauto.functor - implicit def fold: Foldable[TextLineFmt] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TextLineFmt[T]] = - t => t.quote + t.text + t.quote - implicit def ozip[T: HasSpan]: OffsetZip[TextLineFmt, T] = { t => - var offset = Index(t.quote.span) - val text2 = for (elem <- t.text) yield { - val offElem = elem.map(offset -> _) - offset += Size(elem.span()) - offElem - } - TextLineFmt(text2) - } - implicit def span[T: HasSpan]: HasSpan[TextLineFmt[T]] = - t => (2 * t.quote.span) + t.text.map(_.span()).sum - } - - object TextBlock extends IntermediateTrait[TextBlock] { - def lineRepr[T: Repr](off: Int, l: TextBlockLine[SegmentFmt[T]]): Builder = - R + l.empty_lines.map(newline + _) + newline + off + l.text - def lineSpan[T: HasSpan](off: Int, l: TextBlockLine[SegmentFmt[T]]): Int = { - val emptyLinesSpan = l.empty_lines.map(newline.span + _).sum - emptyLinesSpan + newline.span + off + l.text.span() - } - - implicit def ftor: Functor[TextBlock] = semiauto.functor - implicit def fold: Foldable[TextBlock] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[TextBlock, T] = { - case body: TextBlockRaw[T] => OffsetZip(body) - case body: TextBlockFmt[T] => OffsetZip(body) - } - } - - object TextBlockRaw { - implicit def ftor: Functor[TextBlockRaw] = semiauto.functor - implicit def fold: Foldable[TextBlockRaw] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TextBlockRaw[T]] = - t => t.quote + t.spaces + t.text.map(TextBlock.lineRepr(t.offset, _)) - implicit def ozip[T: HasSpan]: OffsetZip[TextBlockRaw, T] = t => t.coerce - implicit def span[T: HasSpan]: HasSpan[TextBlockRaw[T]] = { t => - val linesSpan = t.text.map(TextBlock.lineSpan(t.offset, _)).sum - t.quote.span + t.spaces + linesSpan - } - } - - object TextBlockFmt { - implicit def ftor: Functor[TextBlockFmt] = semiauto.functor - implicit def fold: Foldable[TextBlockFmt] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TextBlockFmt[T]] = - t => t.quote + t.spaces + t.text.map(TextBlock.lineRepr(t.offset, _)) - implicit def ozip[T: HasSpan]: OffsetZip[TextBlockFmt, T] = { body => - var offset = Index(body.quote.span) - val text = - for (line <- body.text) yield { - offset += Size(line.empty_lines.length + line.empty_lines.sum) - offset += Size(1 + body.offset) - val text = for (elem <- line.text) yield { - val offElem = elem.map(offset -> _) - offset += Size(elem.span()) - offElem - } - line.copy(text = text) - } - body.copy(text = text) - } - implicit def span[T: HasSpan]: HasSpan[TextBlockFmt[T]] = { t => - val linesSpan = t.text.map(TextBlock.lineSpan(t.offset, _)).sum - t.quote.span + t.spaces + linesSpan - } - } - - object Segment { - implicit def ftor: Functor[Segment] = semiauto.functor - implicit def fold: Foldable[Segment] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Segment[T]] = { - case t: SegmentRaw[T] => Repr(t) - case t: SegmentFmt[T] => Repr(t) - } - implicit def ozip[T: HasSpan]: OffsetZip[Segment, T] = { - case t: SegmentRaw[T] => OffsetZip(t) - case t: SegmentFmt[T] => OffsetZip(t) - } - implicit def span[T: HasSpan]: HasSpan[Segment[T]] = { - case t: SegmentRaw[T] => t.span() - case t: SegmentFmt[T] => t.span() - } - } - object SegmentFmt { - implicit def ftor[T]: Functor[SegmentFmt] = semiauto.functor - implicit def fold: Foldable[SegmentFmt] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SegmentFmt[T]] = { - case t: SegmentPlain[T] => Repr(t) - case t: SegmentExpr[T] => Repr(t) - case t: SegmentEscape[T] => Repr(t) - case t: SegmentRawEscape[T] => Repr(t) - } - implicit def ozip[T]: OffsetZip[SegmentFmt, T] = { - case t: SegmentPlain[T] => OffsetZip(t) - case t: SegmentExpr[T] => OffsetZip(t) - case t: SegmentEscape[T] => OffsetZip(t) - case t: SegmentRawEscape[T] => OffsetZip(t) - } - implicit def span[T: HasSpan]: HasSpan[SegmentFmt[T]] = { - case t: SegmentPlain[T] => t.span() - case t: SegmentExpr[T] => t.span() - case t: SegmentEscape[T] => t.span() - case t: SegmentRawEscape[T] => t.span() - } - } - object SegmentRaw { - implicit def ftor[T]: Functor[SegmentRaw] = semiauto.functor - implicit def fold: Foldable[SegmentRaw] = semiauto.foldable - implicit def repr[T]: Repr[SegmentRaw[T]] = { - case t: SegmentPlain[T] => Repr(t) - case t: SegmentRawEscape[T] => Repr(t) - } - implicit def ozip[T]: OffsetZip[SegmentRaw, T] = { - case t: SegmentPlain[T] => OffsetZip(t) - case t: SegmentRawEscape[T] => OffsetZip(t) - } - implicit def span[T]: HasSpan[SegmentRaw[T]] = { - case t: SegmentPlain[T] => t.span() - case t: SegmentRawEscape[T] => t.span() - } - } - object SegmentPlain { - implicit def txtFromString[T](str: String): SegmentPlain[T] = - SegmentPlain(str) - - implicit def fold: Foldable[SegmentPlain] = semiauto.foldable - implicit def ftor[T]: Functor[SegmentPlain] = semiauto.functor - implicit def repr[T]: Repr[SegmentPlain[T]] = _.value - implicit def ozip[T]: OffsetZip[SegmentPlain, T] = - t => t.coerce - implicit def span[T]: HasSpan[SegmentPlain[T]] = _.value.length - } - object SegmentExpr { - val quote: Repr.Builder = "`" - - implicit def ftor[T]: Functor[SegmentExpr] = semiauto.functor - implicit def fold: Foldable[SegmentExpr] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SegmentExpr[T]] = - R + quote + _.value + quote - implicit def ozip[T]: OffsetZip[SegmentExpr, T] = - _.map(Index.Start -> _) - implicit def span[T: HasSpan]: HasSpan[SegmentExpr[T]] = - quote.span + _.value.span() + quote.span - } - object SegmentEscape { - val introducer: Repr.Builder = "\\" - - implicit def ftor: Functor[SegmentEscape] = semiauto.functor - implicit def fold: Foldable[SegmentEscape] = semiauto.foldable - implicit def repr[T]: Repr[SegmentEscape[T]] = - t => introducer + t.code.repr - implicit def ozip[T]: OffsetZip[SegmentEscape, T] = - t => t.coerce - implicit def span[T: HasSpan]: HasSpan[SegmentEscape[T]] = - introducer.span + _.code.repr.length - } - object SegmentRawEscape { - val introducer: Repr.Builder = "\\" - - implicit def ftor: Functor[SegmentRawEscape] = semiauto.functor - implicit def fold: Foldable[SegmentRawEscape] = semiauto.foldable - implicit def repr[T]: Repr[SegmentRawEscape[T]] = - t => introducer + t.code.repr - implicit def ozip[T]: OffsetZip[SegmentRawEscape, T] = - t => t.coerce - implicit def span[T]: HasSpan[SegmentRawEscape[T]] = - introducer.span + _.code.repr.length - } - object App extends IntermediateTrait[App] { - implicit def ftor[T]: Functor[App] = semiauto.functor - implicit def fold: Foldable[App] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[App, T] = - t => OffsetZip[Shape, T](t).asInstanceOf - } - object Prefix { - implicit def ftor: Functor[Prefix] = semiauto.functor - implicit def fold: Foldable[Prefix] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Prefix[T]] = - t => R + t.func + t.off + t.arg - implicit def ozip[T: HasSpan]: OffsetZip[Prefix, T] = - t => - t.copy( - func = (Index.Start, t.func), - arg = (Index(t.func.span() + t.off), t.arg) - ) - implicit def span[T: HasSpan]: HasSpan[Prefix[T]] = - t => t.func.span() + t.off + t.arg.span() - - } - object Infix { - implicit def ftor: Functor[Infix] = semiauto.functor - implicit def fold: Foldable[Infix] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Infix[T]] = - t => R + t.larg + t.loff + t.opr + t.roff + t.rarg - implicit def ozip[T: HasSpan]: OffsetZip[Infix, T] = - t => { - val larg = Index.Start -> t.larg - val opr = Index(t.larg.span() + t.loff) -> t.opr - val rarg = - Index(t.larg.span() + t.loff + t.opr.span() + t.roff) -> t.rarg - t.copy(larg = larg, opr = opr, rarg = rarg) - } - implicit def span[T: HasSpan]: HasSpan[Infix[T]] = - t => t.larg.span() + t.loff + t.opr.span() + t.roff + t.rarg.span() - } - - object Section extends IntermediateTrait[Section] { - implicit def ftor[T]: Functor[Section] = semiauto.functor - implicit def fold: Foldable[Section] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Section, T] = - t => OffsetZip[Shape, T](t).asInstanceOf - } - object SectionLeft { - implicit def ftor: Functor[SectionLeft] = semiauto.functor - implicit def fold: Foldable[SectionLeft] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SectionLeft[T]] = - t => R + t.arg + t.off + t.opr - implicit def ozip[T]: OffsetZip[SectionLeft, T] = - t => t.copy(arg = (Index.Start, t.arg)) - implicit def span[T: HasSpan]: HasSpan[SectionLeft[T]] = - t => t.arg.span() + t.off + t.opr.span - } - object SectionRight { - implicit def ftor: Functor[SectionRight] = semiauto.functor - implicit def fold: Foldable[SectionRight] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SectionRight[T]] = - t => R + t.opr + t.off + t.arg - implicit def ozip[T]: OffsetZip[SectionRight, T] = - t => t.copy(arg = (Index(t.opr.span + t.off), t.arg)) - implicit def span[T: HasSpan]: HasSpan[SectionRight[T]] = - t => t.opr.span + t.off + t.arg.span() - } - object SectionSides { - implicit def ftor: Functor[SectionSides] = semiauto.functor - implicit def fold: Foldable[SectionSides] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SectionSides[T]] = t => R + t.opr - implicit def ozip[T]: OffsetZip[SectionSides, T] = t => t.coerce - implicit def span[T: HasSpan]: HasSpan[SectionSides[T]] = - t => t.opr.span - } - - object Block { - implicit def ftorBlock: Functor[Block] = semiauto.functor - implicit def fold: Foldable[Block] = semiauto.foldable - implicit def reprBlock[T: Repr]: Repr[Block[T]] = - t => { - val headRepr = if (t.isOrphan) R else newline - val emptyLinesRepr = t.emptyLines.map(R + _ + newline) - val firstLineRepr = R + t.indent + t.firstLine - val linesRepr = t.lines.map { line => - newline + line.elem.map(_ => t.indent) + line - } - headRepr + emptyLinesRepr + firstLineRepr + linesRepr - } - implicit def ozipBlock[T: HasSpan]: OffsetZip[Block, T] = - t => { - var index = 0 - index += (if (t.isOrphan) 0 else 1) - index += t.emptyLines.map(_ + 1).sum - index += t.indent - val line = t.firstLine.copy(elem = (Index(index), t.firstLine.elem)) - index += t.firstLine.span() + newline.span - val lines = for (line <- t.lines) yield { - val elem = line.elem.map(elem => { - index += t.indent - (Index(index), elem) - }) - index += line.span() + newline.span - line.copy(elem = elem) - } - t.copy(firstLine = line, lines = lines) - } - implicit def span[T: HasSpan]: HasSpan[Block[T]] = - t => { - val headSpan = if (t.isOrphan) 0 else 1 - val emptyLinesSpan = t.emptyLines.map(_ + 1).sum - val firstLineSpan = t.indent + t.firstLine.span() - def lineSpan(line: Shape.Block.OptLine[T]): Int = { - val indentSpan = if (line.elem.isDefined) t.indent else 0 - newline.span + indentSpan + line.span() - } - val linesSpan = t.lines.map(lineSpan).sum - headSpan + emptyLinesSpan + firstLineSpan + linesSpan - } - - /// Block type /// - sealed trait Type - final case object Continuous extends Type - final case object Discontinuous extends Type - - /// Block Line /// - type OptLine[T] = Line[Option[T]] - final case class Line[+T](elem: T, off: Int) { - // FIXME: Compatibility mode - def toOptional: Line[Option[T]] = copy(elem = Some(elem)) - } - object Line { - implicit def ftor: Functor[Line] = semiauto.functor - implicit def fold: Foldable[Line] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Line[T]] = t => R + t.elem + t.off - implicit def span[T: HasSpan]: HasSpan[Line[T]] = - t => t.elem.span() + t.off - implicit def spanOpt[T: HasSpan]: HasSpan[OptLine[T]] = - t => t.elem.map(_.span()).getOrElse(0) + t.off - } - } - - object Module { - implicit def ftor: Functor[Module] = semiauto.functor - implicit def fold: Foldable[Module] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Module, T] = - t => { - var index = 0 - val lines = t.lines.map { line => - val elem = line.elem.map((Index(index), _)) - index += line.span() + newline.span - line.copy(elem = elem) - } - t.copy(lines = lines) - } - - implicit def repr[T: Repr]: Repr[Module[T]] = - t => R + t.lines.head + t.lines.tail.map(newline + _) - implicit def span[T: HasSpan]: HasSpan[Module[T]] = - t => t.lines.span() + (t.lines.size - 1) * newline.span - } - - object Macro extends IntermediateTrait[Macro] { - implicit def ftor[T]: Functor[Macro] = semiauto.functor - implicit def fold: Foldable[Macro] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Macro, T] = - t => OffsetZip[Shape, T](t).asInstanceOf - } - - object Match { - /// Instances /// - implicit def ftor: Functor[Match] = semiauto.functor - implicit def fold: Foldable[Match] = semiauto.foldable - implicit def ozip[T: HasSpan]: OffsetZip[Match, T] = - t => { - var off = 0 - t.copy(segs = t.segs.map { seg => - OffsetZip(seg).map(_.map(_.map(s => { - val loff = off - off = s._2.span() - (s._1 + Size(loff), s._2) - }))) - }) - } - @nowarn("cat=unused-imports") - implicit def repr[T: Repr]: Repr[Match[T]] = - t => { - import AST.ASTOf._ - val pfxStream = t.pfx.map(_.toStream.reverse).getOrElse(List()) - val pfxRepr = pfxStream.map(t => R + t.wrapped + t.off) - R + pfxRepr + t.segs - } - implicit def span[T: HasSpan]: HasSpan[Match[T]] = { t => - val pfxSpan = t.pfx.span() - val segsSpan = t.segs.span() - pfxSpan + segsSpan - } - - /// Segment /// - final case class Segment[T]( - head: AST.Ident, - body: Pattern.MatchOf[Shifted[T]] - ) { - def isValid: Boolean = body.isValid - def map( - f: Pattern.MatchOf[Shifted[T]] => Pattern.MatchOf[Shifted[T]] - ): Segment[T] = - copy(body = f(body)) - } - - object Segment { - implicit def repr[T: Repr]: Repr[Segment[T]] = - t => R + t.head + t.body - implicit def ozip[T: HasSpan]: OffsetZip[Segment, T] = - t => { - t.copy(body = OffsetZip(t.body).map { case (i, s) => - s.map((i + Size(t.head.span), _)) - }) - } - implicit def span[T: HasSpan]: HasSpan[Segment[T]] = - t => t.head.span + t.body.span() - - def apply[T](head: AST.Ident): Shape.Match.Segment[T] = - Shape.Match.Segment(head, Pattern.Match.Nothing()) - } - } - - object Ambiguous { - implicit def ftor: Functor[Ambiguous] = semiauto.functor - implicit def fold: Foldable[Ambiguous] = semiauto.foldable - implicit def repr[T]: Repr[Ambiguous[T]] = - t => R + t.segs.map(Repr(_)) - implicit def ozip[T]: OffsetZip[Ambiguous, T] = - _.map(Index.Start -> _) - implicit def span[T: HasSpan]: HasSpan[Ambiguous[T]] = t => t.segs.span() - - final case class Segment(head: AST, body: Option[AST.SAST]) - object Segment { - def apply(head: AST): Segment = Segment(head, None) - implicit def repr: Repr[Segment] = t => R + t.head + t.body - implicit def span: HasSpan[Segment] = t => t.head.span + t.body.span() - } - } - - object Comment { - val symbol = "#" - implicit def ftor: Functor[Comment] = semiauto.functor - implicit def fold: Foldable[Comment] = semiauto.foldable - implicit def repr[T]: Repr[Comment[T]] = - R + symbol + symbol + _.lines.mkString("\n") - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Comment, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Comment[T]] = _ => 0 - } - - object Documented { - import Comment.symbol - implicit def ftor[T]: Functor[Documented] = semiauto.functor - implicit def fold[T]: Foldable[Documented] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Documented[T]] = - t => { - val symbolRepr = R + symbol + symbol - val betweenDocAstRepr = - R + newline + newline.build() * t.emptyLinesBetween - R + symbolRepr + t.doc + betweenDocAstRepr + t.ast - } - implicit def offsetZip[T]: OffsetZip[Documented, T] = - _.map(Index.Start -> _) - implicit def span[T: HasSpan]: HasSpan[Documented[T]] = _ => 0 - - implicit def toJson[T]: Encoder[Documented[T]] = - _ => throw new NotImplementedError() - } - - object Import { - implicit def ftor: Functor[Import] = semiauto.functor - implicit def fold: Foldable[Import] = semiauto.foldable - implicit def repr[T]: Repr[Import[T]] = - t => R + "import" + t.path.repr.build() - - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Import, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Import[T]] = _ => 0 - } - - object Export { - implicit def ftor: Functor[Export] = semiauto.functor - implicit def fold: Foldable[Export] = semiauto.foldable - implicit def repr[T]: Repr[Export[T]] = - t => R + "export" + t.path.repr.build() - - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Export, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Export[T]] = _ => 0 - } - - object JavaImport { - implicit def ftor: Functor[JavaImport] = semiauto.functor - implicit def fold: Foldable[JavaImport] = semiauto.foldable - implicit def repr[T]: Repr[JavaImport[T]] = - t => - R + ("polyglot java import " + t.path - .map(_.repr.build()) - .toList - .mkString(".")) - - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[JavaImport, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[JavaImport[T]] = _ => 0 - } - - object Mixfix { - implicit def ftor: Functor[Mixfix] = semiauto.functor - implicit def fold: Foldable[Mixfix] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Mixfix[T]] = - t => { - val lastRepr = if (t.name.length == t.args.length) List() else List(R) - val argsRepr = t.args.toList.map(R + " " + _) ++ lastRepr - val nameRepr = t.name.toList.map(Repr(_)) - R + nameRepr.lazyZip(argsRepr).map(_ + _) - } - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Mixfix, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Mixfix[T]] = _ => 0 - } - - object Group { - implicit def ftor: Functor[Group] = semiauto.functor - implicit def fold: Foldable[Group] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Group[T]] = - R + "(" + _.body + ")" - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Group, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Group[T]] = _ => 0 - } - - object SequenceLiteral { - implicit def ftor: Functor[SequenceLiteral] = semiauto.functor - implicit def fold: Foldable[SequenceLiteral] = semiauto.foldable - implicit def repr[T: Repr]: Repr[SequenceLiteral[T]] = - t => R + "[" + t.items.map(_.repr.build()).mkString(", ") + "]" - implicit def ozip[T]: OffsetZip[SequenceLiteral, T] = - _.map(Index.Start -> _) - implicit def span[T]: HasSpan[SequenceLiteral[T]] = _ => 0 - - } - - object TypesetLiteral { - implicit def ftor: Functor[TypesetLiteral] = semiauto.functor - implicit def fold: Foldable[Def] = semiauto.foldable - implicit def repr[T: Repr]: Repr[TypesetLiteral[T]] = - t => s"{ ${t.expression.repr.build()} }" - implicit def ozip[T]: OffsetZip[TypesetLiteral, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[TypesetLiteral[T]] = _ => 0 - } - - object Def { - implicit def ftor: Functor[Def] = semiauto.functor - implicit def fold: Foldable[Def] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Def[T]] = - t => R + Def.symbol + 1 + t.name + t.args.map(R + 1 + _) + t.body - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Def, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Def[T]] = _ => 0 - - val symbol = "def" - } - - object Foreign { - implicit def ftor: Functor[Foreign] = semiauto.functor - implicit def fold: Foldable[Foreign] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Foreign[T]] = - t => { - val code2 = t.code.map(R + t.indent + _).mkString("\n") - R + "foreign " + t.lang + "\n" + code2 - } - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Foreign, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Foreign[T]] = _ => 0 - } - - object Modified { - implicit def ftor: Functor[Modified] = semiauto.functor - implicit def fold: Foldable[Modified] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Modified[T]] = - t => { - R + t.modifier + t.definition.repr.build() - } - // FIXME: How to make it automatic for non-spaced AST? - implicit def ozip[T]: OffsetZip[Modified, T] = _.map(Index.Start -> _) - implicit def span[T]: HasSpan[Modified[T]] = _ => 0 - } - - //// Implicits //// - - object implicits { - implicit class ToShapeOps[T[S] <: Shape[S]](t: T[AST])(implicit - functor: Functor[T], - ozip: OffsetZip[T, AST] - ) { - - def map(f: AST => AST): T[AST] = { - Functor[T].map(t)(f) - } - - def mapWithOff(f: (Index, AST) => AST): T[AST] = - Functor[T].map(ozip.zipWithOffset(t))(f.tupled) - } - - implicit class ToShapeOpsRepr[T[S] <: Shape[S]](t: T[AST])(implicit - repr: Repr[T[AST]] - ) { - def show(): String = repr.repr(t).build() - } - } -} - -//////////////////////////////////////////////////////////////////////////////// -//// Shape Boilerplate ///////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// -// TODO [MWU] The repr, ozip and span are almost entirely boilerplate. -// Consider providing them using macros. -sealed trait ShapeImplicit { - import Shape._ - - implicit def ftor: Functor[Shape] = semiauto.functor - implicit def fold: Foldable[Shape] = semiauto.foldable - implicit def repr[T: Repr]: Repr[Shape[T]] = { - case s: Unrecognized[T] => s.repr - case s: Unexpected[T] => s.repr - case s: Blank[T] => s.repr - case s: Var[T] => s.repr - case s: Cons[T] => s.repr - case s: Opr[T] => s.repr - case s: Annotation[T] => s.repr - case s: Mod[T] => s.repr - case s: InvalidSuffix[T] => s.repr - case s: Number[T] => s.repr - case s: DanglingBase[T] => s.repr - case s: TextUnclosed[T] => s.repr - case s: InvalidQuote[T] => s.repr - case s: InlineBlock[T] => s.repr - case s: TextLineRaw[T] => s.repr - case s: TextLineFmt[T] => s.repr - case s: TextBlockRaw[T] => s.repr - case s: TextBlockFmt[T] => s.repr - case s: Prefix[T] => s.repr - case s: Infix[T] => s.repr - case s: SectionLeft[T] => s.repr - case s: SectionRight[T] => s.repr - case s: SectionSides[T] => s.repr - case s: Block[T] => s.repr - case s: Module[T] => s.repr - case s: Ambiguous[T] => s.repr - case s: Match[T] => s.repr - // spaceless - case s: Comment[T] => s.repr - case s: Documented[T] => s.repr - case s: Import[T] => s.repr - case s: Export[T] => s.repr - - case s: JavaImport[T] => s.repr - case s: Mixfix[T] => s.repr - case s: Group[T] => s.repr - case s: SequenceLiteral[T] => s.repr - case s: TypesetLiteral[T] => s.repr - case s: Def[T] => s.repr - case s: Foreign[T] => s.repr - case s: Modified[T] => s.repr - } - implicit def ozip[T: HasSpan]: OffsetZip[Shape, T] = { - case s: Unrecognized[T] => OffsetZip[Unrecognized, T].zipWithOffset(s) - case s: Unexpected[T] => OffsetZip[Unexpected, T].zipWithOffset(s) - case s: Blank[T] => OffsetZip[Blank, T].zipWithOffset(s) - case s: Var[T] => OffsetZip[Var, T].zipWithOffset(s) - case s: Cons[T] => OffsetZip[Cons, T].zipWithOffset(s) - case s: Opr[T] => OffsetZip[Opr, T].zipWithOffset(s) - case s: Annotation[T] => OffsetZip[Annotation, T].zipWithOffset(s) - case s: Mod[T] => OffsetZip[Mod, T].zipWithOffset(s) - case s: InvalidSuffix[T] => OffsetZip[InvalidSuffix, T].zipWithOffset(s) - case s: Number[T] => OffsetZip[Number, T].zipWithOffset(s) - case s: DanglingBase[T] => OffsetZip[DanglingBase, T].zipWithOffset(s) - case s: TextUnclosed[T] => OffsetZip[TextUnclosed, T].zipWithOffset(s) - case s: InvalidQuote[T] => OffsetZip[InvalidQuote, T].zipWithOffset(s) - case s: InlineBlock[T] => OffsetZip[InlineBlock, T].zipWithOffset(s) - case s: TextLineRaw[T] => OffsetZip[TextLineRaw, T].zipWithOffset(s) - case s: TextLineFmt[T] => OffsetZip[TextLineFmt, T].zipWithOffset(s) - case s: TextBlockRaw[T] => OffsetZip[TextBlockRaw, T].zipWithOffset(s) - case s: TextBlockFmt[T] => OffsetZip[TextBlockFmt, T].zipWithOffset(s) - case s: Prefix[T] => OffsetZip[Prefix, T].zipWithOffset(s) - case s: Infix[T] => OffsetZip[Infix, T].zipWithOffset(s) - case s: SectionLeft[T] => OffsetZip[SectionLeft, T].zipWithOffset(s) - case s: SectionRight[T] => OffsetZip[SectionRight, T].zipWithOffset(s) - case s: SectionSides[T] => OffsetZip[SectionSides, T].zipWithOffset(s) - case s: Block[T] => OffsetZip[Block, T].zipWithOffset(s) - case s: Module[T] => OffsetZip[Module, T].zipWithOffset(s) - case s: Ambiguous[T] => OffsetZip[Ambiguous, T].zipWithOffset(s) - case s: Match[T] => OffsetZip[Match, T].zipWithOffset(s) - // spaceless - case s: Comment[T] => OffsetZip[Comment, T].zipWithOffset(s) - case s: Documented[T] => OffsetZip[Documented, T].zipWithOffset(s) - case s: Import[T] => OffsetZip[Import, T].zipWithOffset(s) - case s: Export[T] => OffsetZip[Export, T].zipWithOffset(s) - - case s: JavaImport[T] => OffsetZip[JavaImport, T].zipWithOffset(s) - case s: Mixfix[T] => OffsetZip[Mixfix, T].zipWithOffset(s) - case s: Group[T] => OffsetZip[Group, T].zipWithOffset(s) - case s: SequenceLiteral[T] => OffsetZip[SequenceLiteral, T].zipWithOffset(s) - case s: TypesetLiteral[T] => OffsetZip[TypesetLiteral, T].zipWithOffset(s) - case s: Def[T] => OffsetZip[Def, T].zipWithOffset(s) - case s: Foreign[T] => OffsetZip[Foreign, T].zipWithOffset(s) - case s: Modified[T] => OffsetZip[Modified, T].zipWithOffset(s) - } - - implicit def span[T: HasSpan]: HasSpan[Shape[T]] = { - case s: Unrecognized[T] => s.span() - case s: Unexpected[T] => s.span() - case s: Blank[T] => s.span() - case s: Var[T] => s.span() - case s: Cons[T] => s.span() - case s: Opr[T] => s.span() - case s: Annotation[T] => s.span() - case s: Mod[T] => s.span() - case s: InvalidSuffix[T] => s.span() - case s: Number[T] => s.span() - case s: DanglingBase[T] => s.span() - case s: TextUnclosed[T] => s.span() - case s: InvalidQuote[T] => s.span() - case s: InlineBlock[T] => s.span() - case s: TextLineRaw[T] => s.span() - case s: TextLineFmt[T] => s.span() - case s: TextBlockRaw[T] => s.span() - case s: TextBlockFmt[T] => s.span() - case s: Prefix[T] => s.span() - case s: Infix[T] => s.span() - case s: SectionLeft[T] => s.span() - case s: SectionRight[T] => s.span() - case s: SectionSides[T] => s.span() - case s: Block[T] => s.span() - case s: Module[T] => s.span() - case s: Ambiguous[T] => s.span() - case s: Match[T] => s.span() - // spaceless - case s: Comment[T] => s.span() - case s: Documented[T] => s.span() - case s: Import[T] => s.span() - case s: Export[T] => s.span() - case s: JavaImport[T] => s.span() - case s: Mixfix[T] => s.span() - case s: Group[T] => s.span() - case s: SequenceLiteral[T] => s.span() - case s: TypesetLiteral[T] => s.span() - case s: Def[T] => s.span() - case s: Foreign[T] => s.span() - case s: Modified[T] => s.span() - } -} - -////////////////////////////////////////////////////////////////////////////// -//// AST ///////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -/** =AST= - * - * AST is encoded as a simple recursion scheme. See the following links to - * learn more about the concept: - * - https://wiki.haskell.org/Catamorphisms - * - https://www.schoolofhaskell.com/user/edwardk/recursion-schemes/catamorphisms - * - https://www.schoolofhaskell.com/user/bartosz/understanding-algebras - * - http://hackage.haskell.org/package/free-5.1.2/docs/Control-Comonad-Cofree.html - * - https://www.47deg.com/blog/basic-recursion-schemes-in-scala/ - * - * ==AST Shape== - * - * Every AST node like [[AST.Ident.Var]] or [[AST.App.Prefix]] defines a shape - * of its subtree. Shapes extend [[Shape]], are parametrized with a child type, - * and are defined within the [[Shape]] object. Shapes contain information - * about names of children and spacing between them, for example, the - * [[Shape.Prefix]] shape contains reference to function being its first child - * ([[Shape.Prefix.func]]), spacing between the function and its argument - * ([[Shape.Prefix.off]]), and the argument itself ([[Shape.Prefix.arg]]). - * - * ==[[ASTOf]] as Catamorphism== - * - * In order to keep the types simple and make the inference predictive, we - * are not using standard catamorphism implementations. Instead, we have - * implemented a simple recursion scheme in [[ASTOf]]. Every AST node uses it - * as the wrapping layer. For example, the most generic AST type, [[AST]] is - * defined just as an alias to [[(ASTOf[Shape])]]. Every AST node follows - * the same scheme, including [[AST.Ident.Var]] being an alias to - * [[(ASTOf[Shape.Var])]], or [[AST.App.Prefix]] being an alias to - * [[(ASTOf[Shape.Prefix])]]. - * - * ==[[ASTOf]] as Cofree== - * - * [[ASTOf]] adds a layer of additional information to each AST node. - * - * Currently the additional information include only an optional [[UUID]] and - * cached span value, however this set might grow in the future. This design - * minimizes the necessary boilerplate in storing repeatable information across - * AST. Moreover, we can easily make [[ASTOf]] polymorphic and allow the Syntax - * Tree to be tagged with different information in different compilation stages - * if necessary. - * - * ==[[ASTOf]] as Cache Layer== - * - * Shapes and AST nodes implement several type classes, including: - * - [[Functor]] - Defines mapping over every element in a shape. - * - [[Repr]] - Defines shape to code translation. - * - [[OffsetZip]] - Zips every shape element with offset from the left side - * of the shape. - * - * [[ASTOf]] caches the span value. This way querying AST subtree for it span - * is always O(1). - * - * ==[[ASTOf]] as Method Provider== - * - * Because [[ASTOf]] has access to all the type class instances of the child - * element (and they cannot be further exposed because [[ASTOf]] type parameter - * has to be variant), it is a perfect place for exposing common utils for AST - * nodes. Please note, that "exposing" means both providing as well as caching. - * For example, when we eval `myAST.map(a => a)` we are not doing pattern match - * as one may expect. During the creation of [[ASTOf]], the functor of the - * shape was obtained and the `map` method references it, so instead of pattern - * matching, we are acessing the `map` method directly. - * - * ==Fields Access== - * - * Please note, that [[ASTOf]] is "transparent". There are - * implicit defs of both wrapping and unwrapping functions, which makes - * working with AST nodes very convenient. For example, there is no need to - * write `myVar.shape.name` to first unpack the node from the [[ASTOf]] layer - * and then access its name. It's possible to just write `myVar.name`, and - * the unpacking will be performed automatically. - * - * ==Pattern Matching== - * - * Please note that due to type erasure, it is impossible to pattern match on - * AST types. Never use `case _: Var => ...` statement, as it will probably - * crash at runtime. In order to pattern match on AST type, each AST node - * provides a special "any" matcher, which results in a type narrowed version - * of the AST node. For example, `case Var.any(v) => ...` will succeed if the - * match was performed on any [[AST.Ident.Var]] and its result `v` will be of - * [[AST.Ident.Var]] type. Of course, it is possible to use structural matching - * without any restrictions. - */ -object AST { - import Shape.implicits._ - - ////////////////////////////////////////////////////////////////////////////// - //// Reexports /////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Assoc = opr.Assoc - val Assoc = opr.Assoc - val Prec = opr.Prec - - ////////////////////////////////////////////////////////////////////////////// - //// Definition ////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - //// Structure //// - - // type Shape = Shape[AST] - type _AST = ASTOf[Shape] - - //// Aliases //// - - type SAST = Shifted[AST] - type StreamOf[T] = List[Shifted[T]] - type StreamOf1[T] = List1[Shifted[T]] - type Stream = StreamOf[AST] - type Stream1 = StreamOf1[AST] - type ID = UUID - - //// API //// - - def tokenize(ast: AST): Shifted.List1[AST] = { - @tailrec - def go(ast: AST, out: AST.Stream): Shifted.List1[AST] = - ast match { - case App.Prefix.any(t) => go(t.func, Shifted(t.off, t.arg) :: out) - case _ => Shifted.List1(ast, out) - } - go(ast, List()) - } - - //// Conversions //// - - object conversions extends conversions - sealed trait conversions extends Ident.conversions { - implicit def intToAST(int: Int): AST = - Literal.Number(int) - - implicit def stringToAST(str: String): AST = { - if (str == "") throw new Error("Empty literal") - if (str == "_") Blank() - else if (str.head.isLower) Var(str) - else if (str.head.isUpper) Cons(str) - else Opr(str) - } - } - - //////////////////////////////////// - //// Apply / Unapply Generators //// - //////////////////////////////////// - - /** [[Unapply]] and [[UnapplyByType]] are unapply generators for AST Shapes. - * The implementation may seem complex, but this is just a scala way for - * deconstructing types. When provided with a AST type, like [[Ident.Var]], - * [[Unapply]] deconstructs it to [[(ASTOf[VarOf])]] and then generates - * an object providing unapply implementation for the [[VarOf]] type. - */ - sealed trait Unapply[T] { - type In - def run[Out](f: In => Out)(t: AST): Option[Out] - } - object Unapply { - def apply[T](implicit t: Unapply[T]): Unapply[T] { type In = t.In } = t - implicit def inst[T[_]](implicit - ev: ClassTag[T[AST]] - ): Unapply[ASTOf[T]] { type In = T[AST] } = - new Unapply[ASTOf[T]] { - type In = T[AST] - val ct = implicitly[ClassTag[T[AST]]] - def run[Out](fn: In => Out)(t: AST) = ct.unapply(t.shape).map(fn) - } - } - - /** See the documentation for [[Unapply]] */ - sealed trait UnapplyByType[T] { - def unapply(t: AST): Option[T] - } - object UnapplyByType { - def apply[T](implicit ev: UnapplyByType[T]) = ev - implicit def instance[T[_]](implicit - ct: ClassTag[T[_]] - ): UnapplyByType[ASTOf[T]] = - new UnapplyByType[ASTOf[T]] { - def unapply(t: AST) = - // Note that the `asInstanceOf` usage is safe here. - // It is used only for performance reasons, otherwise we would need - // to create a new object which would look exactly the same way - // as the original one. - ct.unapply(t.shape).map(_ => t.asInstanceOf[ASTOf[T]]) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// ASTOf /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - //// Definition //// - - /** The [[ASTOf]] class wraps each AST node. The implementation is similar to - * standard catamorphic Fix, however, it is parametrized with the head shape - * type. In combination with covariance of [[T]], it allows us to both keep - * information about the specific shape of the AST, as well as get natural - * subtyping behavior. For example, [[AST]] and [[Var]] are aliases to - * [[(ASTOf[Shape])]] and [[(AST[VarOf])]] respectively, and while - * [[(VarOf[T] <: Shape[T])]], also [[Var <: AST]]. - * - * Another important role of [[ASTOf]] is caching of [[Repr.Builder]] and - * allowing for fast method redirection. When [[ASTOf]] is created, it - * remembers a bunch of stuff, which can be fast accessed even if we cast the - * type to generic [[AST]]. - */ - final case class ASTOf[+T[_]]( - shape: T[AST], - span: Int, - id: Option[ID] = None, - location: Option[Location] = None - ) { - override def toString = s"Node($id,$location,$shape)" - override def hashCode(): Int = shape.hashCode() - - def setID(newID: Option[ID]): ASTOf[T] = copy(id = newID) - def setID(newID: ID): ASTOf[T] = setID(Some(newID)) - def withNewID(): ASTOf[T] = copy(id = Some(UUID.randomUUID())) - def withNewIDIfMissing(): ASTOf[T] = - id match { - case Some(_) => this - case None => this.withNewID() - } - def setLocation(newLocation: Option[Location]): ASTOf[T] = - copy(location = newLocation) - def setLocation(newLocation: Location): ASTOf[T] = - setLocation(Some(newLocation)) - - /** Compares ignoring cached span value and node ID. - * - * Cached span values become invalid e.g. after macros are resolved. - * Also, they are not really a part data information, they are more like - * intermediary cache used by some components that are aware of its - * caveats. - * - * ID also does not should be treated as part of node contents, it is used - * e.g. to map external data onto AST. - */ - override def equals(rhs: Any): Boolean = { - rhs match { - case rhs: ASTOf[_] => - this.shape == rhs.shape - case _ => false - } - } - } - - object ASTOf extends AstImplicits - - trait AstImplicits extends AstImplicits2 { - implicit def unwrap[T[_]](t: ASTOf[T]): T[AST] = t.shape - implicit def repr[T[S] <: Shape[S]]: Repr[ASTOf[T]] = - t => implicitly[Repr[Shape[AST]]].repr(t.shape) - implicit def span[T[_]]: HasSpan[ASTOf[T]] = t => t.span - implicit def wrap[T[_]]( - t: T[AST] - )(implicit ev: HasSpan[T[AST]]): ASTOf[T] = ASTOf(t, ev.span(t)) - - implicit def encoder_spec(implicit - ev: Encoder[Shape[AST]] - ): Encoder[AST] = encoder - } - - trait AstImplicits2 { - // Note: [JSON Schema] - implicit def encoder[T[S] <: Shape[S]](implicit - ev: Encoder[Shape[AST]] - ): Encoder[ASTOf[T]] = - ast => { - import io.circe.syntax._ - - val shape = "shape" -> ev(ast.shape) - val id = ast.id.map("id" -> _.asJson) - val span = "span" -> ast.span.asJson - val fields = Seq(shape) ++ id.toSeq :+ span - Json.fromFields(fields) - } - } - - /* Note: [JSON Schema] - * ~~~~~~~~~~~~~~~~~~~ - * Each AST node is serialized to a map with `shape`, `span` and, - * optionally, `id` keys. `shape` is always serialized as if base trait - * were encoded, even if the final case class type is known. This is - * required for consistency with Rust AST implementation, which uses a - * monomorphic AST type and has no means of expressing types like - * `AstOf[Shape.Var]`. - */ - - //// ASTOps //// - - /** [[ASTOps]] implements handy AST operations. - * - * Implementations in this class do not require any special knowledge of the - * underlying shape and thus are just a high-level AST addons. - */ - implicit class AstOps[T[S] <: Shape[S]](t: ASTOf[T])(implicit - functor: Functor[T], - fold: Foldable[T], - repr: Repr[T[AST]], - ozip: OffsetZip[T, AST] - ) { - def show(): String = repr.repr(t.shape).build() - - def map(f: AST => AST): ASTOf[T] = - t.copy(shape = t.shape.map(f)) - - def foldMap[A](f: AST => A)(implicit A: Monoid[A]): A = - fold.foldMap(t.shape)(f) - - def mapWithOff(f: (Index, AST) => AST): ASTOf[T] = - t.copy(shape = ToShapeOps(t.shape).mapWithOff(f)) - - def traverseWithOff(f: (Index, AST) => AST): ASTOf[T] = { - def go(i: Index, ast: AST): AST = - ast.mapWithOff((j, ast) => go(i + j.asSize, f(i + j.asSize, ast))) - t.mapWithOff((j, ast) => go(j, f(j, ast))) - } - - def zipWithOffset(): T[(Index, AST)] = { - OffsetZip(t.shape) - } - - def idMap(implicit ev: Foldable[Shape]): List[(Span, AST.ID)] = { - var ids = List[(Span, AST.ID)]() - var asts = List[(Index, AST)](Index.Start -> t) - while (asts.nonEmpty) { - val (off, ast) = asts.head - val children = ast.zipWithOffset().toList.map { case (o, ast) => - (o + off.asSize, ast) - } - if (ast.id.nonEmpty) - ids +:= Span(off, ast) -> ast.id.get - asts = children ++ asts.tail - } - ids.reverse - } - - // Note [JSON Serialization] - def toJson(): Json = { - import io.circe.generic.auto._ - import io.circe.syntax._ - - // Note [JSON Format Customizations] - @nowarn("msg=parameter value evidence") - @unused implicit def blockEncoder[A: Encoder]: Encoder[Shape.Block[A]] = - block => - Json.obj( - "ty" -> block.ty.asJson, - "indent" -> block.indent.asJson, - "empty_lines" -> block.emptyLines.asJson, - "first_line" -> block.firstLine.asJson, - "lines" -> block.lines.asJson, - "is_orphan" -> block.isOrphan.asJson - ) - - // Note (below) [JSON Format Customizations] - @unused implicit def escapeEncoder: Encoder[Escape] = { - case e: Escape.Character => - Json.obj("Character" -> Json.obj("c" -> e.repr.asJson)) - case e: Escape.Control => - val fields = - Json.obj("name" -> e.repr.asJson, "code" -> e.code.asJson) - Json.obj("Control" -> fields) - case e: Escape.Number => - Json.obj("Number" -> Json.obj("digits" -> e.repr.asJson)) - case e: Escape.Unicode.U16 => - Json.obj("Unicode16" -> Json.obj("digits" -> e.digits.asJson)) - case e: Escape.Unicode.U21 => - Json.obj("Unicode21" -> Json.obj("digits" -> e.digits.asJson)) - case e: Escape.Unicode.U32 => - Json.obj("Unicode32" -> Json.obj("digits" -> e.digits.asJson)) - case _ => throw new RuntimeException("Cannot encode escape.") - } - - val ast: AST = t - ast.asJson - } - } - - /* Note [JSON Format Customizations] - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * The JSON generated from serializing AST with `toJson` method is meant to be - * consumed by Rust. To ensure compatibility, several adjustments are made: - * * custom encoding for Escape, as Rust currently does not have the same - * hierarchy of types for that AST nodes. - * * custom encoding of fields which are not compatible with snake_case - * style. - */ - - ////////////////////////////////////////////////////////////////////////////// - //// Invalid ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Invalid = ASTOf[Shape.Invalid] - object Invalid { - type Unrecognized = AST.ASTOf[Shape.Unrecognized] - type Unexpected = AST.ASTOf[Shape.Unexpected] - - val any = UnapplyByType[Invalid] - - object Unrecognized { - val any = UnapplyByType[Unrecognized] - def unapply(t: AST) = - Unapply[Unrecognized].run(_.str)(t) - def apply(str: String): Unrecognized = Shape.Unrecognized[AST](str) - } - object Unexpected { - val any = UnapplyByType[Unexpected] - def unapply(t: AST) = - Unapply[Unexpected].run(t => (t.msg, t.stream))(t) - def apply(msg: String, str: Stream): Unexpected = - Shape.Unexpected(msg, str) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Ident /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - //// Reexports //// - - type Blank = Ident.Blank - type Var = Ident.Var - type Cons = Ident.Cons - type Opr = Ident.Opr - type Mod = Ident.Mod - type Annotation = Ident.Annotation - - val Blank = Ident.Blank - val Var = Ident.Var - val Cons = Ident.Cons - val Opr = Ident.Opr - val Mod = Ident.Mod - val Annotation = Ident.Annotation - - //// Definition //// - - type Ident = ASTOf[Shape.Ident] - - object Ident { - type Blank = ASTOf[Shape.Blank] - type Var = ASTOf[Shape.Var] - type Cons = ASTOf[Shape.Cons] - type Mod = ASTOf[Shape.Mod] - type Opr = ASTOf[Shape.Opr] - type Annotation = ASTOf[Shape.Annotation] - - type InvalidSuffix = ASTOf[Shape.InvalidSuffix] - - object InvalidSuffix { - val any = UnapplyByType[InvalidSuffix] - def unapply(t: AST) = - Unapply[InvalidSuffix].run(t => (t.elem, t.suffix))(t) - def apply(elem: Ident, suffix: String): InvalidSuffix = - Shape.InvalidSuffix[AST](elem, suffix) - } - - //// Conversions //// - - trait Conversions1 { - implicit def strToVar(str: String): Var = Var(str) - implicit def strToCons(str: String): Cons = Cons(str) - implicit def strToOpr(str: String): Opr = Opr(str) - implicit def strToMod(str: String): Mod = Mod(str) - } - - trait conversions extends Conversions1 { - implicit def stringToIdent(str: String): Ident = { - if (str == "") throw new Error("Empty literal") - if (str == "_") Blank() - else if (str.head.isLower) Var(str) - else if (str.head.isUpper) Cons(str) - else Opr(str) - } - } - - //// Smart Constructors //// - - val any = UnapplyByType[Ident] - - object Blank { - private val cachedBlank = Shape.Blank[AST]() - val any = UnapplyByType[Blank] - def unapply(t: AST) = Unapply[Blank].run(_ => true)(t) - def apply(): Blank = cachedBlank - } - object Var { - val any = UnapplyByType[Var] - def unapply(t: AST) = Unapply[Var].run(_.name)(t) - def apply(name: String): Var = Shape.Var[AST](name) - } - object Cons { - val any = UnapplyByType[Cons] - def unapply(t: AST) = Unapply[Cons].run(_.name)(t) - def apply(name: String): Cons = Shape.Cons[AST](name) - } - object Mod { - val any = UnapplyByType[Mod] - def unapply(t: AST) = Unapply[Mod].run(_.name)(t) - def apply(name: String): Mod = Shape.Mod[AST](name) - } - object Opr { - val app = Opr(" ") - val any = UnapplyByType[Opr] - def unapply(t: AST) = Unapply[Opr].run(_.name)(t) - def apply(name: String): Opr = Shape.Opr[AST](name) - } - object Annotation { - val any = UnapplyByType[Annotation] - def unapply(t: AST) = Unapply[Annotation].run(_.name)(t) - def apply(name: String): Annotation = Shape.Annotation[AST](name) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Literal ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - //// Reexports //// - - type Number = Literal.Number - type Text = Literal.Text - val Number = Literal.Number - val Text = Literal.Text - - //// Definition //// - - type Literal = ASTOf[Shape.Literal] - object Literal { - - val any = UnapplyByType[Literal] - - //////////////// - //// Number //// - //////////////// - - type Number = ASTOf[Shape.Number] - object Number { - type DanglingBase = ASTOf[Shape.DanglingBase] - - //// Smart Constructors //// - def apply(i: String): Number = Number(None, i) - def apply(b: String, i: String): Number = Number(Some(b), i) - def apply(i: Int): Number = Number(i.toString) - def apply(b: Int, i: String): Number = Number(b.toString, i) - def apply(b: String, i: Int): Number = Number(b, i.toString) - def apply(b: Int, i: Int): Number = Number(b.toString, i.toString) - def apply(b: Option[String], i: String): Number = - Shape.Number[AST](b, i) - def unapply(t: AST) = Unapply[Number].run(t => (t.base, t.int))(t) - val any = UnapplyByType[Number] - - //// DanglingBase //// - object DanglingBase { - val any = UnapplyByType[DanglingBase] - def apply(base: String): DanglingBase = Shape.DanglingBase[AST](base) - def unapply(t: AST) = - Unapply[DanglingBase].run(_.base)(t) - } - } - - ////////////// - //// Text //// - ////////////// - - type Text = ASTOf[Shape.Text] - object Text { - val any = UnapplyByType[Text] - - //// Definition //// - - object Line { - val Raw = Shape.TextLineRaw - type Raw[T] = Shape.TextLineRaw[T] - val Fmt = Shape.TextLineFmt - type Fmt[T] = Shape.TextLineFmt[T] - } - - object Block { - val Line = Shape.TextBlockLine - type Line[T] = Shape.TextBlockLine[T] - val Raw = Shape.TextBlockRaw - type Raw[T] = Shape.TextBlockRaw[T] - val Fmt = Shape.TextBlockFmt - type Fmt[T] = Shape.TextBlockFmt[T] - } - - ////// CONSTRUCTORS /////// - type Unclosed = ASTOf[Shape.TextUnclosed] - object Unclosed { - val any = UnapplyByType[Unclosed] - def unapply(t: AST) = - Unapply[Unclosed].run(t => t.line)(t) - def apply(segment: Segment.Fmt*): Unclosed = - Shape.TextUnclosed(Line.Fmt(segment.toList)) - object Raw { - def apply(segment: Segment.Raw*): Unclosed = - Shape.TextUnclosed(Line.Raw(segment.toList)) - } - } - type InvalidQuote = ASTOf[Shape.InvalidQuote] - object InvalidQuote { - val any = UnapplyByType[InvalidQuote] - def unapply(t: AST) = - Unapply[InvalidQuote].run(t => t.quote)(t) - def apply(quote: String): InvalidQuote = Shape.InvalidQuote[AST](quote) - } - type InlineBlock = ASTOf[Shape.InlineBlock] - object InlineBlock { - val any = UnapplyByType[InlineBlock] - def unapply(t: AST) = - Unapply[InlineBlock].run(t => t.quote)(t) - def apply(quote: String): InlineBlock = Shape.InlineBlock[AST](quote) - } - - def apply(text: Shape.Text[AST]): Text = text - def apply(segment: Segment.Fmt*): Text = Text(Line.Fmt(segment.toList)) - def apply( - spaces: Int, - off: Int, - line: Shape.TextBlockLine[Segment.Fmt]* - ): Text = - Text(Shape.TextBlockFmt(line.toList, spaces, off)) - - object Raw { - def apply(segment: Segment.Raw*): Text = - Text(Line.Raw(segment.toList)) - def apply( - spaces: Int, - off: Int, - line: Shape.TextBlockLine[Segment.Raw]* - ): Text = - Text(Shape.TextBlockRaw(line.toList, spaces, off)) - } - - ///////////////// - //// Segment //// - ///////////////// - type Segment[T] = Shape.Segment[T] - object Segment { - - val Escape = org.enso.syntax.text.ast.text.Escape - type Escape = org.enso.syntax.text.ast.text.Escape - type RawEscape = org.enso.syntax.text.ast.text.RawEscape - - //// Definition //// - - val EscT = Shape.SegmentEscape - type EscT = Shape.SegmentEscape[AST] - val RawEscT = Shape.SegmentRawEscape - type RawEscT = Shape.SegmentRawEscape[AST] - val Fmt = Shape.SegmentFmt - type Fmt = Shape.SegmentFmt[AST] - val Raw = Shape.SegmentRaw - type Raw = Shape.SegmentRaw[AST] - - object Esc { - def apply(code: Escape): EscT = Shape.SegmentEscape(code) - def unapply(shape: EscT): Option[Escape] = - Shape.SegmentEscape.unapply(shape) - } - object RawEsc { - def apply(code: RawEscape): RawEscT = Shape.SegmentRawEscape(code) - def unapply(shape: RawEscT): Option[RawEscape] = - Shape.SegmentRawEscape.unapply(shape) - } - object Expr { - def apply(t: Option[AST]): Fmt = Shape.SegmentExpr(t) - def unapply(shape: Shape.SegmentExpr[AST]): Option[Option[AST]] = - Shape.SegmentExpr.unapply(shape) - } - object Plain { - def apply(s: String): Raw = Shape.SegmentPlain(s) - def unapply(shape: Shape.SegmentPlain[AST]): Option[String] = - Shape.SegmentPlain.unapply(shape) - } - } - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// App ///////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - //// Definition //// - - type App = ASTOf[Shape.App] - object App { - val any = UnapplyByType[App] - - //// Constructors //// - - type Prefix = ASTOf[Shape.Prefix] - type Infix = ASTOf[Shape.Infix] - - //// Smart Constructors //// - - object Prefix { - val any = UnapplyByType[Prefix] - def unapply(t: AST) = Unapply[Prefix].run(t => (t.func, t.arg))(t) - def apply(fn: AST, off: Int, arg: AST): Prefix = - Shape.Prefix(fn, off, arg) - def apply(fn: AST, arg: AST): Prefix = Prefix(fn, 1, arg) - } - - object Infix { - val any = UnapplyByType[Infix] - def unapply(t: AST) = Unapply[Infix].run(t => (t.larg, t.opr, t.rarg))(t) - def apply(larg: AST, loff: Int, opr: Opr, roff: Int, rarg: AST): Infix = - Shape.Infix(larg, loff, opr, roff, rarg) - def apply(larg: AST, loff: Int, opr: Opr, rarg: AST): Infix = - Infix(larg, loff, opr, 1, rarg) - def apply(larg: AST, opr: Opr, roff: Int, rarg: AST): Infix = - Infix(larg, 1, opr, roff, rarg) - def apply(larg: AST, opr: Opr, rarg: AST): Infix = - Infix(larg, 1, opr, 1, rarg) - } - - ///////////////// - //// Section //// - ///////////////// - - //// Reexports //// - - type Left = Section.Left - type Right = Section.Right - type Sides = Section.Sides - - val Left = Section.Left - val Right = Section.Right - val Sides = Section.Sides - - //// Definition //// - - type Section = ASTOf[Shape.Section] - object Section { - - val any = UnapplyByType[Section] - - //// Constructors //// - - type Left = ASTOf[Shape.SectionLeft] - type Right = ASTOf[Shape.SectionRight] - type Sides = ASTOf[Shape.SectionSides] - - //// Smart Constructors //// - - object Left { - val any = UnapplyByType[Left] - def unapply(t: AST) = Unapply[Left].run(t => (t.arg, t.opr))(t) - - def apply(arg: AST, off: Int, opr: Opr): Left = - Shape.SectionLeft(arg, off, opr) - def apply(arg: AST, opr: Opr): Left = Left(arg, 1, opr) - } - object Right { - val any = UnapplyByType[Right] - def unapply(t: AST) = Unapply[Right].run(t => (t.opr, t.arg))(t) - - def apply(opr: Opr, off: Int, arg: AST): Right = - Shape.SectionRight(opr, off, arg) - def apply(opr: Opr, arg: AST): Right = Right(opr, 1, arg) - } - object Sides { - val any = UnapplyByType[Sides] - def unapply(t: AST) = Unapply[Sides].run(_.opr)(t) - def apply(opr: Opr): Sides = Shape.SectionSides[AST](opr) - } - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Block /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Block = ASTOf[Shape.Block] - - object Block { - type Type = Shape.Block.Type - val Continuous = Shape.Block.Continuous - val Discontinuous = Shape.Block.Discontinuous - - //// Smart Constructors //// - - // FIXME: Compatibility mode - def apply( - @unused isOrphan: Boolean, - typ: Type, - indent: Int, - emptyLines: List[Int], - firstLine: Line, - lines: List[OptLine] - ): Block = { - Shape.Block(typ, indent, emptyLines, firstLine, lines, isOrphan) - } - - def apply( - typ: Type, - indent: Int, - emptyLines: List[Int], - firstLine: Line, - lines: List[OptLine] - ): Block = Shape.Block(typ, indent, emptyLines, firstLine, lines) - - def apply( - indent: Int, - firstLine: AST, - lines: AST* - ): Block = - Block( - Continuous, - indent, - List(), - Line(firstLine), - lines.toList.map(ast => Line(Some(ast))) - ) - - val any = UnapplyByType[Block] - def unapply(t: AST) = - Unapply[Block].run(t => (t.ty, t.indent, t.firstLine, t.lines))(t) - - //// Line //// - - type Line = Shape.Block.Line[AST] - type OptLine = Shape.Block.OptLine[AST] - object Line { - // FIXME: Compatibility mode - type NonEmpty = Line - val Required = Line - def apply[T](elem: T, off: Int) = Shape.Block.Line(elem, off) - def apply[T](elem: T) = Shape.Block.Line(elem, 0) - } - object OptLine { - def apply(): OptLine = Line(None, 0) - def apply(elem: AST): OptLine = Line(Some(elem)) - def apply(off: Int): OptLine = Line(None, off) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Module ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Module = ASTOf[Shape.Module] - - object Module { - import Block._ - type M = Module - val any = UnapplyByType[M] - def unapply(t: AST) = Unapply[M].run(_.lines)(t) - def apply(ls: List1[OptLine]): M = Shape.Module(ls) - def apply(l: OptLine): M = Module(List1(l)) - def apply(l: OptLine, ls: OptLine*): M = Module(List1(l, ls.toList)) - def apply(l: OptLine, ls: List[OptLine]): M = Module(List1(l, ls)) - def traverseWithOff(m: M)(f: (Index, AST) => AST): M = { - val lines2 = m.lines.map { line: OptLine => - // FIXME: Why line.map does not work? - Shape.Block.Line.ftor.map(line)(_.map(_.traverseWithOff(f))) - } - m.shape.copy(lines = lines2) - } - } - - //////////////////////////////////////////////////////////////////////////// - //// Macro /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Macro = ASTOf[Shape.Macro] - object Macro { - - //// Matched //// - - type Match = ASTOf[Shape.Match] - - object Match { - val any = UnapplyByType[Match] - def apply( - pfx: Option[Pattern.Match], - segs: Shifted.List1[Match.Segment], - resolved: Option[AST] - ): Match = Shape.Match[AST](pfx, segs, resolved) - - type Segment = Shape.Match.Segment[AST] - val Segment = Shape.Match.Segment - implicit class SegmentOps(t: Segment) { - def toStream: AST.Stream = Shifted(t.head) :: t.body.toStream - } - - } - - //// Ambiguous //// - - type Ambiguous = ASTOf[Shape.Ambiguous] - object Ambiguous { - type Segment = Shape.Ambiguous.Segment - val Segment = Shape.Ambiguous.Segment - def apply( - segs: Shifted.List1[Segment], - paths: Tree[AST, Unit] - ): Ambiguous = ASTOf.wrap(Shape.Ambiguous(segs, paths)) - - def unapply(t: AST) = - Unapply[Ambiguous].run(t => (t.segs, t.paths))(t) - } - - //// Resolver //// - - type Resolver = Resolver.Context => AST - object Resolver { - type Context = ContextOf[AST] - final case class ContextOf[T]( - prefix: Option[Pattern.Match], - body: List[Shape.Match.Segment[T]], - id: ID - ) - object Context { - def apply( - prefix: Option[Pattern.Match], - body: List[Macro.Match.Segment], - id: ID - ): Context = ContextOf(prefix, body, id) - } - } - - //// Definition //// - - type Definition = __Definition__ - final case class __Definition__( - back: Option[Pattern], - init: List[Definition.Segment], - last: Definition.LastSegment, - resolver: Resolver - ) { - def path: List1[AST] = init.map(_.head) +: List1(last.head) - def fwdPats: List1[Pattern] = - init.map(_.pattern) +: List1(last.pattern.getOrElse(Pattern.Nothing())) - } - object Definition { - import Pattern._ - - final case class Segment(head: AST, pattern: Pattern) { - def map(f: Pattern => Pattern): Segment = copy(pattern = f(pattern)) - } - object Segment { - type Tup = (AST, Pattern) - def apply(t: Tup): Segment = Segment(t._1, t._2) - } - - final case class LastSegment(head: AST, pattern: Option[Pattern]) { - def map(f: Pattern => Pattern): LastSegment = - copy(pattern = pattern.map(f)) - } - object LastSegment { - type Tup = (AST, Option[Pattern]) - def apply(t: Tup): LastSegment = LastSegment(t._1, t._2) - } - - def apply( - precSection: Option[Pattern], - t1: Segment.Tup, - ts: List[Segment.Tup] - )( - fin: Resolver - ): Definition = { - val segs = List1(t1, ts) - val init = segs.init - val lastTup = segs.last - val last = (lastTup._1, Some(lastTup._2)) - Definition(precSection, init, last, fin) - } - - def apply( - precSection: Option[Pattern], - t1: Segment.Tup, - ts: Segment.Tup* - )( - fin: Resolver - ): Definition = Definition(precSection, t1, ts.toList)(fin) - - def apply(t1: Segment.Tup, t2_ : Segment.Tup*)( - fin: Resolver - ): Definition = Definition(None, t1, t2_.toList)(fin) - - def apply(initTups: List[Segment.Tup], lastHead: AST)( - fin: Resolver - ): Definition = - Definition(None, initTups, (lastHead, None), fin) - - def apply(t1: Segment.Tup, last: AST)(fin: Resolver): Definition = - Definition(List(t1), last)(fin) - - def apply( - back: Option[Pattern], - initTups: List[Segment.Tup], - lastTup: LastSegment.Tup, - resolver: Resolver - ): Definition = { - type PP = Pattern => Pattern - val applyValidChecker: PP = _ | ErrTillEnd("unmatched pattern") - val applyFullChecker: PP = _ :: ErrUnmatched("unmatched tokens") - val applyDummyFullChecker: PP = _ :: Nothing() - - val unapplyValidChecker: Pattern.Match => Pattern.Match = { - case Pattern.Match.Or(_, Left(tgt)) => tgt - case _ => - throw new Error("Internal error") - } - val unapplyFullChecker: Pattern.Match => Pattern.Match = { - case Pattern.Match.Seq(_, (tgt, _)) => tgt - case _ => - throw new Error("Internal error") - } - val applySegInitCheckers: List[Segment] => List[Segment] = - _.map(_.map(p => applyFullChecker(applyValidChecker(p)))) - - val applySegLastCheckers: LastSegment => LastSegment = - _.map(p => applyDummyFullChecker(applyValidChecker(p))) - - val unapplySegCheckers - : List[AST.Macro.Match.Segment] => List[AST.Macro.Match.Segment] = - _.map(_.map({ - case m @ Pattern.Match.Nothing(_) => m - case m => - unapplyValidChecker(unapplyFullChecker(m)) - })) - - val initSegs = initTups.map(Segment(_)) - val lastSeg = LastSegment(lastTup) - val backPatWithCheck = back.map(applyValidChecker) - val initSegsWithChecks = applySegInitCheckers(initSegs) - val lastSegWithChecks = applySegLastCheckers(lastSeg) - - def unexpected(ctx: Resolver.Context, msg: String): AST = { - import AST.Macro.Match.SegmentOps - val pfxStream = ctx.prefix.map(_.toStream).getOrElse(List()) - val segsStream = ctx.body.flatMap(_.toStream) - val stream = pfxStream ++ segsStream - AST.Invalid.Unexpected(msg, stream) - } - - def resolverWithChecks(ctx: Resolver.Context) = { - val pfxFail = !ctx.prefix.forall(_.isValid) - val segsFail = !ctx.body.forall(_.isValid) - if (pfxFail || segsFail) unexpected(ctx, "invalid statement") - else { - try { - val ctx2 = ctx.copy( - prefix = ctx.prefix.map(unapplyValidChecker), - body = unapplySegCheckers(ctx.body) - ) - resolver(ctx2) - } catch { - case _: Throwable => - val tokens = ctx.body.flatMap(mat => { - val firstElem = Shifted(mat.head) - val rest = mat.body.toStream - firstElem :: rest - }) - AST.Invalid.Unexpected("Unexpected tokens", tokens) - } - } - } - __Definition__( - backPatWithCheck, - initSegsWithChecks, - lastSegWithChecks, - resolverWithChecks - ) - } - - } - } - - ////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - //// Space-Unaware AST /////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - sealed trait SpacelessASTOf[T] extends Shape[T] - - ////////////////////////////////////////////////////////////////////////////// - /// Comment ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Comment = ASTOf[Shape.Comment] - object Comment { - val any = UnapplyByType[Comment] - def apply(lines: List[String]): Comment = - Shape.Comment(lines): Shape.Comment[AST] - def unapply(t: AST): Option[List[String]] = - Unapply[Comment].run(t => t.lines)(t) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Documented ////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Documented = ASTOf[Shape.Documented] - - object Documented { - val any = UnapplyByType[Documented] - def apply(doc: Doc, emp: Int, ast: AST): Documented = - Shape.Documented(doc, emp, ast) - def unapply(t: AST): Option[(Doc, Int, AST)] = - Unapply[Documented].run(t => (t.doc, t.emptyLinesBetween, t.ast))(t) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Import ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Import = ASTOf[Shape.Import] - - object Import { - def apply( - path: List1[AST.Ident], - rename: Option[AST.Ident.Cons], - isAll: Boolean, - onlyNames: Option[List1[AST.Ident]], - hidingNames: Option[List1[AST.Ident]] - ): Import = - Shape.Import[AST](path, rename, isAll, onlyNames, hidingNames) - def unapply(t: AST): Option[ - ( - List1[AST.Ident], - Option[AST.Ident.Cons], - Boolean, - Option[List1[AST.Ident]], - Option[List1[AST.Ident]] - ) - ] = - Unapply[Import].run(t => - (t.path, t.rename, t.isAll, t.onlyNames, t.hidingNames) - )(t) - val any = UnapplyByType[Import] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Export ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Export = ASTOf[Shape.Export] - - object Export { - def apply( - path: List1[AST.Ident], - rename: Option[AST.Ident.Cons], - isAll: Boolean, - onlyNames: Option[List1[AST.Ident]], - hidingNames: Option[List1[AST.Ident]] - ): Export = - Shape.Export[AST](path, rename, isAll, onlyNames, hidingNames) - def unapply(t: AST): Option[ - ( - List1[AST.Ident], - Option[AST.Ident.Cons], - Boolean, - Option[List1[AST.Ident]], - Option[List1[AST.Ident]] - ) - ] = - Unapply[Export].run(t => - (t.path, t.rename, t.isAll, t.onlyNames, t.hidingNames) - )(t) - val any = UnapplyByType[Export] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Java Import ///////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type JavaImport = ASTOf[Shape.JavaImport] - - object JavaImport { - def apply(path: List1[Ident], rename: Option[Ident.Cons]): JavaImport = - Shape.JavaImport[AST](path, rename) - def unapply(t: AST): Option[List1[Ident]] = - Unapply[JavaImport].run(t => t.path)(t) - val any = UnapplyByType[JavaImport] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Mixfix ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Mixfix = ASTOf[Shape.Mixfix] - - object Mixfix { - def apply(name: List1[Ident], args: List1[AST]): Mixfix = - Shape.Mixfix(name, args) - def unapply(t: AST) = Unapply[Mixfix].run(t => (t.name, t.args))(t) - val any = UnapplyByType[Mixfix] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Group /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Group = ASTOf[Shape.Group] - object Group { - val any = UnapplyByType[Group] - def unapply(t: AST): Option[Option[AST]] = Unapply[Group].run(_.body)(t) - def apply(body: Option[AST]): Group = Shape.Group(body) - def apply(body: AST): Group = Group(Some(body)) - def apply(body: SAST): Group = Group(body.wrapped) - def apply(): Group = Group(None) - } - - ////////////////////////////////////////////////////////////////////////////// - //// SequenceLiteral ///////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type SequenceLiteral = ASTOf[Shape.SequenceLiteral] - object SequenceLiteral { - val any = UnapplyByType[SequenceLiteral] - def unapply(t: AST): Option[List[AST]] = - Unapply[SequenceLiteral].run(_.items)(t) - def apply(items: List[AST]): SequenceLiteral = Shape.SequenceLiteral(items) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Typeset Literal ///////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type TypesetLiteral = ASTOf[Shape.TypesetLiteral] - object TypesetLiteral { - val any = UnapplyByType[TypesetLiteral] - def unapply(t: AST): Option[Option[AST]] = - Unapply[TypesetLiteral].run(_.expression)(t) - def apply(expression: Option[AST]): TypesetLiteral = - Shape.TypesetLiteral(expression) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Def ///////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Def = ASTOf[Shape.Def] - object Def { - val any = UnapplyByType[Def] - def apply(name: Cons): Def = Def(name, List()) - def apply(name: Cons, args: List[AST]): Def = Def(name, args, None) - def apply(name: Cons, args: List[AST], body: Option[AST]): Def = - Shape.Def(name, args, body) - def unapply(t: AST): Option[(Cons, List[AST], Option[AST])] = - Unapply[Def].run(t => (t.name, t.args, t.body))(t) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Foreign ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Foreign = ASTOf[Shape.Foreign] - object Foreign { - def apply(indent: Int, lang: String, code: List[String]): Foreign = - Shape.Foreign(indent, lang, code): Shape.Foreign[AST] - def unapply(t: AST): Option[(Int, String, List[String])] = - Unapply[Foreign].run(t => (t.indent, t.lang, t.code))(t) - val any = UnapplyByType[Foreign] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Modified //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - type Modified = ASTOf[Shape.Modified] - object Modified { - def apply(modifier: String, definition: AST): Modified = { - Shape.Modified(modifier, definition) - } - def unapply(t: AST): Option[(String, AST)] = { - Unapply[Modified].run(t => (t.modifier, t.definition))(t) - } - val any = UnapplyByType[Modified] - } - - ////////////////////////////////////////////////////////////////////////////// - //// Main //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - def main(): Unit = { - val v1 = Ident.Var("foo") - // val v1_ = v1: AST - - println(v1.span) - println((v1: AST.Ident).span) - println(v1.span) - - // println(v1_.asJson) - // val opr1 = Ident.Opr("+") - val v2 = App.Prefix(Ident.Var("x"), 10, Ident.Var("z")) - // val v2_ = v2: AST - // - // println(v2.asJson) - // println(v2_.asJson) - println(OffsetZip(v2.shape)) - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/DSL.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/DSL.scala deleted file mode 100644 index 8b97aee526de..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/DSL.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.enso.syntax.text.ast - -import org.enso.syntax.text.AST - -object DSL { - import AST.conversions._ - - implicit final class ASTHelper(self: AST) { - private def smartApp(off: Int, r: AST): AST = self match { - case AST.App.Section.Left.any(t) => - AST.App.Infix(t.arg, t.off, t.opr, off, r) - case _ => smartAppRaw(off, r) - } - - private def smartAppRaw(off: Int, r: AST): AST = (self, r) match { - case (l, AST.Opr.any(r)) => AST.App.Left(l, off, r) - case (AST.Opr.any(l), r) => AST.App.Right(l, off, r) - case (l, r) => AST.App.Prefix(l, off, r) - } - - def $(t: AST) = smartApp(0, t) - def $_(t: AST) = smartApp(1, t) - def $__(t: AST) = smartApp(2, t) - def $___(t: AST) = smartApp(3, t) - - def $$(t: AST) = smartAppRaw(0, t) - def $$_(t: AST) = smartAppRaw(1, t) - def $$__(t: AST) = smartAppRaw(2, t) - def $$___(t: AST) = smartAppRaw(3, t) - } - - implicit final class StringHelpers(self: String) { - def $(t: AST) = (self: AST) $ t - def $_(t: AST) = (self: AST) $_ t - def $__(t: AST) = (self: AST) $__ t - def $___(t: AST) = (self: AST) $___ t - - def $$(t: AST) = (self: AST) $$ t - def $$_(t: AST) = (self: AST) $$_ t - def $$__(t: AST) = (self: AST) $$__ t - def $$___(t: AST) = (self: AST) $$___ t - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Doc.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Doc.scala deleted file mode 100644 index 09794092444c..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Doc.scala +++ /dev/null @@ -1,748 +0,0 @@ -package org.enso.syntax.text.ast - -import org.enso.data.List1 -import org.enso.flexer.ADT -import org.enso.syntax.text.ast.Repr.R -import scalatags.Text.all._ -import scalatags.Text.TypedTag -import scalatags.Text.{all => HTML} -import scalatags.generic -import scalatags.text.Builder - -import scala.util.Random - -//////////////////////////////////////////////////////////////////////////////// -//// Doc /////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -/** Doc - The highest level container, the output of Doc Parser - * - * Doc can be made of up to 3 elements: - * - * @param tags - If exists, holds [[Doc#Tags]] to documented text - * @param synopsis - If exists, holds [[Doc#Synopsis]] of documented text - * @param body - If exists, holds [[Doc#Body]] of documented text - */ -final case class Doc( - tags: Option[Doc.Tags], - synopsis: Option[Doc.Synopsis], - body: Option[Doc.Body] -) extends Doc.Symbol { - val repr: Repr.Builder = R + tags + synopsis + body - val htmlWoTags: Doc.HTML = Seq( - HTML.div(synopsis.html)(body.html) - ) - val htmlWoTagsMain: Doc.HTML = synopsis match { - case Some(s) => Seq(HTML.div(s.htmlBig)(body.html)) - case None => Seq(HTML.div(body.html)) - } - val html: Doc.HTML = Seq( - HTML.div(tags.html)(synopsis.html)(body.html) - ) - - def htmlWithTitle(title: String): Doc.HTML = { - if (title != "") { - Seq( - HTML.div( - HTML.div(HTML.`class` := "doc-title-container")( - HTML.div(HTML.`class` := "doc-title-name")(title), - tags.html - ) - )(synopsis.html)(body.html) - ) - } else { - Seq(HTML.div(tags.html)(synopsis.html)(body.html)) - } - } -} - -object Doc { - def apply(): Doc = Doc(None, None, None) - def apply(tags: Tags): Doc = Doc(Some(tags), None, None) - def apply(synopsis: Synopsis): Doc = - Doc(None, Some(synopsis), None) - def apply(synopsis: Synopsis, body: Body): Doc = - Doc(None, Some(synopsis), Some(body)) - def apply(tags: Tags, synopsis: Synopsis): Doc = - Doc(Some(tags), Some(synopsis), None) - def apply(tags: Tags, synopsis: Synopsis, body: Body): Doc = - Doc(Some(tags), Some(synopsis), Some(body)) - - type HTML = Seq[Modifier] - type HTMLTag = TypedTag[String] - - ////////////////////////////////////////////////////////////////////////////// - //// Symbol ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Symbol - the most low-level element, on top of which every other element - * is built - * - * It extends Repr.Provider, so it also contain repr method, as well as - * span and show values. In addition to that it specifies html method for - * extending tokens and getting HTML file out of Doc Parser - */ - sealed trait Symbol extends Repr.Provider { - def show(): String = repr.build() - def html: HTML - - def htmlCls(): generic.AttrPair[Builder, String] = - HTML.`class` := getClass.toString.split('$').last.split('.').last - } - - implicit final class ExtForSymbol[T <: Symbol](val self: Option[T]) { - val dummyText = Elem.Text("") - val html: HTML = self.getOrElse(dummyText).html - } - - ////////////////////////////////////////////////////////////////////////////// - //// Elem //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Elem - the trait for proper element of Doc, which elements can be used in - * higher level elements - * Invalid - trait for invalid element of Doc, which elements can be used in - * higher level elements - */ - sealed trait Elem extends Symbol - object Elem { - sealed trait Invalid extends Elem - - //////////////////////////////////////////////////////////////////////////// - //// Normal text & Newline ///////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////// - - /** Text - used to hold normal string as Elem - * Newline - used to hold newline ('\n') as elem - */ - final case class Text(text: String) extends Elem { - val repr: Repr.Builder = text - val html: HTML = Seq(text) - } - - implicit def stringToText(str: String): Elem.Text = Elem.Text(str) - - case object Newline extends Elem { - val repr: Repr.Builder = R + "\n" - val html: HTML = Seq(" ") - } - - //////////////////////////////////////////////////////////////////////////// - //// Text Formatter - Bold, Italic, Strikeout ////////////////////////////// - //////////////////////////////////////////////////////////////////////////// - - /** Formatter - element used to hold formatted text - * - * @param typ - specifies type of formatter (Bold, Italic, Strikeout) - * @param elems - elems which make up formatter - */ - final case class Formatter(typ: Formatter.Type, elems: scala.List[Elem]) - extends Elem { - val repr: Repr.Builder = R + typ.marker + elems + typ.marker - val html: HTML = Seq(typ.htmlMarker(elems.html)) - } - - object Formatter { - def apply(typ: Type): Formatter = Formatter(typ, Nil) - def apply(typ: Type, elem: Elem): Formatter = - Formatter(typ, elem :: Nil) - def apply(typ: Type, elems: Elem*): Formatter = - Formatter(typ, elems.toList) - - abstract class Type(val marker: Char, val htmlMarker: HTMLTag) - case object Bold extends Type('*', HTML.b) - case object Italic extends Type('_', HTML.i) - case object Strikeout extends Type('~', HTML.s) - - /** Unclosed - Invalid formatter made by parser if user has invoked - * formatter but hasn't ended it - * - * @param typ - specifies type of formatter (Bold, Italic, Strikeout) - * @param elems - elems which make up unclosed formatter - */ - final case class Unclosed(typ: Type, elems: scala.List[Elem]) - extends Elem.Invalid { - val repr: Repr.Builder = R + typ.marker + elems - val html: HTML = Seq(HTML.div(htmlCls())(typ.htmlMarker(elems.html))) - } - - object Unclosed { - def apply(typ: Type): Unclosed = Unclosed(typ, Nil) - def apply(typ: Type, elem: Elem): Unclosed = Unclosed(typ, elem :: Nil) - def apply(typ: Type, elems: Elem*): Unclosed = - Unclosed(typ, elems.toList) - } - } - - implicit final class ExtForListOfElem(val self: scala.List[Elem]) - extends Symbol { - val repr: Repr.Builder = R + self.map(_.repr) - val html: HTML = Seq(self.map(_.html)) - } - - //////////////////////////////////////////////////////////////////////////// - //// Code ////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////// - - /** Code - block used to hold lines of code in Documentation - * - * @param elems - lines of code - */ - final case class CodeBlock(elems: List1[CodeBlock.Line]) extends Elem { - val newLn: Elem = Elem.Newline - val repr: Repr.Builder = R + elems.head + elems.tail.map(R + newLn + _) - val ran: Random = new Random() - val html: HTML = { - val uniqueIDCode = ran.alphanumeric.take(8).mkString("") - val uniqueIDBtn = ran.alphanumeric.take(8).mkString("") - val htmlIdCode = HTML.`id` := uniqueIDCode - val htmlIdBtn = HTML.`id` := uniqueIDBtn - val firstIndent = elems.head.indent - val elemsHTML = elems.toList.map(_.htmlOffset(firstIndent)) - val copyClass = HTML.`class` := "doc-copy-btn flex" - val codeClass = HTML.`class` := "doc-code-container" - val copyBtn = HTML.button(htmlIdBtn)(copyClass)("Copy") - Seq( - HTML.div( - HTML.div(codeClass)(htmlIdCode)(HTML.pre(elemsHTML)), - copyBtn - ) - ) - } - } - object CodeBlock { - def apply(elem: CodeBlock.Line): CodeBlock = CodeBlock(List1(elem)) - def apply(elems: CodeBlock.Line*): CodeBlock = - CodeBlock( - List1(elems.head, elems.tail.toList) - ) - - /** Inline - line of code which is in line with other elements - * Line - elem which is a part of Code Block - */ - final case class Inline(str: String) extends Elem { - val marker = '`' - val repr: Repr.Builder = R + marker + str + marker - val html: HTML = Seq(HTML.code(str)) - } - final case class Line(indent: Int, elem: String) extends Elem { - val repr: Repr.Builder = R + indent + elem - val html: HTML = Seq(HTML.code(" " * indent + elem), HTML.br) - def htmlOffset(off: Int): HTML = - Seq(HTML.code(" " * (indent - off) + elem), HTML.br) - } - } - - //////////////////////////////////////////////////////////////////////////// - //// Link - URL & Image //////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////// - - /** Link - element used to hold links - * - * @param name - specifies where does the link take us - * @param url - specifies address - * - * there are two kinds of links - normal URL and Image embedded in text - * - * Link.Invalid - something that couldn't be pattern matched to create link - */ - abstract class Link(name: String, url: String, val marker: Option[String]) - extends Elem { - val repr: Repr.Builder = R + marker + "[" + name + "](" + url + ")" - val html: HTML = this match { - case _: Link.URL => Seq(HTML.a(HTML.href := url)(name)) - case _: Link.Image => Seq(HTML.img(HTML.src := url), name) - } - } - - object Link { - final case class URL(name: String, url: String) - extends Link(name, url, None) - object URL { - def apply(): URL = URL("", "") - } - - final case class Image(name: String, url: String) - extends Link(name, url, Some("!")) - object Image { - def apply(): Image = Image("", "") - } - - final case class Invalid(elem: String) extends Elem { - val repr: Repr.Builder = R + elem - val html: HTML = { - val htmlClass = HTML.`class` := this.productPrefix + getObjectName - Seq(HTML.div(htmlClass)(elem.html)) - } - } - - def getObjectName: String = { - getClass.toString.split('$').last - } - } - - //////////////////////////////////////////////////////////////////////////// - //// List - Ordered & Unordered, Invalid Indent //////////////////////////// - //////////////////////////////////////////////////////////////////////////// - - /** An element of the list. - * - * The list can contain following elements: - * - [[ListItem]] a plain list element - * - [[List]] a sublist - * - [[MisalignedItem]] a list item that is not aligned correctly with the - * previous list item - */ - sealed trait ListElem extends Elem { - - /** Append elements to this list item. - * - * @param xs elements to append - */ - def append(xs: scala.List[Elem]): ListElem - } - - /** A list item that can hold a complex element structure. - * - * @param elems the elements that make up this list item - */ - final case class ListItem(elems: scala.List[Elem]) extends ListElem { - - override val repr: Repr.Builder = R + elems - - override def html: HTML = elems.map(_.html) - - /** @inheritdoc */ - override def append(xs: scala.List[Elem]): ListItem = - copy(elems = elems :++ xs) - } - object ListItem { - - /** Create a list item from the provided elemenets - * - * @param elems the elements that make up the list item - * @return the list item - */ - def apply(elems: Elem*): ListItem = - ListItem(elems.toList) - } - - /** The list item that is not aligned correctly with the previous list item. - * - * @param indent the indentation of this list item - * @param typ the list type - * @param elems the elements that make up this list item - */ - final case class MisalignedItem( - indent: Int, - typ: List.Type, - elems: scala.List[Elem] - ) extends ListElem { - - override val repr: Repr.Builder = - R + indent + typ.marker + List.ElemIndent + elems - - override def html: HTML = - elems.map(_.html) - - /** @inheritdoc */ - override def append(xs: scala.List[Elem]): MisalignedItem = - copy(elems = elems :++ xs) - } - object MisalignedItem { - - /** Create a misaligned item from the provided elements. - * - * @param indent the indentation of this list item - * @param typ the list type - * @param elems the elements that make up the list item - * @return the new misaligned item - */ - def apply(indent: Int, typ: List.Type, elems: Elem*): MisalignedItem = - new MisalignedItem(indent, typ, elems.toList) - } - - /** List - block used to hold ordered and unordered lists - * - * Indent.Invalid - holds list element with invalid indent - * - * @param indent specifies indentation of list - * @param typ the list type - * @param elems the elements that make up this list - */ - final case class List(indent: Int, typ: List.Type, elems: List1[ListElem]) - extends ListElem { - - val repr: Repr.Builder = { - val listElems = elems.reverse - R + indent + typ.marker + List.ElemIndent + listElems.head + - listElems.tail.map { - case elem: List => - R + Newline + elem - case elem: ListItem => - R + Newline + indent + typ.marker + List.ElemIndent + elem - case elem: MisalignedItem => - R + Newline + elem - } - } - - val html: HTML = { - val elemsHTML = elems.reverse.toList.map { - case elem: List => elem.html - case elem: ListElem => Seq(HTML.li(elem.html)) - } - Seq(typ.HTMLMarker(elemsHTML)) - } - - /** @inheritdoc */ - override def append(xs: scala.List[Elem]): List = { - val newElems = List1(elems.head.append(xs), elems.tail) - this.copy(elems = newElems) - } - - /** Add a new list item. - * - * @param item the list item to add - * @return the new list with this item added - */ - def addItem(item: ListElem): List = { - val newElems = elems.prepend(item) - this.copy(elems = newElems) - } - - /** Add an empty list item. - * - * @return the new list with an empty list item added - */ - def addItem(): List = - this.copy(elems = elems.prepend(ListItem(Nil))) - } - - object List { - - val ElemIndent: Int = 1 - - /** Create an empty list. - * - * @param indent the list indentation - * @param typ the list type - * @return the new list - */ - def empty(indent: Int, typ: Type): List = - new List(indent, typ, List1(ListItem(Nil))) - - /** Create a new list. - * - * @param indent the list indentation - * @param typ the list type - * @param elem the first elements of this list - * @param elems the rest of the list elements - * @return the new list - */ - def apply(indent: Int, typ: Type, elem: Elem, elems: Elem*): List = { - val listItems = (elem :: elems.toList).reverse.map { - case list: List => list - case elem: ListItem => elem - case elem: MisalignedItem => elem - case elem => ListItem(elem) - } - new List( - indent, - typ, - List1.fromListOption(listItems).get - ) - } - - /** The list type. */ - abstract class Type(val marker: Char, val HTMLMarker: HTMLTag) - final case object Unordered extends Type('-', HTML.ul) - final case object Ordered extends Type('*', HTML.ol) - - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Sections - Raw & Marked ///////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Section - block used to hold one section of text - * - * indent - specifies indentation of section - * elems - elements which make up section - * - * Marked - Section which is marked as Important, Info or Example - * Raw - normal, unmarked block of text - */ - sealed trait Section extends Symbol { - def indent: Int - def elems: List[Elem] - - def reprOfNormalText(elem: Elem, prevElem: Elem): Repr.Builder = { - prevElem match { - case Elem.Newline => R + indent + elem - case _ => R + elem - } - } - - val html: HTML = Seq(HTML.div(htmlCls())(elems.map(_.html))) - } - - object Section { - - /** Header - element used to hold header for section - * - * @param elems - elements which make up header - */ - final case class Header(elems: List[Elem]) extends Elem { - val repr: Repr.Builder = R + elems.map(_.repr) - val html: HTML = Seq( - HTML.div(HTML.`class` := "summary")( - elems.map(_.html) - ) - ) - } - object Header { - def apply(elem: Elem): Header = Header(elem :: Nil) - def apply(elems: Elem*): Header = Header(elems.toList) - } - - final case class Marked( - indentBeforeMarker: Int, - indentAfterMarker: Int, - typ: Marked.Type, - elems: List[Elem] - ) extends Section { - val marker: String = typ.marker.toString - val firstIndentRepr: Repr.Builder = - R + indentBeforeMarker + marker + indentAfterMarker - - val dummyElem = Elem.Text("") - val elemsRepr: List[Repr.Builder] = elems.zip(dummyElem :: elems).map { - case (elem @ (_: Elem.List), _) => R + elem - case (elem @ (_: Elem.CodeBlock), _) => R + elem - case (elem, prevElem) => reprOfNormalText(elem, prevElem) - } - - val repr: Repr.Builder = R + firstIndentRepr + elemsRepr - override def htmlCls(): generic.AttrPair[Builder, String] = { - HTML.`class` := typ.toString.toLowerCase - } - - override def indent: Int = - indentBeforeMarker + marker.length + indentAfterMarker - } - - object Marked { - def apply( - indentBeforeMarker: Int, - indentAfterMarker: Int, - typ: Type - ): Marked = Marked(indentBeforeMarker, indentAfterMarker, typ, Nil) - def apply( - indentBeforeMarker: Int, - indentAfterMarker: Int, - typ: Type, - elem: Elem - ): Marked = - Marked(indentBeforeMarker, indentAfterMarker, typ, elem :: Nil) - def apply( - indentBeforeMarker: Int, - indentAfterMarker: Int, - typ: Type, - elems: Elem* - ): Marked = - Marked(indentBeforeMarker, indentAfterMarker, typ, elems.toList) - val defaultIndent = 0 - def apply(typ: Type): Marked = - Marked(defaultIndent, defaultIndent, typ, Nil) - def apply(typ: Type, elem: Elem): Marked = - Marked(defaultIndent, defaultIndent, typ, elem :: Nil) - def apply(typ: Type, elems: Elem*): Marked = - Marked(defaultIndent, defaultIndent, typ, elems.toList) - - abstract class Type(val marker: Char) - case object Important extends Type('!') - case object Info extends Type('?') - case object Example extends Type('>') - } - - final case class Raw(indent: Int, elems: List[Elem]) extends Section { - val dummyElem = Elem.Text("") - val newLn: Elem = Elem.Newline - val elemsRepr: List[Repr.Builder] = elems.zip(dummyElem :: elems).map { - case (elem @ (_: Section.Header), _) => R + newLn + indent + elem - case (elem @ (_: Elem.List), _) => R + elem - case (elem @ (_: Elem.CodeBlock), _) => R + elem - case (elem, prevElem) => reprOfNormalText(elem, prevElem) - } - - val repr: Repr.Builder = R + indent + elemsRepr - - override def htmlCls(): generic.AttrPair[Builder, String] = { - HTML.`class` := "raw" - } - - override val html: HTML = Seq(HTML.p(elems.map(_.html))) - } - - object Raw { - def apply(indent: Int): Raw = Raw(indent, Nil) - def apply(indent: Int, elem: Elem): Raw = Raw(indent, elem :: Nil) - def apply(indent: Int, elems: Elem*): Raw = Raw(indent, elems.toList) - val defaultIndent = 0 - def apply(): Raw = Raw(defaultIndent, Nil) - def apply(elem: Elem): Raw = Raw(defaultIndent, elem :: Nil) - def apply(elems: Elem*): Raw = Raw(defaultIndent, elems.toList) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Synopsis //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Synopsis - block used to hold section as a synopsis of documentation - * - * @param elems - sections which make up synopsis - */ - final case class Synopsis(elems: List1[Section]) extends Symbol { - val newLn: Elem = Elem.Newline - val repr: Repr.Builder = R + elems.head + elems.tail.map(R + newLn + _) - val html: HTML = { - Seq( - HTML.div(HTML.`class` := "synopsis")( - elems.toList.map(_.html) - ) - ) - } - val htmlBig: HTML = { - Seq( - HTML.div(HTML.`class` := "summary")( - elems.toList.map(_.html) - ) - ) - } - } - object Synopsis { - def apply(elem: Section): Synopsis = Synopsis(List1(elem)) - def apply(elems: Section*): Synopsis = - Synopsis(List1(elems.head, elems.tail.toList)) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Body //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Body - block used to hold proper body of documentation - * - * @param elems - sections which make up body - */ - final case class Body(elems: List1[Section]) extends Symbol { - val newLn: Elem = Elem.Newline - val repr: Repr.Builder = R + newLn + elems.head + elems.tail.map( - R + newLn + _ - ) - val html: HTML = Seq( - HTML.div(HTML.`class` := "body")(elems.toList.map(_.html)) - ) - } - - object Body { - def apply(elem: Section): Body = Body(List1(elem)) - def apply(elems: Section*): Body = - Body(List1(elems.head, elems.tail.toList)) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Tags //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Tags - block used to hold tags for documentation - * - * @param elems - list of Tag of which Tags is made of - */ - final case class Tags(elems: List1[Tags.Tag]) extends Symbol { - val newLn: Elem = Elem.Newline - val repr: Repr.Builder = - R + elems.head + elems.tail.map(R + newLn + _) + newLn - val html: HTML = Seq( - HTML.div(HTML.`class` := "tags")( - elems.toList.map(_.html) - ) - ) - } - object Tags { - def apply(elem: Tag): Tags = Tags(List1(elem)) - def apply(elems: Tag*): Tags = Tags(List1(elems.head, elems.tail.toList)) - - /** Tag - one single tag for Tags - * - * @param indent - indent of tag - * @param typ - type of tag, which can be one of listed in object `Type` - * @param details - optional information for tag - */ - final case class Tag(indent: Int, typ: Tag.Type, details: Option[String]) - extends Elem { - val name: String = typ.toString.toUpperCase - val cName: String = typ.toString.toLowerCase - val repr: Repr.Builder = typ match { - case Tag.Unrecognized => R + indent + details - case _ => R + indent + name + details - } - val html: HTML = { - val htmlClass = HTML.`class` := "tag" - typ match { - case Tag.Unrecognized => - Seq( - HTML.p(htmlClass)( - HTML.span(HTML.`class` := cName)(details.html) - ) - ) - case Tag.Type.TextOnly => - Seq( - HTML.p(htmlClass)( - HTML.span(HTML.`class` := cName)("TEXT ONLY")( - details.html - ) - ) - ) - case _ => - Seq( - HTML.p(htmlClass)( - HTML.span(HTML.`class` := cName)(name)(details.html) - ) - ) - } - } - } - object Tag { - val defaultIndent = 0 - def apply(typ: Type): Tag = Tag(defaultIndent, typ, None) - def apply(typ: Type, details: String): Tag = - Tag(defaultIndent, typ, Some(details)) - def apply(indent: Int, typ: Type): Tag = Tag(indent, typ, None) - def apply(indent: Int, typ: Type, details: String): Tag = - Tag(indent, typ, Some(details)) - - sealed trait Type - object Type { - case object Added extends Type - case object Advanced extends Type - case object Alias extends Type - case object Deprecated extends Type - case object Modified extends Type - case object Private extends Type - case object Removed extends Type - case object TextOnly extends Type - case object Unstable extends Type - case object Upcoming extends Type - val codes = ADT.constructors[Type] - } - case object Unrecognized extends Type - - } - - implicit final class ExtForTagDetails(val self: Option[String]) { - val html: HTML = Seq(self.map(HTML.span(HTML.`class` := "details")(_))) - } - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr.scala deleted file mode 100644 index cbe5b0ff3e04..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr.scala +++ /dev/null @@ -1,140 +0,0 @@ -package org.enso.syntax.text.ast - -import java.nio.charset.StandardCharsets - -import org.enso.data.List1 -import org.enso.data.Shifted -import cats.Monoid -import cats.implicits._ - -import scala.annotation.tailrec - -//////////////////////////////////////////////////////////////////////////////// -//// Repr ////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -trait Repr[T] { - def repr(a: T): Repr.Builder -} -object Repr { - - //// Smart Constructors //// - - def apply[T: Repr](t: T): Builder = implicitly[Repr[T]].repr(t) - - val R = Repr.Builder.Empty() - - //// Operations //// - - implicit class ToReprOps[T: Repr](t: T) { - def repr: Builder = Repr(t) - } - - ///// Instances //// - - implicit def reprForUnit: Repr[Unit] = - _ => Repr.Builder.Empty() - implicit def reprForString: Repr[String] = - Repr.Builder.Text(_) - implicit def reprForInt: Repr[Int] = { - case 0 => R - case i => Repr.Builder.Space(i) - } - implicit def reprForChar: Repr[Char] = - Repr.Builder.Letter(_) - implicit def reprForTuple2[T1: Repr, T2: Repr]: Repr[(T1, T2)] = - t => Repr.Builder.Seq(Repr(t._1), Repr(t._2)) - implicit def reprForProvider[T <: Repr.Provider]: Repr[T] = - _.repr - implicit def reprForList[T: Repr]: Repr[List[T]] = - _.map(_.repr).fold(R: Builder)(Repr.Builder.Seq(_, _)) - implicit def reprForList1[T: Repr]: Repr[List1[T]] = - t => R + t.head + t.tail - implicit def reprForShifted[T: Repr]: Repr[Shifted[T]] = - t => R + t.off + t.wrapped - implicit def reprForShiftedList1[T: Repr]: Repr[Shifted.List1[T]] = - t => R + t.head + t.tail - implicit def reprForOption[T: Repr]: Repr[Option[T]] = - _.map(_.repr).getOrElse(R) - implicit def reprForNone: Repr[None.type] = - _ => R - implicit def reprForSome[T: Repr]: Repr[Some[T]] = - _.map(_.repr).getOrElse(R) - - ////////////////////////////////////////////////////////////////////////////// - //// Provider //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - trait Provider { - val repr: Builder - } - - ////////////////////////////////////////////////////////////////////////////// - //// Builder ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - sealed trait Builder { - import Builder._ - - val byteSpan: Int - val span: Int - - def +[T: Repr](that: T): Builder = this |+| Repr(that) - def build(): String = { - val bldr = new StringBuilder() - @tailrec - def go(lst: List[Builder]): Unit = lst match { - case Nil => - case r :: rs => - r match { - case _: Empty => go(rs) - case r: Letter => bldr += r.char; go(rs) - case r: Space => for (_ <- 1 to r.span) { bldr += ' ' }; go(rs) - case r: Text => bldr ++= r.str; go(rs) - case r: Seq => go(r.first :: r.second :: rs) - } - } - go(List(this)) - bldr.result() - } - } - object Builder { - - //// Constructors //// - - final case class Empty() extends Builder { - val byteSpan = 0 - val span = 0 - } - final case class Letter(char: Char) extends Builder { - val byteSpan = char.toString.getBytes(StandardCharsets.UTF_8).length - val span = 1 - } - final case class Space(span: Int) extends Builder { - val byteSpan = span - } - final case class Text(str: String) extends Builder { - val byteSpan = str.getBytes(StandardCharsets.UTF_8).length - val span = str.length - } - final case class Seq(first: Builder, second: Builder) extends Builder { - val byteSpan = first.byteSpan + second.byteSpan - val span = first.span + second.span - } - - //// Instances //// - - implicit def fromString(a: String): Builder = Repr(a) - implicit def fromChar(a: Char): Builder = Repr(a) - implicit def reprForBuilder[T <: Builder]: Repr[T] = identity(_) - implicit val monoidForBuilder: Monoid[Builder] = new Monoid[Builder] { - def empty: Builder = R - def combine(l: Builder, r: Builder): Builder = (l, r) match { - case (_: Empty, t) => t - case (t, _: Empty) => t - case _ => Seq(l, r) - } - } - } - -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr2.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr2.scala deleted file mode 100644 index 66850008285a..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr2.scala +++ /dev/null @@ -1,176 +0,0 @@ -package org.enso.syntax.text.ast2 - -//// README //// -// -// This is a WORK IN PROGRESS in-place replacement for Repr implementation. -// It is meant to lower the memory consumption of AST by removing Repr cache -// from AST and keeping only the span information. In order to do so, we have -// to do the following steps: -// 1. ASTOf should keep val span:Int -// 2. ASTOf should keep def repr instead of val repr -// 3. ASTOf should implement the Spanned type class -// 4. In every definition of AST Shape we should annotate the Child to be both -// Spanned as well as Repr. Otherwise, the Spanned instance will be -// automatically created from Repr. See below for explanation. -// -// This implementation works even if the above points are not done, because -// Spanned implementation fallbacks to Repr when needed. However, this may not -// be the best design as its very error prone. If we provide `Repr` as super -// class constraint, the `Spanned` is auto-derived, which may make the code -// run very slow. -// -// Anyway, before moving to this implementation we need to solve another -// problem. Printing of the AST throws stackoverflow here, as we are not caching -// everything in memory anymore, but we are recursively printing each component, -// and for a deeply nested components (like 100k open braces) we reach the limit -// of the recursion in the lambda created inside of [[Repr.Builder.+]] -// implementation. -// -// def newBuiltMe(bldr: StringBuilder): Unit = { -// buildMe(bldr) -// sr.repr.repr(that).buildMe(bldr) -// } -// -// We should probably apply CPS transformation to the code builder here, -// unless there is any other better solution. - -import cats.Monoid -import cats.implicits._ -import org.enso.data.{List1, Shifted} - -trait SpannedRepr[T] { - val spanned: Spanned[T] - val repr: Repr[T] -} -object SpannedRepr { - def apply[T: SpannedRepr]: SpannedRepr[T] = implicitly[SpannedRepr[T]] - - implicit def auto[T: Repr: Spanned]: SpannedRepr[T] = new SpannedRepr[T] { - val spanned = implicitly[Spanned[T]] - val repr = implicitly[Repr[T]] - } - implicit def asSpanned[T: SpannedRepr]: Spanned[T] = SpannedRepr[T].spanned - implicit def asRepr[T: SpannedRepr]: Repr[T] = SpannedRepr[T].repr -} - -trait Spanned[T] { - def span(t: T): Int -} -object Spanned { - def apply[T: Spanned](t: T): Int = implicitly[Spanned[T]].span(t) - - implicit def spannedForInt: Spanned[Int] = identity(_) - implicit def spannedForList[T: Spanned]: Spanned[List[T]] = - _.map(Spanned(_)).sum - implicit def spannedForList1[T: Spanned]: Spanned[List1[T]] = - _.map(Spanned(_)).foldLeft(0)(_ + _) - implicit def spannedForShifted[T: Spanned]: Spanned[Shifted[T]] = - t => t.off + Spanned(t.wrapped) - implicit def spannedForRepr[T: Repr]: Spanned[T] = t => Repr(t).span - -} - -//////////////////////////////////////////////////////////////////////////////// -//// Repr ////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -trait Repr[T] { - def repr(a: T): Repr.Builder -} -object Repr { - - //// Smart Constructors //// - - def apply[T: Repr](t: T): Builder = implicitly[Repr[T]].repr(t) - val R = Monoid[Builder].empty - - //// Operations //// - - implicit class ToReprOps[T: Repr](t: T) { - def repr: Builder = Repr(t) - def span: Int = repr.span - } - - ///// Instances //// - - implicit def reprForUnit: Repr[Unit] = - _ => Monoid[Builder].empty - implicit def reprForString: Repr[String] = - Repr.Builder.Text(_) - implicit def reprForInt: Repr[Int] = { - case 0 => R - case i => Repr.Builder.Space(i) - } - implicit def reprForChar: Repr[Char] = - Repr.Builder.Letter(_) - implicit def reprForTuple2[T1: Repr, T2: Repr]: Repr[(T1, T2)] = - t => Repr(t._1) |+| Repr(t._2) - implicit def reprForProvider[T <: Repr.Provider]: Repr[T] = - _.repr - implicit def reprForList[T: Repr]: Repr[List[T]] = - _.map(_.repr).fold(R: Builder)(Repr.Builder.Seq) - implicit def reprForList1[T: Repr: Spanned]: Repr[List1[T]] = - t => R + t.head + t.tail - implicit def reprForShifted[T: Repr: Spanned]: Repr[Shifted[T]] = - t => R + t.off + t.wrapped - implicit def reprForShiftedList1[T: Repr: Spanned]: Repr[Shifted.List1[T]] = - t => R + t.head + t.tail - implicit def reprForOption[T: Repr]: Repr[Option[T]] = - _.map(_.repr).getOrElse(R) - implicit def reprForNone: Repr[None.type] = - _ => R - implicit def reprForSome[T: Repr]: Repr[Some[T]] = - _.map(_.repr).getOrElse(R) - - ////////////////////////////////////////////////////////////////////////////// - //// Provider //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - trait Provider { - val repr: Builder - } - - case class Builder(span: Int, buildMe: StringBuilder => Unit) { - - def build(): String = { - val bldr = new StringBuilder() - buildMe(bldr) - bldr.result() - } - - def +[T: SpannedRepr](that: T) = { - val sr = SpannedRepr[T] - val newSpan = span + sr.spanned.span(that) - def newBuiltMe(bldr: StringBuilder): Unit = { - buildMe(bldr) - sr.repr.repr(that).buildMe(bldr) - } - Builder(newSpan, newBuiltMe) - } - - def ++[T: Repr](that: T): Builder = this + " " + that - } - object Builder { - implicit val monoidForBuilder: Monoid[Builder] = new Monoid[Builder] { - def empty: Builder = Empty() - def combine(l: Builder, r: Builder): Builder = - Builder( - l.span + r.span, - (bldr: StringBuilder) => { - l.buildMe(bldr) - r.buildMe(bldr) - } - ) - } - - def Empty() = Builder(0, identity(_): Unit) - def Space(span: Int) = Builder(span, _ ++= (" " * span)) - def Letter(char: Char) = Builder(1, _ += char) - def Text(str: String) = Builder(str.length, _ ++= str) - def Seq(l: Builder, r: Builder) = l |+| r - - implicit def fromString(a: String): Builder = Repr(a) - implicit def fromChar(a: Char): Builder = Repr(a) - implicit def reprForBuilder[T <: Builder]: Repr[T] = identity(_) - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr3.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr3.scala deleted file mode 100644 index 879b353abfc9..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/Repr3.scala +++ /dev/null @@ -1,212 +0,0 @@ -package org.enso.syntax.text.ast3 - -//// README //// -// -// SEE THE README OF Repr2.scala first. -// -// Here is a simple Repr implementation with a trampoline in Shape._Seq. - -import java.nio.charset.StandardCharsets - -import org.enso.data.List1 -import org.enso.data.Shifted -import cats.Monoid -import cats.implicits._ - -import scala.annotation.tailrec - -trait SpannedRepr[T] { - val spanned: Spanned[T] - val repr: Repr[T] -} -object SpannedRepr { - def apply[T: SpannedRepr]: SpannedRepr[T] = implicitly[SpannedRepr[T]] - - implicit def auto[T: Repr: Spanned]: SpannedRepr[T] = new SpannedRepr[T] { - val spanned = implicitly[Spanned[T]] - val repr = implicitly[Repr[T]] - } - implicit def asSpanned[T: SpannedRepr]: Spanned[T] = SpannedRepr[T].spanned - implicit def asRepr[T: SpannedRepr]: Repr[T] = SpannedRepr[T].repr -} - -trait Spanned[T] { - def span(t: T): Int -} -object Spanned { - def apply[T: Spanned](t: T): Int = implicitly[Spanned[T]].span(t) - - implicit def spannedForInt: Spanned[Int] = identity(_) - implicit def spannedForList[T: Spanned]: Spanned[List[T]] = - _.map(Spanned(_)).sum - implicit def spannedForList1[T: Spanned]: Spanned[List1[T]] = - _.map(Spanned(_)).foldLeft(0)(_ + _) - implicit def spannedForShifted[T: Spanned]: Spanned[Shifted[T]] = - t => t.off + Spanned(t.wrapped) - implicit def spannedForRepr[T: Repr]: Spanned[T] = t => Repr(t).span - -} - -//////////////////////////////////////////////////////////////////////////////// -//// Repr ////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -trait Repr[T] { - def repr(a: T): Repr.Builder -} -object Repr { - - //// Smart Constructors //// - - def apply[T: Repr](t: T): Builder = implicitly[Repr[T]].repr(t) - val R = Monoid[Builder].empty - - //// Operations //// - - implicit class ToReprOps[T: Repr](t: T) { - def repr: Builder = Repr(t) - def span: Int = repr.span - } - - ///// Instances //// - - implicit def reprForUnit: Repr[Unit] = - _ => Monoid[Builder].empty - implicit def reprForString: Repr[String] = - Repr.Builder.Text(_) - implicit def reprForInt: Repr[Int] = { - case 0 => R - case i => Repr.Builder.Space(i) - } - implicit def reprForChar: Repr[Char] = - Repr.Builder.Letter(_) - implicit def reprForTuple2[T1: Repr, T2: Repr]: Repr[(T1, T2)] = - t => Repr(t._1) |+| Repr(t._2) - implicit def reprForProvider[T <: Repr.Provider]: Repr[T] = - _.repr - implicit def reprForList[T: Repr]: Repr[List[T]] = - _.map(_.repr).fold(R: Builder)(Repr.Builder.Seq) - implicit def reprForList1[T: Repr: Spanned]: Repr[List1[T]] = - t => R + t.head + t.tail - implicit def reprForShifted[T: Repr: Spanned]: Repr[Shifted[T]] = - t => R + t.off + t.wrapped - implicit def reprForShiftedList1[T: Repr: Spanned]: Repr[Shifted.List1[T]] = - t => R + t.head + t.tail - implicit def reprForOption[T: Repr]: Repr[Option[T]] = - _.map(_.repr).getOrElse(R) - implicit def reprForNone: Repr[None.type] = - _ => R - implicit def reprForSome[T: Repr]: Repr[Some[T]] = - _.map(_.repr).getOrElse(R) - - ////////////////////////////////////////////////////////////////////////////// - //// Provider //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - trait Provider { - val repr: Builder - } - - case class Builder(span: Int, shape: () => Shape) { - - def build(): String = shape().build() - - def +[T: SpannedRepr](that: T) = { - val sr = SpannedRepr[T] - val newSpan = span + sr.spanned.span(that) - def newShape(): Shape = Shape._Seq(shape, sr.repr.repr(that).shape) - - Builder(newSpan, newShape) - } - - def ++[T: Repr](that: T): Builder = this + " " + that - } - object Builder { - implicit val monoidForBuilder: Monoid[Builder] = new Monoid[Builder] { - def empty: Builder = Empty() - def combine(l: Builder, r: Builder): Builder = - Builder( - l.span + r.span, - () => Shape._Seq(() => l.shape(), () => r.shape()) - ) - } - - def Empty() = Builder(0, () => Shape.Empty()) - def Space(span: Int) = Builder(span, () => Shape.Space(span)) - def Letter(char: Char) = Builder(1, () => Shape.Letter(char)) - def Text(str: String) = Builder(str.length, () => Shape.Text(str)) - def Seq(l: Builder, r: Builder) = l |+| r - - implicit def fromString(a: String): Builder = Repr(a) - implicit def fromChar(a: Char): Builder = Repr(a) - implicit def reprForBuilder[T <: Builder]: Repr[T] = identity(_) - } - - ////////////////////////////////////////////////////////////////////////////// - //// Shape /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - sealed trait Shape { - import Shape._ - -// val byteSpan: Int -// val span: Int - - def +[T: Repr](that: T): Shape = Seq(this, Repr(that).shape()) - def ++[T: Repr](that: T): Shape = this + " " + that - def build(): String = { - val bldr = new StringBuilder() - @tailrec - def go(lst: List[Shape]): Unit = lst match { - case Nil => - case r :: rs => - r match { - case _: Empty => go(rs) - case r: Letter => bldr += r.char; go(rs) - case r: Space => for (_ <- 1 to r.span) { bldr += ' ' }; go(rs) - case r: Text => bldr ++= r.str; go(rs) - case r: _Seq => go(r.first() :: r.second() :: rs) - } - } - go(List(this)) - bldr.result() - } - } - object Shape { - - //// Constructors //// - - final case class Empty() extends Shape { - val byteSpan = 0 - val span = 0 - } - final case class Letter(char: Char) extends Shape { - val byteSpan = char.toString.getBytes(StandardCharsets.UTF_8).length - val span = 1 - } - final case class Space(span: Int) extends Shape { - val byteSpan = span - } - final case class Text(str: String) extends Shape { - val byteSpan = str.getBytes(StandardCharsets.UTF_8).length - val span = str.length - } - final case class _Seq(first: () => Shape, second: () => Shape) - extends Shape { -// val byteSpan = first.byteSpan + second.byteSpan -// val span = first.span + second.span - } - object Seq { def apply(l: Shape, r: Shape): Shape = l |+| r } - - //// Instances //// - - implicit val monoidForShape: Monoid[Shape] = new Monoid[Shape] { - def empty: Shape = Empty() - def combine(l: Shape, r: Shape): Shape = (l, r) match { - case (_: Empty, t) => t - case (t, _: Empty) => t - case _ => _Seq(() => l, () => r) - } - } - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builder.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builder.scala deleted file mode 100644 index 93c3b416ec4b..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builder.scala +++ /dev/null @@ -1,216 +0,0 @@ -package org.enso.syntax.text.ast.meta - -import cats.data.NonEmptyList -import org.enso.data -import org.enso.data.{List1, Shifted} -import org.enso.syntax.text.{AST, Shape} -import org.enso.syntax.text.AST.{Ident, Macro} -import org.enso.syntax.text.ast.meta.Pattern.streamShift - -import scala.annotation.tailrec - -///////////////// -//// Builder //// -///////////////// - -final class Builder( - head: Ident, - offset: Int = 0, - lineBegin: Boolean = false, - val isModuleBuilder: Boolean = false -) { - var context: Builder.Context = Builder.Context() - var macroDef: Option[Macro.Definition] = None - var current: Builder.Segment = new Builder.Segment(head, offset, lineBegin) - var revSegs: List[Builder.Segment] = List() - - def beginSegment(ast: Ident, off: Int): Unit = { - revSegs ::= current - current = new Builder.Segment(ast) - current.offset = off - } - - def merge(that: Builder): Unit = { - val revLeftStream = current.revStream - val (revUnusedLeftTgt, matched, rightUnusedTgt) = - that.build(revLeftStream) - val result = List1(matched: AST.SAST, rightUnusedTgt) - current.revStream = result.toList.reverse ++ revUnusedLeftTgt - } - - def build( - revStreamL: AST.Stream - ): (AST.Stream, Shifted[Macro], AST.Stream) = { - val revSegBldrs = List1(current, revSegs) - macroDef match { - case None => - val revSegs = revSegBldrs.map { segBldr => - val optAst = segBldr.buildAST() - val seg = Macro.Ambiguous.Segment(segBldr.ast, optAst) - Shifted(segBldr.offset, seg) - } - val segments = revSegs.reverse - val head = segments.head - val tail = segments.tail - val paths = context.tree.dropValues() - val stream = Shifted.List1(head.wrapped, tail) - val template = Macro.Ambiguous(stream, paths) - val newTok = Shifted(head.off, template) - (revStreamL, newTok, List()) - - case Some(mdef) => - val revSegPats = mdef.fwdPats.reverse - val revSegsOuts = zipWith(revSegBldrs, revSegPats)(_.build(_)) - val revSegs = revSegsOuts.map(_._1) - val revSegStreams = revSegsOuts.map(_._2) - val tailStream = revSegStreams.head - val segs = revSegs.reverse - - val (segs2, pfxMatch, newLeftStream) = mdef.back match { - case None => (segs, None, revStreamL) - case Some(pat) => - val fstSegOff = segs.head.off - val (revStreamL2, lastLOff) = streamShift(fstSegOff, revStreamL) - val pfxMatch = pat.matchRevUnsafe(revStreamL2) - val revStreamL3 = pfxMatch.stream - val streamL3 = revStreamL3.reverse - val (streamL4, newFstSegOff) = streamShift(lastLOff, streamL3) - val revStreamL4 = streamL4.reverse - val newFirstSeg = segs.head.copy(off = newFstSegOff) - val newSegs = segs.copy(head = newFirstSeg) - (newSegs, Some(pfxMatch.elem), revStreamL4) - - } - - val shiftSegs = Shifted.List1(segs2.head.wrapped, segs2.tail) - - if (!revSegStreams.tail.forall(_.isEmpty)) { - throw new Error( - "Internal error: not all template segments were fully matched" - ) - } - - // val resolved = mdef.fin(pfxMatch, shiftSegs.toList().map(_.el)) - val template = Macro.Match(pfxMatch, shiftSegs, None) - val newTok = Shifted(segs2.head.off, template) - - (newLeftStream, newTok, tailStream) - - } - } - - // FIXME This is here because of bug in scalajs https://github.com/scala-js/scala-js/issues/3885 - private def zipWith[A, B, C](a: NonEmptyList[A], b: NonEmptyList[B])( - f: (A, B) => C - ): NonEmptyList[C] = { - - @tailrec - def zwRev(as: List[A], bs: List[B], acc: List[C]): List[C] = - (as, bs) match { - case (Nil, Nil) => acc // without this we get match error - case (Nil, _) => acc - case (_, Nil) => acc - case (x :: xs, y :: ys) => zwRev(xs, ys, f(x, y) :: acc) - } - - NonEmptyList(f(a.head, b.head), zwRev(a.tail, b.tail, Nil).reverse) - } - - if (isModuleBuilder) - macroDef = Some( - Macro.Definition((AST.Blank(): AST) -> Pattern.Expr()) { ctx => - ctx.body match { - case List(seg) => - seg.body.toStream match { - case List(mod) => mod.wrapped - case _ => throw new scala.Error("The impossible happened") - } - case _ => throw new scala.Error("The impossible happened") - } - } - ) - - def buildAsModule(): AST = { - build(List())._2.wrapped match { - case Macro.Match.any(m) => - m.segs.head.body.toStream match { - case s :: Nil => s.wrapped - case _ => throw new scala.Error("The impossible happened.") - } - case _ => throw new scala.Error("The impossible happened.") - } - } -} - -object Builder { - def moduleBuilder(): Builder = - new Builder(AST.Blank(), isModuleBuilder = true, lineBegin = true) - - ///////////////// - //// Context //// - ///////////////// - - case class Context(tree: Registry.Tree, parent: Option[Context]) { - def lookup(t: AST): Option[Registry.Tree] = tree.get(t) - def isEmpty: Boolean = tree.isLeaf - - @tailrec - final def parentLookup(t: AST): Boolean = { - parent match { - case None => false - case Some(p) => - p.lookup(t) match { - case None => p.parentLookup(t) - case Some(_) => true - } - } - } - } - object Context { - def apply(): Context = Context(data.Tree(), None) - def apply(tree: Registry.Tree): Context = Context(tree, None) - } - - ///////////////// - //// Segment //// - ///////////////// - - class Segment( - val ast: Ident, - var offset: Int = 0, - val lineBegin: Boolean = false - ) { - import Macro._ - var revStream: AST.Stream = List() - - def buildAST(): Option[Shifted[AST]] = - Pattern.buildASTFrom(revStream.reverse) - - def build( - pat: Pattern, - reversed: Boolean = false - ): (Shifted[Match.Segment], AST.Stream) = { - val stream = revStream.reverse - pat.matchOpt(stream, lineBegin, reversed) match { - case None => - ( - Shifted( - offset, - Shape.Match.Segment( - ast, - Pattern.Match.FailedMatch(Pattern.FailedMatch(None)) - ) - ), - stream - ) - case Some(rr) => - (Shifted(offset, Match.Segment(ast, rr.elem)), rr.stream) - } - } - - ////////////////////////////////////// - - override def toString: String = - s"SegmentBuilder($offset, $revStream)" - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builtin.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builtin.scala deleted file mode 100644 index 6a316f4495a4..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Builtin.scala +++ /dev/null @@ -1,488 +0,0 @@ -package org.enso.syntax.text.ast.meta - -import org.enso.data.{List1, Shifted} -import org.enso.syntax.text.AST -import org.enso.syntax.text.AST.Macro.Definition -import org.enso.syntax.text.AST.{Opr, Var} -import org.enso.syntax.text.ast.Repr -import org.enso.syntax.text.ast.meta.Pattern.Match - -import scala.annotation.{nowarn, tailrec} - -/** It contains definitions of built-in macros, like if-then-else or (-). These - * macros might get moved to stdlib in the future. - */ -@nowarn("cat=unused") -object Builtin { - - val registry: Registry = { - - def internalError = throw new Error("Internal error") - - val group = Definition(Opr("(") -> Pattern.Expr().opt, Opr(")")) { ctx => - ctx.body match { - case List(st1, _) => - st1.body.toStream match { - case List() => AST.Group() - case List(t) => AST.Group(t) - case _ => internalError - } - case _ => internalError - } - } - - val sequenceLiteral = { - val items = Pattern.SepList(Pattern.ExprUntilOpr(","), Opr(",")).opt - Definition( - Opr("[") -> items, - Opr("]") - ) { ctx => - ctx.body match { - case List(items, _) => - val realItems = - items.body.toStream - .map(_.wrapped) - .filterNot(AST.Ident.Opr.unapply(_).contains(",")) - AST.SequenceLiteral(realItems) - case _ => internalError - } - } - } - - val typesetLiteral = { - val expression = Pattern.Expr(allowBlocks = false).opt - Definition( - Opr("{") -> expression, - Opr("}") - ) { ctx => - ctx.body match { - case List(seg1, _) => - val body = seg1.body.toStream.map(_.wrapped) - body match { - case List(expr) => - AST.TypesetLiteral(Some(expr)) - case _ => AST.TypesetLiteral(None) - } - case _ => internalError - } - } - } - - val defn = Definition(Var("type") -> { - val head = Pattern.Cons().or("missing name").tag("name") - val args = - Pattern.NonSpacedExpr_().tag("parameter").many.tag("parameters") - val body = Pattern.Block().tag("body").opt - head :: args :: body - }) { ctx => - ctx.body match { - case List(st1) => - import Pattern.Match._ - st1.body match { - case Seq(_, (namePat, Seq(_, (argsPat, bodyPat)))) => - val args = argsPat.toStream.map(_.wrapped) - val body = bodyPat.toStream match { - case List(Shifted(_, AST.Block.any(block))) => Some(block) - case List() => None - case _ => internalError - } - namePat.toStream match { - case List(Shifted(_, AST.Cons.any(n))) => AST.Def(n, args, body) - case _ => internalError - } - case _ => internalError - } - case _ => internalError - } - } - - val polyglotJavaImport = { - val javaQualName = - Pattern.SepList(Pattern.Cons().or(Pattern.Var()), AST.Opr(".")) - val rename = Var("as") :: Pattern.Cons() - Definition( - Var("polyglot") -> Pattern.fromAST(Var("java")), - Var("import") -> (javaQualName :: rename.opt) - ) { ctx => - ctx.body match { - case List(_, nameAndRename) => - val (nameMatch, renameMatch) = - nameAndRename.body.asInstanceOf[Match.Seq[Shifted[AST]]].elem - val nonDot: List[AST.Ident] = - nameMatch.toStream.map(_.wrapped).collect { - case AST.Ident.Var.any(v) => v: AST.Ident - case AST.Ident.Cons.any(c) => c: AST.Ident - } - val rename: Option[AST.Ident.Cons] = { - renameMatch.toStream.map(_.wrapped).collectFirst { - case AST.Ident.Cons.any(n) => n - } - } - // The optional unwrap is safe by construction - the pattern - // guarantees at least one Var or Cons in the match result. - AST.JavaImport( - List1.fromListOption(nonDot).getOrElse(internalError), - rename - ) - case _ => internalError - } - } - } - - val if_then = Definition( - Var("if") -> Pattern.Expr(allowBlocks = false), - Var("then") -> Pattern.Expr() - ) { ctx => - ctx.body match { - case List(s1, s2) => - (s1.body.toStream, s2.body.toStream) match { - case (List(t1), List(t2)) => - AST.Mixfix(List1(s1.head, s2.head), List1(t1.wrapped, t2.wrapped)) - case _ => internalError - } - case _ => internalError - } - } - - val if_then_else = Definition( - Var("if") -> Pattern.Expr(allowBlocks = false), - Var("then") -> Pattern.Expr(), - Var("else") -> Pattern.Expr() - ) { ctx => - ctx.body match { - case List(s1, s2, s3) => - (s1.body.toStream, s2.body.toStream, s3.body.toStream) match { - case (List(t1), List(t2), List(t3)) => - AST.Mixfix( - List1(s1.head, s2.head, s3.head), - List1(t1.wrapped, t2.wrapped, t3.wrapped) - ) - case _ => internalError - } - case _ => internalError - } - } - - val case_of = Definition( - Var("case") -> Pattern.Expr(allowBlocks = false), - Var("of") -> Pattern.Block() - ) { ctx => - ctx.body match { - case List(scrutineePart, branchesPart) => - (scrutineePart.body.toStream, branchesPart.body.toStream) match { - case (List(scrutinee), List(branches)) => - AST.Mixfix( - List1(scrutineePart.head, branchesPart.head), - List1(scrutinee.wrapped, branches.wrapped) - ) - case _ => internalError - } - case _ => internalError - } - } - - val nonSpacedExpr = Pattern.Any(Some(false)).many1.build - - // NOTE: The macro engine currently resolves ahead of all operators, meaning - // that `->` doesn't obey the right precedence (e.g. with respect to `:`). - val arrow = Definition( - Some(nonSpacedExpr.or(Pattern.ExprUntilOpr("->"))), - Opr("->") -> Pattern.NonSpacedExpr().or(Pattern.Expr()) - ) { ctx => - (ctx.prefix, ctx.body) match { - case (Some(pfx), List(s1)) => - (pfx.toStream, s1.body.toStream) match { - case (List(l), List(r)) => - AST.App.Infix(l.wrapped, Opr("->"), r.wrapped) - case _ => internalError - } - case _ => internalError - } - } - - val skip = Definition( - Var("skip") -> Pattern.Expr() - ) { ctx => - ctx.body match { - case List(s1) => - s1.body.toStream match { - case List(Shifted(_, body: AST)) => - @tailrec - def go(t: AST): AST = - t match { - case AST.App.Prefix(_, arg) => arg - case AST.App.Infix(self, _, _) => go(self) - case AST.Macro.Match.any(m) => go(m.resolved.orNull) - case AST.Group(None) => t - case AST.Group(Some(s)) => go(s) - case _ => t - } - - go(body) - case _ => internalError - } - case _ => internalError - } - } - - val freeze = Definition( - Var("freeze") -> (Pattern.Var() :: Pattern.Block()) - ) { ctx => - ctx.body match { - case List(s1) => - s1.body.toStream match { - case List(Shifted(_, _)) => - // TODO: Ability to do parsing here - Var(s"Save to file using ${ctx.id}") - case _ => internalError - } - case _ => internalError - } - } - - val docComment = Definition( - Opr("##") -> Pattern - .Any() - .many - .fromBegin - .or(Pattern.Any().but(Pattern.Block()).many) - .tag("comment") - ) { ctx => - ctx.body match { - case List(s1) => - val stream = s1.body.toStream - val indent = 2 - val text = Repr(stream).build() - val lines = text.split("\n").toList - val lines2 = lines.head :: lines.tail.map(_.drop(indent)) - AST.Comment(lines2) - case _ => internalError - } - } - - val (qualifiedImport, itemsImport, qualifiedExport, itemsExport) = { - val qualNamePat = (Pattern.fromAST(Var("project")) | ((Pattern - .Var() | Pattern.Cons()) :: Opr(".") :: Pattern.Cons())) :: (Opr( - "." - ) :: Pattern.SepList(Pattern.Cons(), Opr("."))).opt - val rename = Var("as") :: Pattern.Cons() - - def extractRename( - matched: Pattern.Match - ): (List1[AST.Ident], Option[AST.Ident.Cons]) = { - val (nameMatch, renameMatch) = - matched.asInstanceOf[Match.Seq[Shifted[AST]]].elem - val name: List1[AST.Ident] = - List1( - nameMatch.toStream - .map(_.wrapped) - .collect { - case AST.Ident.Cons.any(n) => n - case AST.Ident.Var.any(n) => n - } - ).get - val rename: Option[AST.Ident.Cons] = { - renameMatch.toStream.map(_.wrapped).collectFirst { - case AST.Ident.Cons.any(n) => n - } - } - (name, rename) - } - - val consOrVar: PartialFunction[AST, AST.Ident] = { - case AST.Ident.Var.any(v) => v: AST.Ident - case AST.Ident.Cons.any(c) => c: AST.Ident - } - - val itemsImport = { - val all: Pattern = Var("all") - val items = Pattern.SepList(Pattern.Cons() | Pattern.Var(), Opr(",")) - val hiding = Var("hiding") :: items - - Definition( - Var("from") -> (qualNamePat :: rename.opt), - Var("import") -> ((all :: hiding.opt) | items) - ) { ctx => - ctx.body match { - case List(imp, itemsMatch) => - val (name, rename) = extractRename(imp.body) - val (hiding, items) = itemsMatch.body match { - case Match.Or(_, Left(hidden)) => - val hiddenItems = hidden.toStream - .map(_.wrapped) - .drop(2) - .collect(consOrVar) - (List1(hiddenItems), None) - - case Match.Or(_, Right(imported)) => - val importedItems = imported.toStream - .map(_.wrapped) - .collect(consOrVar) - (None, List1(importedItems)) - case _ => internalError - } - AST.Import(name, rename, true, items, hiding) - case _ => internalError - } - - } - } - - val itemsExport = { - val all: Pattern = Var("all") - val items = Pattern.SepList(Pattern.Cons() | Pattern.Var(), Opr(",")) - val hiding = Var("hiding") :: items - - Definition( - Var("from") -> (qualNamePat :: rename.opt), - Var("export") -> ((all :: hiding.opt) | items) - ) { ctx => - ctx.body match { - case List(imp, itemsMatch) => - val (name, rename) = extractRename(imp.body) - val (hiding, items) = itemsMatch.body match { - case Match.Or(_, Left(hidden)) => - val hiddenItems = hidden.toStream - .map(_.wrapped) - .drop(2) - .collect(consOrVar) - (List1(hiddenItems), None) - - case Match.Or(_, Right(imported)) => - val importedItems = imported.toStream - .map(_.wrapped) - .collect(consOrVar) - (None, List1(importedItems)) - case _ => internalError - } - AST.Export(name, rename, true, items, hiding) - case _ => internalError - } - - } - } - - def qualifiedConstruct( - constructName: String, - constructFactory: ( - List1[AST.Ident], - Option[AST.Ident.Cons], - Boolean, - Option[List1[AST.Ident.Cons]], - Option[List1[AST.Ident.Cons]] - ) => AST - ): Definition = { - Definition( - Var( - constructName - ) -> (qualNamePat :: rename.opt) - ) { ctx => - ctx.body match { - case List(s1) => - val (name, rename) = extractRename(s1.body) - constructFactory(name, rename, false, None, None) - case _ => internalError - } - } - } - - ( - qualifiedConstruct("import", AST.Import.apply), - itemsImport, - qualifiedConstruct("export", AST.Export.apply), - itemsExport - ) - } - - val privateDef = { - Definition(Var("private") -> Pattern.Expr()) { ctx => - ctx.body match { - case List(s1) => - s1.body.toStream match { - case List(expr) => - AST.Modified("private", expr.wrapped) - case _ => internalError - } - case _ => internalError - } - } - } - - val unsafeDef = { - Definition(Var("unsafe") -> Pattern.Expr()) { ctx => - ctx.body match { - case List(s1) => - s1.body.toStream match { - case List(expr) => - AST.Modified("unsafe", expr.wrapped) - case _ => internalError - } - case _ => internalError - } - } - } - - val fromKeyword = { - Definition(Var("from") -> Pattern.Nothing()) { ctx => - ctx.body match { - case List(_) => AST.Ident.Var("from") - case _ => internalError - } - } - } - - val unsafeKeyword = { - Definition(Var("unsafe") -> Pattern.Nothing()) { ctx => - ctx.body match { - case List(_) => AST.Ident.Var("unsafe") - case _ => internalError - } - } - } - - val privateKeyword = { - Definition(Var("private") -> Pattern.Nothing()) { ctx => - ctx.body match { - case List(_) => AST.Ident.Var("private") - case _ => internalError - } - } - } - - // TODO - // We may want to better represent empty AST. Moreover, there should be a - // way to generate multiple top-level entities from macros (like multiple - // atom definitions). One of the solutions to consider is to make AST - // instance of Monoid, add a `Nothing` node, and replace all lines in a - // block with a `Seq` node. This would allow us here to return `Nothing`, - // and also return many top-level defs connected with a `Seq`. - val disableComment = - Definition(Opr("#") -> Pattern.Expr().tag("disable")) { _ => AST.Blank() } - - Registry( - privateDef, - unsafeDef, - group, - sequenceLiteral, - typesetLiteral, - case_of, - if_then_else, - if_then, - polyglotJavaImport, - itemsImport, - qualifiedImport, - itemsExport, - qualifiedExport, - defn, - arrow, - docComment, - disableComment, - skip, - freeze, - fromKeyword, - unsafeKeyword, - privateKeyword - ) - } - -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala deleted file mode 100644 index 2a7781449a93..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala +++ /dev/null @@ -1,567 +0,0 @@ -package org.enso.syntax.text.ast.meta - -import org.enso.data.{Index, Shifted} -import org.enso.syntax.text.AST.SAST -import org.enso.syntax.text.ast.Repr -import org.enso.syntax.text.prec.Operator -import org.enso.syntax.text.{AST, HasSpan, OffsetZip} - -import scala.annotation.{nowarn, tailrec} - -//////////////////////////////////////////////////////////////////////////////// -//// Pattern /////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -object Pattern { - import cats.{Foldable, Functor, Traverse} - import cats.derived._ - - type P = Pattern - type Spaced = Option[Boolean] // TODO [AA] Make this an actual ADT - - // TODO: Refactorme - def streamShift_(off: Int, revStream: AST.Stream): AST.Stream = - streamShift(off, revStream)._1 - - def streamShift(off: Int, revStream: AST.Stream): (AST.Stream, Int) = { - @tailrec - def go(off: Int, str: AST.Stream, out: AST.Stream): (AST.Stream, Int) = - str match { - case Nil => (out, off) - case t :: ts => go(t.off, ts, Shifted(off, t.wrapped) :: out) - } - val (nStream, nOff) = go(off, revStream, List()) - (nStream.reverse, nOff) - } - - sealed trait Class - object Class { - final case object Normal extends Class - final case object Pattern extends Class - } - - //// Primitive Constructors //// - - // format: off - /** Boundary Patterns */ - final case class Begin () extends P - final case class End () extends P - - /** Structural Patterns */ - final case class Nothing () extends P - final case class Seq (pat1 : P , pat2 : P) extends P - final case class Or (pat1 : P , pat2 : P) extends P - final case class Many (pat : P) extends P - final case class Except (not : P , pat : P) extends P - - /** Meta Patterns */ - final case class Build (pat : P) extends P - final case class Err (msg : String, pat : P) extends P - final case class Tag (tag : String, pat : P) extends P - final case class Cls (cls : Class , pat : P) extends P - - /** Token Patterns */ - final case class Tok (spaced : Spaced, ast : AST) extends P - final case class Blank (spaced : Spaced) extends P - final case class Var (spaced : Spaced) extends P - final case class Cons (spaced : Spaced) extends P - final case class Opr (spaced : Spaced, maxPrec: Option[Int]) extends P - final case class Annotation(spaced : Spaced) extends P - final case class Mod (spaced : Spaced) extends P - final case class Num (spaced : Spaced) extends P - final case class Text (spaced : Spaced) extends P - final case class Block (spaced : Spaced) extends P - final case class Macro (spaced : Spaced) extends P - final case class Invalid (spaced : Spaced) extends P - - final case class FailedMatch(spaced: Spaced) extends P - // format: on - - //// Smart Constructors //// - - object Tok { - def apply(ast: AST): Tok = Tok(None, ast) - } - object Var { - def apply(): Var = Var(None) - def apply(spaced: Boolean): Var = Var(Some(spaced)) - } - object Cons { - def apply(): Cons = Cons(None) - def apply(spaced: Boolean): Cons = Cons(Some(spaced)) - } - object Opr { - def apply(): Opr = Opr(None, None) - def apply(spaced: Spaced): Opr = Opr(spaced, None) - def apply(spaced: Boolean): Opr = Opr(Some(spaced)) - } - object Annotation { - def apply(): Annotation = Annotation(None) - def apply(spaced: Boolean): Annotation = Annotation(Some(spaced)) - } - object Num { - def apply(): Num = Num(None) - def apply(spaced: Boolean): Num = Num(Some(spaced)) - } - object Text { - def apply(): Text = Text(None) - def apply(spaced: Boolean): Text = Text(Some(spaced)) - } - object Block { - def apply(): Block = Block(None) - def apply(spaced: Boolean): Block = Block(Some(spaced)) - } - - def Any(spaced: Spaced = None, allowBlocks: Boolean = true): Pattern = - Blank(spaced) | - Var(spaced) | - Cons(spaced) | - Opr(spaced) | - Annotation(spaced) | - Mod(spaced) | - Num(spaced) | - Text(spaced) | - Macro(spaced) | - (if (allowBlocks) { - Block(spaced) | - Invalid(spaced) - } else { - Invalid(spaced) - }) - def Any(spaced: Boolean): Pattern = Any(Some(spaced)) - def ErrTillEnd(msg: String): Pattern = Any().tillEnd.err(msg) - def ErrUnmatched(msg: String): Pattern = End() | ErrTillEnd(msg) - def Expr(allowBlocks: Boolean = true): Pattern = - Any(allowBlocks = allowBlocks).many1.build - def NonSpacedExpr(): Pattern = Any(spaced = false).many1.build - def NonSpacedExpr_(): Pattern = - (Any().but(Block()) :: Any(spaced = false).many).build - def SepList(pat: Pattern, div: Pattern): Pattern = pat :: (div :: pat).many - def SepList(pat: Pattern, div: Pattern, err: String): Pattern = { - val seg = pat | Any().till(div).err(err) - SepList(seg, div) - } - - def ExprUntilOpr(opr: String) = { - val base = Except(Opr(None, Some(AST.Opr(opr).prec)), Any()) - base.many1.build - } - - //// Utils //// - - def buildASTFrom(stream: AST.Stream): Option[Shifted[AST]] = - Operator.rebuild(stream) - - //// Conversions //// - - implicit def fromAST(ast: AST): Pattern = Tok(ast) - - ////////////////////////////////////////////////////////////////////////////// - //// Pattern.Match /////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - object Match { - type Switch[T] = Either[T, T] - type M[T] = MatchOf[T] - type P = Pattern - val P = Pattern - val A = AST - - //// Primitive Constructors //// - - // format: off - /** Boundary Matches */ - final case class Begin [T](pat:P.Begin) extends M[T] - final case class End [T](pat:P.End) extends M[T] - - /** Structural Matches */ - final case class Nothing [T](pat:P.Nothing) extends M[T] - final case class Seq [T](pat:P.Seq , elem:(M[T], M[T])) extends M[T] - final case class Or [T](pat:P.Or , elem:Switch[M[T]]) extends M[T] - final case class Many [T](pat:P.Many , elem:List[M[T]]) extends M[T] - final case class Except [T](pat:P.Except, elem:M[T]) extends M[T] - - /** Meta Matches */ - final case class Build [T](pat:P.Build , elem:T) extends M[T] - final case class Err [T](pat:P.Err , elem:T) extends M[T] - final case class Tag [T](pat:P.Tag , elem:M[T]) extends M[T] - final case class Cls [T](pat:P.Cls , elem:M[T]) extends M[T] - - /** Token Matches */ - final case class Tok [T](pat:P.Tok , elem:T) extends M[T] - final case class Blank [T](pat:P.Blank , elem:T) extends M[T] - final case class Var [T](pat:P.Var , elem:T) extends M[T] - final case class Cons [T](pat:P.Cons , elem:T) extends M[T] - final case class Opr [T](pat:P.Opr , elem:T) extends M[T] - final case class Annotation[T](pat:P.Annotation , elem:T) extends M[T] - final case class Mod [T](pat:P.Mod , elem:T) extends M[T] - final case class Num [T](pat:P.Num , elem:T) extends M[T] - final case class Text [T](pat:P.Text , elem:T) extends M[T] - final case class Block [T](pat:P.Block , elem:T) extends M[T] - final case class Macro [T](pat:P.Macro , elem:T) extends M[T] - final case class Invalid [T](pat:P.Invalid , elem:T) extends M[T] - - final case class FailedMatch[T](pat:P.FailedMatch) extends M[T] - // format: on - - //// Smart Constructors //// - - object Nothing { - def apply[T](): Match.Nothing[T] = Match.Nothing(Pattern.Nothing()) - } - - //// Result //// - - final case class Result(elem: Match, stream: AST.Stream) { - def map(fn: Match => Match): Result = copy(elem = fn(elem)) - } - - } - - type Match = MatchOf[SAST] - sealed trait MatchOf[T] { - import MatchOf._ - import cats.implicits._ - - val M = Match - val pat: Pattern - - override def toString = s"Pattern.Match(${this.toStream})" - - def toStream: List[T] = this.map(List(_)).fold - - def mapStruct(f: MatchOf[T] => MatchOf[T]): MatchOf[T] = - f(this.mapStructShallow(_.mapStruct(f))) - - @nowarn("cat=unchecked") - def mapStructShallow(f: MatchOf[T] => MatchOf[T]): MatchOf[T] = - this match { - case m: M.Begin[T] => m - case m: M.End[T] => m - case m: M.Nothing[T] => m - case m: M.Seq[T] => m.copy(elem = m.elem.bimap(f, f)) - case m: M.Or[T] => m.copy(elem = m.elem.bimap(f, f)) - case m: M.Many[T] => m.copy(elem = m.elem.map(f)) - case m: M.Except[T] => m.copy(elem = f(m.elem)) - case m: M.Build[T] => m - case m: M.Err[T] => m - case m: M.Tag[T] => m.copy(elem = f(m.elem)) - case m: M.Cls[T] => m.copy(elem = f(m.elem)) - case m: M.Tok[T] => m - case m: M.Blank[T] => m - case m: M.Var[T] => m - case m: M.Cons[T] => m - case m: M.Opr[T] => m - case m: M.Annotation[T] => m - case m: M.Mod[T] => m - case m: M.Num[T] => m - case m: M.Text[T] => m - case m: M.Block[T] => m - case m: M.Macro[T] => m - case m: M.Invalid[T] => m - case m: M.FailedMatch[T] => m - } - - @nowarn("cat=unchecked") - def isValid: Boolean = { - var out = true - this.mapStruct { - case m: M.Err[_] => out = false; m - case m => m - } - out - } - } - object MatchOf { - import cats.implicits._ - - implicit def reprMatch[T: Repr]: Repr[MatchOf[T]] = - _.map(Repr(_)).fold - implicit def ftorMatch: Functor[MatchOf] = _MatchOf.ftorMatch - implicit def travMatch: Traverse[MatchOf] = _MatchOf.travMatch - implicit def foldMatch: Foldable[MatchOf] = _MatchOf.foldMatch - - implicit def offZipMatch[T: HasSpan]: OffsetZip[MatchOf, T] = - t => { - val s = t.map(Shifted(0, _)) - val s2 = mapWithOff(s) { case (i, el) => Shifted(i, el.wrapped) } - val s3 = s2.map(t => (Index(t.off), t.wrapped)) - s3 - } - - import HasSpan.implicits._ - implicit def span[T: HasSpan]: HasSpan[MatchOf[T]] = - t => { - t.toStream.span() - } - - val M = Match - def mapWithOff[T: HasSpan](self: MatchOf[T])(f: (Int, T) => T): MatchOf[T] = - mapWithOff_(self)(f, 0)._1 - - def mapWithOff_[T: HasSpan]( - self: MatchOf[T] - )(f: (Int, T) => T, off: Int): (MatchOf[T], Int) = - self match { - // TODO: [MWU] code below could likely be cleaned up with macro usage - case m: M.Build[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Err[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Tok[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Blank[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Var[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Cons[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Opr[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Annotation[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Mod[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Num[T] => (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Text[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Block[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Macro[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: M.Invalid[T] => - (m.copy(elem = f(off, m.elem)), off + m.elem.span()) - case m: Pattern.MatchOf[T] => - var loff = off - val out = m.mapStructShallow { p => - val (nmatch, noff) = mapWithOff_(p)(f, loff) - loff = noff - nmatch - } - (out, loff) - } - } - object _MatchOf { - def ftorMatch: Functor[MatchOf] = semiauto.functor - def travMatch: Traverse[MatchOf] = semiauto.traverse[MatchOf] - def foldMatch: Foldable[MatchOf] = { - semiauto.foldable[MatchOf] - } - } -} - -//////////////////////////////////////////////////////////////////////////////// -//// API /////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -sealed trait Pattern { - import Pattern._ - - implicit class OptionWhen(v: Option.type) { - def when[A](cond: Boolean)(a: => A): Option[A] = if (cond) Some(a) else None - } - - //////////////////////////// - //// Smart Constructors //// - //////////////////////////// - - def ::(that: Pattern): Pattern = Seq(that, this) - def !(that: Pattern): Pattern = Except(that, this) - def |(that: Pattern): Pattern = Or(this, that) - def |(msg: String): Pattern = this | Err(msg, Nothing()) - def |?(tag: String): Pattern = Tag(tag, this) - - def or(that: Pattern): Pattern = Or(this, that) - def or(msg: String): Pattern = this | Err(msg, Nothing()) - def err(msg: String): Pattern = Err(msg, this) - def but(pat: Pattern): Pattern = Except(pat, this) - def many: Pattern = Many(this) - def many1: Pattern = this :: this.many - def tag(tag: String): Pattern = Tag(tag, this) - def opt: Pattern = this | Nothing() - def build: Pattern = Build(this) - def till(end: Pattern): Pattern = this.but(end).many - def tillEnd: Pattern = this :: End() // fixme: rename - def fromBegin: Pattern = Begin() :: this - - def matchRevUnsafe( - stream: AST.Stream, - lineBegin: Boolean = false - ): Match.Result = - this.matchUnsafe(stream, lineBegin = lineBegin, reversed = true) - - ////////////////////////////////// - //// Pattern Match Resolution //// - ////////////////////////////////// - - /** Unsafe variant of AST Macro tokens pattern matching. If you want to use - * patterns that could not match all input tokens, use [[matchOpt]] instead. - */ - def matchUnsafe( - stream: AST.Stream, - lineBegin: Boolean = false, - reversed: Boolean = false - ): Match.Result = { - matchOpt(stream, lineBegin, reversed).getOrElse { - val badMatch: Match = Match.FailedMatch(FailedMatch(None)) - Match.Result(badMatch, stream) - } - } - - /** This function takes a pattern and applies it to AST input stream. The - * optional parameter 'reversed' is used for prefix (reverse) matching and is - * used for prefix macro matching. The function assumes that the pattern does - * not fail. - */ - def matchOpt( - stream0: AST.Stream, - lineBegin: Boolean, - reversed: Boolean - ): Option[Match.Result] = { - - val P = Pattern - val M = Match - - def matchList(p: Pattern, stream: AST.Stream): (List[Match], AST.Stream) = { - @tailrec - def go( - stream: AST.Stream, - revOut: List[Match] - ): (List[Match], AST.Stream) = - step(p, stream) match { - case None => (revOut.reverse, stream) - case Some(t) => go(t.stream, t.elem :: revOut) - } - go(stream, Nil) - } - - def stepWith(p: Pattern, stream: AST.Stream)( - f: Match => Match - ): Option[Match.Result] = step(p, stream).map(_.map(f)) - - def step(p: Pattern, stream: AST.Stream): Option[Match.Result] = { - - def out(m: Match, s: AST.Stream) = Match.Result(m, s) - def ret(m: Match, s: AST.Stream) = Some(Match.Result(m, s)) - def ret_(m: Match) = Some(Match.Result(m, stream)) - def retIf(b: Boolean)(m: Match, s: AST.Stream) = Option.when(b)(out(m, s)) - def retIf_(b: Boolean)(m: Match) = retIf(b)(m, stream) - - def matchByCls_[T: AST.UnapplyByType]( - spaced: Pattern.Spaced, - f: Shifted[T] => Match - ) = matchByCls[T](spaced)(a => Some(f(a))) - - def matchByCls[T](spaced: Pattern.Spaced)( - f: Shifted[T] => Option[Match] - )(implicit pat: AST.UnapplyByType[T]): Option[Match.Result] = - stream match { - case Shifted(off, pat(t)) :: ss => - val ok = spaced match { - case None => true - case Some(s) => - @nowarn("cat=unchecked") - val isBlock = t match { - case AST.Block.any(_) => true - case _ => false - } - (s == (off > 0)) && (!isBlock) - } - if (ok) f(Shifted(off, t)).map(out(_, ss)) else None - case _ => None - } - - p match { - - //// Boundary Matches //// - - case p @ P.Begin() => retIf_(lineBegin)(M.Begin(p)) - case p @ P.End() => retIf_(stream.isEmpty)(M.End(p)) - - //// Structural Matches //// - - case p @ P.Nothing() => ret_(M.Nothing(p)) - case p @ P.Seq(p1, p2) => - for { - r1 <- step(p1, stream) - r2 <- step(p2, r1.stream) - } yield out(M.Seq(p, (r1.elem, r2.elem)), r2.stream) - - case p @ P.Or(p1, p2) => - val m1 = stepWith(p1, stream)(r => M.Or(p, Left(r))) - m1.orElse(stepWith(p2, stream)(r => M.Or(p, Right(r)))) - - case p @ P.Many(p1) => - val (lst, rest) = matchList(p1, stream) - ret(M.Many(p, lst), rest) - - case p @ P.Except(p1, p2) => - step(p1, stream) match { - case Some(_) => None - case None => stepWith(p2, stream)(M.Except(p, _)) - } - - //// Meta Matches //// - - // When performing reverse pattern match, tokens use right-offsets - // instead of left ones, so we need to push them back before computing - // AST. - case p @ P.Build(p1) => - stepWith(p1, stream) { patMatch => - val stream = patMatch.toStream - val ast = - if (!reversed) buildASTFrom(stream).get - else { - val (shiftedStream, off) = streamShift(0, stream.reverse) - val shiftedAst = buildASTFrom(shiftedStream).get - shiftedAst.copy(off = off) - } - M.Build(p, ast) - } - - case p @ P.Err(msg, p1) => - step(p1, stream).map { - _.map(m => - M.Err(p, Shifted(AST.Invalid.Unexpected(msg, m.toStream))) - ) - } - - case p @ P.Tag(_, p1) => stepWith(p1, stream)(M.Tag(p, _)) - case p @ P.Cls(_, p1) => stepWith(p1, stream)(M.Cls(p, _)) - - //// Token Matches //// - - case p @ P.Tok(spaced, tok) => - stream match { - case Shifted(off, t) :: ss => - val ok = spaced.forall(_ == (off > 0)) - Option.when(tok == t && ok)(out(M.Tok(p, Shifted(off, t)), ss)) - case _ => None - } - - case p @ P.Blank(spaced) => - matchByCls_[AST.Blank](spaced, M.Blank(p, _)) - case p @ P.Var(spaced) => matchByCls_[AST.Var](spaced, M.Var(p, _)) - case p @ P.Cons(spaced) => matchByCls_[AST.Cons](spaced, M.Cons(p, _)) - case p @ P.Num(spaced) => matchByCls_[AST.Number](spaced, M.Num(p, _)) - case p @ P.Text(spaced) => matchByCls_[AST.Text](spaced, M.Text(p, _)) - case p @ P.Block(spaced) => - matchByCls_[AST.Block](spaced, M.Block(p, _)) - case p @ P.Opr(spaced, maxPrec) => - matchByCls[AST.Opr](spaced) { sast => - Option.when(maxPrec.forall(_ >= sast.wrapped.prec))(M.Opr(p, sast)) - } - case p @ P.Annotation(spaced) => - matchByCls_[AST.Annotation](spaced, M.Annotation(p, _)) - case p @ P.Mod(spaced) => matchByCls_[AST.Mod](spaced, M.Mod(p, _)) - - case p @ P.Macro(spaced) => - matchByCls_[AST.Macro](spaced, M.Macro(p, _)) - - case p @ P.Invalid(spaced) => - matchByCls_[AST.Invalid](spaced, M.Invalid(p, _)) - - case _: P.FailedMatch => - throw new RuntimeException( - "Should not occur during pattern matching." - ) - } - } - step(this, stream0) - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Registry.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Registry.scala deleted file mode 100644 index 6e63ee90f5ff..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Registry.scala +++ /dev/null @@ -1,28 +0,0 @@ -package org.enso.syntax.text.ast.meta - -import org.enso.data -import org.enso.data.List1 -import org.enso.syntax.text.AST -import org.enso.syntax.text.AST.Macro.Definition - -final case class Registry() { - var tree: Registry.Tree = data.Tree() - - override def toString: String = - tree.toString - - def insert(defn: Definition): Unit = - tree += defn.path.toList -> defn - - def get(path: List1[AST]): Option[Definition] = - tree.getValue(path.toList) -} - -object Registry { - type Tree = data.Tree[AST, Definition] - def apply(defs: Definition*): Registry = { - val registry = new Registry() - defs.foreach(registry.insert) - registry - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Assoc.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Assoc.scala deleted file mode 100644 index b01d6703066b..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Assoc.scala +++ /dev/null @@ -1,32 +0,0 @@ -package org.enso.syntax.text.ast.opr - -sealed trait Assoc -object Assoc { - case object Left extends Assoc - case object Right extends Assoc - - private val applicativePat = "?".r - - private def isApplicative(s: String) = s match { - case applicativePat() => s.length > 1 - case _ => false - } - - private def charAssoc(c: Char) = c match { - case '=' => -1 - case ',' => -1 - case '>' => -1 - case '<' => 1 - case _ => 0 - } - - def of(op: String): Assoc = - if (isApplicative(op)) Assoc.Left - else if (op == "in") Assoc.Left - else if (op == "<|") Assoc.Right - else if (op == "|>") Assoc.Left - else if (op == "<<") Assoc.Right - else if (op == ">>") Assoc.Left - else if (op.foldLeft(0)(_ + charAssoc(_)) >= 0) Assoc.Left - else Assoc.Right -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Info.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Info.scala deleted file mode 100644 index fe822d4bf09c..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Info.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.enso.syntax.text.ast.opr - -object Info { - val map: Map[String, (Int, Assoc)] = Prec.map.map { case (name, prec) => - name -> ((prec, Assoc.of(name))) - } - def of(op: String) = - map.getOrElse(op, (Prec.default, Assoc.of(op))) -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Prec.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Prec.scala deleted file mode 100644 index 3a3fd2f5f158..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/opr/Prec.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.enso.syntax.text.ast.opr - -object Prec { - - /** The precedence hierarchy, from loosest binding to tightest binding. */ - val hierarchy = List( - List("=", "#="), - List(";"), - List(":="), - List(":"), - List("->", "<-"), - List("~>", "<~"), - List("!"), - List("in"), - List("<:", "~"), - List("|"), - List("&"), - List("\\"), - List("?"), - List("|>", "<|", ">>", "<<"), - List("<*", "<*>", "*>", "<$", "<$>", "$>", "<+", "<+>", "+>"), - List(","), - List("==", ">", "<", ">=", "<="), - List("+", "-"), - List("*", "/", "%"), - List("^"), - List("."), - List(" ") - ) - - val map: Map[String, Int] = - hierarchy.zipWithIndex.flatMap { case (ops, prec) => - ops.map(_ -> prec) - }.toMap - - val default = map.getOrElse("^", 0) -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala deleted file mode 100644 index f5e6e177240f..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala +++ /dev/null @@ -1,350 +0,0 @@ -package org.enso.syntax.text.ast.text - -import org.enso.flexer.ADT - -sealed trait Escape { - val repr: String -} -sealed trait RawEscape { - val repr: String -} - -object Escape { - - final case object Unfinished extends RawEscape { - val repr = "" - } - - final case class Invalid(str: Char) extends RawEscape { - val repr = str.toString - } - - final case class Number(int: Int) extends Escape { - val repr = int.toString - } - - // Reference: https://en.wikipedia.org/wiki/String_literal - sealed trait Unicode extends Escape - object Unicode { - - /* Note [Circe and Naming] */ - type Invalid = InvalidUnicode - - /* Note [Circe and Naming] */ - val Invalid = InvalidUnicode - - /* Note [Circe and Naming] */ - final case class InvalidUnicode(unicode: Unicode) extends Unicode { - val repr = unicode.repr - } - - /* NOTE [Circe and Naming] - * Name of the class above cannot be Invalid, as we already have - * Escape.Invalid. And to be able to derive JSON serialization with circe - * case class names within a trait subtree need to be unique. - * - * To keep this unpleasant detail hidden from library users, we introduce - * aliases for type and object named `Invalid`. - */ - - type U16 = _U16 - final case class _U16(digits: String) extends Unicode { - val pfx = "u" - val sfx = "" - val repr = pfx + digits + sfx - } - - type U32 = _U32 - final case class _U32(digits: String) extends Unicode { - val pfx = "U" - val sfx = "" - val repr = pfx + digits + sfx - } - - type U21 = _U21 - final case class _U21(digits: String) extends Unicode { - val pfx = "u{" - val sfx = "}" - val repr = pfx + digits + sfx - } - - object Validator { - val hexChars = - (('a' to 'f') ++ ('A' to 'F') ++ ('0' to '9')).toSet - def isHexChar(char: Char) = - hexChars.contains(char) - } - - object U16 { - def apply(digits: String): Unicode = - if (validate(digits)) _U16(digits) - else Invalid(_U16(digits)) - def validate(digits: String) = { - import Validator._ - val validLength = digits.length == 4 - val validChars = digits.forall(isHexChar) - validLength && validChars - } - } - object U32 { - def apply(digits: String): Unicode = - if (validate(digits)) _U32(digits) - else Invalid(_U32(digits)) - def validate(digits: String) = { - import Validator._ - val validLength = digits.length == 8 - val validPrefix = digits.startsWith("00") - val validChars = digits.forall(isHexChar) - validLength && validPrefix && validChars - } - } - object U21 { - def apply(digits: String): Unicode = - if (validate(digits)) _U21(digits) - else Invalid(_U21(digits)) - def validate(digits: String) = { - import Validator._ - val validLength = digits.length >= 1 && digits.length <= 6 - val validChars = digits.forall(isHexChar) - validLength && validChars - } - } - } - - case object Slash extends RawEscape { - val code: Int = '\\' - def name: String = toString - override val repr = "\\" - } - case object Quote extends RawEscape { - val code: Int = '\'' - def name: String = toString - override val repr = "\'" - } - case object RawQuote extends RawEscape { - val code: Int = '"' - def name: String = toString - override val repr = "\"" - } - - // Reference: https://en.wikipedia.org/wiki/String_literal - sealed trait Character extends Escape { - def code: Char - } - object Character { - case object a extends Character { - override val code: Char = '\u0007' - def name: String = toString - override val repr = name - } - case object b extends Character { - override val code: Char = '\u0008' - def name: String = toString - override val repr = name - } - case object f extends Character { - override val code: Char = '\u000C' - def name: String = toString - override val repr = name - } - case object n extends Character { - override val code: Char = '\n' - def name: String = toString - override val repr = name - } - case object r extends Character { - override val code: Char = '\r' - def name: String = toString - override val repr = name - } - case object t extends Character { - override val code: Char = '\u0009' - def name: String = toString - override val repr = name - } - case object v extends Character { - override val code: Char = '\u000B' - def name: String = toString - override val repr = name - } - case object e extends Character { - override val code: Char = '\u001B' - def name: String = toString - override val repr = name - } - val codes = ADT.constructors[Character] - } - - // Reference: https://en.wikipedia.org/wiki/Control_character - sealed trait Control extends Escape { - val code: Int - } - object Control { - case object NUL extends Control { - val code: Int = 0x00 - def name: String = toString - override val repr = name - } - case object SOH extends Control { - val code: Int = 0x01 - def name: String = toString - override val repr = name - } - case object STX extends Control { - val code: Int = 0x02 - def name: String = toString - override val repr = name - } - case object ETX extends Control { - val code: Int = 0x03 - def name: String = toString - override val repr = name - } - case object EOT extends Control { - val code: Int = 0x04 - def name: String = toString - override val repr = name - } - case object ENQ extends Control { - val code: Int = 0x05 - def name: String = toString - override val repr = name - } - case object ACK extends Control { - val code: Int = 0x06 - def name: String = toString - override val repr = name - } - case object BEL extends Control { - val code: Int = 0x07 - def name: String = toString - override val repr = name - } - case object BS extends Control { - val code: Int = 0x08 - def name: String = toString - override val repr = name - } - case object TAB extends Control { - val code: Int = 0x09 - def name: String = toString - override val repr = name - } - case object LF extends Control { - val code: Int = 0x0a - def name: String = toString - override val repr = name - } - case object VT extends Control { - val code: Int = 0x0b - def name: String = toString - override val repr = name - } - case object FF extends Control { - val code: Int = 0x0c - def name: String = toString - override val repr = name - } - case object CR extends Control { - val code: Int = 0x0d - def name: String = toString - override val repr = name - } - case object SO extends Control { - val code: Int = 0x0e - def name: String = toString - override val repr = name - } - case object SI extends Control { - val code: Int = 0x0f - def name: String = toString - override val repr = name - } - case object DLE extends Control { - val code: Int = 0x10 - def name: String = toString - override val repr = name - } - case object DC1 extends Control { - val code: Int = 0x11 - def name: String = toString - override val repr = name - } - case object DC2 extends Control { - val code: Int = 0x12 - def name: String = toString - override val repr = name - } - case object DC3 extends Control { - val code: Int = 0x13 - def name: String = toString - override val repr = name - } - case object DC4 extends Control { - val code: Int = 0x14 - def name: String = toString - override val repr = name - } - case object NAK extends Control { - val code: Int = 0x15 - def name: String = toString - override val repr = name - } - case object SYN extends Control { - val code: Int = 0x16 - def name: String = toString - override val repr = name - } - case object ETB extends Control { - val code: Int = 0x17 - def name: String = toString - override val repr = name - } - case object CAN extends Control { - val code: Int = 0x18 - def name: String = toString - override val repr = name - } - case object EM extends Control { - val code: Int = 0x19 - def name: String = toString - override val repr = name - } - case object SUB extends Control { - val code: Int = 0x1a - def name: String = toString - override val repr = name - } - case object ESC extends Control { - val code: Int = 0x1b - def name: String = toString - override val repr = name - } - case object FS extends Control { - val code: Int = 0x1c - def name: String = toString - override val repr = name - } - case object GS extends Control { - val code: Int = 0x1d - def name: String = toString - override val repr = name - } - case object RS extends Control { - val code: Int = 0x1e - def name: String = toString - override val repr = name - } - case object US extends Control { - val code: Int = 0x1f - def name: String = toString - override val repr = name - } - case object DEL extends Control { - val code: Int = 0x7f - def name: String = toString - override val repr = name - } - val codes = ADT.constructors[Control] - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/package.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/package.scala index d2dd13ec69b5..8fad1e43ab62 100644 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/package.scala +++ b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/package.scala @@ -1,7 +1,8 @@ package org.enso.syntax -import org.enso.syntax.text.{AST => ASTDef} +import org.enso.data.Span +import java.util.UUID -package object text { - type AST = ASTDef._AST +object Parser { + type IDMap = Seq[(Span, UUID)] } diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Distance.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Distance.scala deleted file mode 100644 index 29ac7304bafc..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Distance.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.enso.syntax.text.prec - -import org.enso.syntax.text.AST -import org.enso.data.List1 -import org.enso.data.Shifted -import scala.annotation.tailrec - -object Distance { - - /** Segment is a list of AST tokens which are not separated with spaces */ - type Segment = List1[AST] - - def partition(lst: List1[Shifted[AST]]): List1[Shifted[Segment]] = { - @tailrec - def go( - input: List[Shifted[AST]], - currentOff: Int, - current: List1[AST], - out: List[Shifted[Segment]] - ): List1[Shifted[Segment]] = input match { - case Nil => List1(Shifted(currentOff, current.reverse), out).reverse - case ast1 :: ast2_ => - val isBlock = ast1.wrapped match { - case AST.Block.any(_) => true - case _ => false - } - val isGlued = (ast1.off == 0) && (!isBlock) - isGlued match { - case true => go(ast2_, currentOff, ast1.wrapped :: current, out) - case false => - val out2 = Shifted(currentOff, current.reverse) :: out - go(ast2_, ast1.off, List1(ast1.wrapped), out2) - } - } - go(lst.tail, lst.head.off, List1(lst.head.wrapped), Nil) - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Macro.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Macro.scala deleted file mode 100644 index 33e0a88bcadf..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Macro.scala +++ /dev/null @@ -1,149 +0,0 @@ -package org.enso.syntax.text.prec - -import org.enso.Logger -import org.enso.data.Shifted -import org.enso.syntax.text.AST -import org.enso.syntax.text.ast.meta.Builder -import org.enso.syntax.text.ast.meta.Builtin - -import scala.annotation.{tailrec, unused} - -object Macro { - val logger = new Logger() - - ////////////////// - //// Registry //// - ////////////////// - - def run(module: AST.Module): AST.Module = - module.map(transform) - - private def transform(t: AST): AST = - new Transformer(t).run(AST.tokenize(t).toList()) - - final private class Transformer(@unused t: AST) { - val root: Builder.Context = Builder.Context(Builtin.registry.tree) - - var builder: Builder = Builder.moduleBuilder() - var builderStack: List[Builder] = Nil - var isLineBegin: Boolean = true - - def pushBuilder(name: AST.Ident, off: Int, lineBegin: Boolean): Unit = - logger.trace { - builderStack +:= builder - builder = new Builder(name, off, lineBegin) - } - - def popBuilder(): Option[Builder] = logger.trace { - builderStack match { - case Nil => None - case b :: bs => - val out = builder - builder = b - builderStack = bs - Some(out) - } - } - - @tailrec - def finish(): AST = { - popBuilder() match { - case Some(bldr) => - logger.log("End of input (in stack)") - builder.merge(bldr) - finish() - case None => - logger.log("End of input (not in stack)") - builder.buildAsModule() - } - } - - @tailrec - def run(input: AST.Stream): AST = { - input match { - case Nil => - val builders = builder :: builderStack - var newRevBuilders: List[Builder] = List() - var subStream: AST.Stream = List() - for (bldr <- builders) { - val noLastPattern = bldr.macroDef.map(_.last.pattern).contains(None) - if (noLastPattern) { - val (revLeftUnusedStream, matched, rightUnusedStream) = - bldr.build(List()) - subStream = - subStream ++ (rightUnusedStream.reverse :+ matched) ++ revLeftUnusedStream - } else { - bldr.current.revStream = subStream ++ bldr.current.revStream - subStream = List() - newRevBuilders +:= bldr - } - - } - val newBuilders = newRevBuilders.reverse - - builder = newBuilders.head - builderStack = newBuilders.tail - finish() - case (t1 @ Shifted(off, AST.Ident.any(el1))) :: t2_ => - logger.log(s"Token $t1") - logger.beginGroup() - val wasLineBegin = isLineBegin - isLineBegin = false - builder.context.lookup(el1) match { - case Some(tr) => - logger.log("New segment") - builder.beginSegment(el1, off) - builder.macroDef = - tr.value.map(Some(_)).getOrElse(builder.macroDef) - builder.context = builder.context.copy(tree = tr) - logger.endGroup() - run(t2_) - - case None => - root.lookup(el1) match { - case Some(tr) => - logger.log("New macro") - val context = builder.context - pushBuilder(el1, t1.off, wasLineBegin) - builder.macroDef = tr.value - builder.context = Builder.Context(tr, Some(context)) - logger.endGroup() - run(t2_) - - case _ => - val currentClosed = builder.context.isEmpty - val parentPrecWin = (builder.current.ast, el1) match { - case (AST.Opr.any(_), _) => false - case (_, AST.Opr.any(_)) => true - case _ => false - } - val parentBreak = builder.context.parentLookup(el1) - (currentClosed || parentPrecWin) && parentBreak match { - case true => - logger.log("Parent close") - val subBuilder = builder - popBuilder() - builder.merge(subBuilder) - logger.endGroup() - run(input) - case false => - logger.log("Add token") - builder.current.revStream +:= t1 - logger.endGroup() - run(t2_) - } - } - } - case Shifted(off, AST.Block.any(el1)) :: t2_ => - val nt1 = Shifted(off, el1.map(transform)) - builder.current.revStream +:= nt1 - run(t2_) - - case t1 :: t2_ => - builder.current.revStream +:= t1 - run(t2_) - - } - } - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Operator.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Operator.scala deleted file mode 100644 index f5b1fa1fe56a..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/prec/Operator.scala +++ /dev/null @@ -1,133 +0,0 @@ -package org.enso.syntax.text.prec - -import org.enso.data.Compare._ -import org.enso.data.List1 -import org.enso.data.Shifted -import org.enso.syntax.text.AST -import org.enso.syntax.text.prec - -import scala.annotation.tailrec - -// format: off -// Without it, the code is a mess: -// https://github.com/scalameta/scalafmt/issues/1454 - -object Operator { - import Internal._ - - /** Build a single AST node from AST stream by applying operator precedence - * rules, including per-operator precedence and distance-based precedence. - */ - def rebuild(stream: AST.Stream1): Shifted[AST] = { - val stream2 = rebuildNonSpaced(stream) - val stream3 = rebuildSpaced(stream2) - stream3 - } - - def rebuildNonSpaced(stream: AST.Stream1): AST.Stream1 = { - val segs = prec.Distance.partition(stream) - segs.map(_.map(rebuildSubExpr)) - } - - def rebuildSpaced(flatExpr: AST.Stream1): Shifted[AST] = { - val list = Shifted.List1(flatExpr.head.wrapped,flatExpr.tail) - val flatExpr2 = Shifted(flatExpr.head.off, list) - flatExpr2.map(rebuildExpr) - } - - def rebuild(stream: AST.Stream): Option[Shifted[AST]] = - List1(stream).map(rebuild) - - final object Internal { - def oprToToken(ast: AST): AST.Opr = ast match { - case AST.Opr.any(t) => t - case _ => AST.Opr.app - } - - def rebuildSubExpr(seg: Distance.Segment): AST = - rebuildExpr(seg) match { - case AST.App.Sides.any(t) => t.opr - case t => t - } - - def rebuildExpr(seg: Distance.Segment): AST = - rebuildExpr(Shifted.List1(seg.head, seg.tail.map(Shifted(_)))) - - def rebuildExpr(seg: Shifted.List1[AST]): AST = { - final case class Input(seg: List[Shifted[AST]], stack: Shifted.List1[AST]) - implicit def _input(t: (List[Shifted[AST]], Shifted.List1[AST])): Input = - Input(t._1, t._2) - - @tailrec - def go(input: Input): AST = input.seg match { - case Nil => reduceAll(input.stack) - case seg1 :: seg2_ => - - val shift = (seg2_, seg1 +: input.stack) - val reduce = (input.seg, reduceHead(input.stack)) - - def handleAssoc(ast1: AST, ast2: AST) = { - val op1 = oprToToken(ast1) - val op2 = oprToToken(ast2) - compare(op1.prec, op2.prec) match { - case GT => shift - case LT => reduce - case EQ => (op1.assoc, op2.assoc) match { - case (AST.Assoc.Left, AST.Assoc.Left) => reduce - case _ => shift - } - } - } - - input.stack.head match { - case AST.Opr.any(stack1) => seg1.wrapped match { - case AST.Opr.any(seg1) => go(handleAssoc(seg1, stack1)) - case _ => go(shift) - } - case _ => input.stack.tail match { - case Nil => go(shift) - case stack2 :: _ => go(handleAssoc(seg1.wrapped, stack2.wrapped)) - } - } - } - - go((seg.tail, Shifted.List1(seg.head))) - } - - def reduceHead(stack: Shifted.List1[AST]): Shifted.List1[AST] = { - stack.head match { - case AST.Opr.any(s1) => stack.tail match { - case Nil => (AST.App.Sides(s1), Nil) - case s2 :: s3_ => s2.wrapped match { - case AST.Opr.any(_) => (AST.App.Sides(s1), s2 :: s3_) - case _ => (AST.App.Left(s2.wrapped, s2.off, s1), s3_) - } - } - case t1 => stack.tail match { - case Nil => stack - case s2 :: s3 :: s4_ => s2.wrapped match { - case AST.Opr.any(v2) => s3.wrapped match { - case AST.Opr.any(_) => (AST.App.Right(v2, s2.off, t1), s3 :: s4_) - case _ => - (AST.App.Infix(s3.wrapped, s3.off, v2, s2.off, t1), s4_) - } - case v2 => (AST.App.Prefix(v2, s2.off, t1), s3 :: s4_) - } - - case s2 :: s3_ => s2.wrapped match { - case AST.Opr.any(v2) => (AST.App.Right(v2, s2.off, t1), s3_) - case v2 => (AST.App.Prefix(v2, s2.off, t1), s3_) - } - } - } - } - - @tailrec - def reduceAll(stack: Shifted.List1[AST]): AST = { - stack.tail match { - case Nil => reduceHead(stack).head - case _ => reduceAll(reduceHead(stack)) - } - } - } -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/DocParserDef.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/DocParserDef.scala deleted file mode 100644 index 38e7bf1d7ea4..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/DocParserDef.scala +++ /dev/null @@ -1,1046 +0,0 @@ -package org.enso.syntax.text.spec - -import org.enso.flexer._ -import org.enso.flexer.automata.Pattern -import org.enso.flexer.automata.Pattern._ -import org.enso.data.List1 -import org.enso.syntax.text.ast.Doc._ -import org.enso.syntax.text.ast.Doc - -case class DocParserDef() extends Parser[Doc] { - - ////////////////////////////////////////////////////////////////////////////// - //// Result ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - override def getResult(): Option[Doc] = result.doc - - /** result - used to manage result from Doc Parser. - * - * current - used to hold elem parser works on. - * doc - used to hold ready to get Doc after parsing. - * stack - used to hold stack of elems. - */ - final object result { - var current: Option[Elem] = None - var doc: Option[Doc] = None - var stack: List[Elem] = Nil - - def push(): Unit = - logger.trace { - if (current.isDefined) { - logger.log(s"Pushed: $current") - stack +:= current.get - current = None - } else { - logger.err("Undefined current") - } - } - - def pop(): Unit = - logger.trace { - if (stack.nonEmpty) { - current = Some(stack.head) - stack = stack.tail - logger.log(s"New result: $current") - } else { - logger.err("Trying to pop empty AST stack") - } - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Basic Char Classification /////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - val lowerChar: Pattern = range('a', 'z') - val upperChar: Pattern = range('A', 'Z') - val digit: Pattern = range('0', '9') - val space: Pattern = ' ' - val whitespace: Pattern = space.many1 - val newline: Char = '\n' - - val char: Pattern = lowerChar | upperChar - val specialChars: Pattern = - "," | "." | ":" | ";" | "/" | "\\" | "’" | "=" | "'" | "|" | "+" | "-" | "#" | "\"" | "(" | ")" - val possibleChars: Pattern = char | digit | whitespace | specialChars - - val normalText: Pattern = possibleChars.many1 - - ////////////////////////////////////////////////////////////////////////////// - //// Text //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** text - used to manage normal text, made of Strings. */ - final object text { - def onPushing(in: String): Unit = - logger.trace { - if (documentation.isBeginning()) { - if (!tags.checkIfTagExistInPushedText(in)) { - val text = removeWhitespaces(in) - push(text) - } - } else if (section.isBeginning()) { - val text = removeWhitespaces(in) - push(text) - } else { - push(in) - } - } - - def removeWhitespaces(in: String): String = - logger.trace { - var text = in - if (text.nonEmpty) { - while (text.head == ' ' && text.length > 1) { - text = text.tail - } - } - text - } - - def push(in: String): Unit = - logger.trace { - result.current = Some(Elem.Text(in)) - result.push() - } - } - - ROOT || normalText || text.onPushing(currentMatch) - - ////////////////////////////////////////////////////////////////////////////// - //// Tags //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** tags - used to manage potentially tagged documentation. - * - * possibleTagsList - holds every correct tag possible to create. - * stack - holds applied tags. - */ - final object tags { - val possibleTagsList: List[Tags.Tag.Type] = - Tags.Tag.Type.codes.-(Tags.Tag.Unrecognized).toList - var stack: List[Tags.Tag] = Nil - - def pushTag(indent: Int, tagType: Tags.Tag.Type, details: String): Unit = - logger.trace { - if (details.replaceAll("\\s", "").length == 0) { - stack +:= Tags.Tag(indent, tagType) - } else { - if (details.nonEmpty) { - var det = text.removeWhitespaces(details) - if (tagType != Tags.Tag.Unrecognized) { - det = s" $det" - } - stack +:= Tags.Tag(indent, tagType, Some(det)) - } else { - Tags.Tag(indent, tagType, None) - } - } - result.current = None - } - - def checkIfTagExistInPushedText(in: String): Boolean = - logger.trace { - var inArray = in.split(" ") - var containsTag = false - - def tryFindingTagInAvailableTags(elem: String): Unit = - logger.trace { - for (tagType <- possibleTagsList) { - if (elem == tagType.toString.toUpperCase) { - containsTag = true - val tagDet = inArray.tail.mkString(" ") - pushTag(section.currentIndentRaw, tagType, tagDet) - } - } - if ( - !containsTag && !elem.contains(newline) && inArray.tail.isEmpty - ) { - pushTag(section.currentIndentRaw, Tags.Tag.Unrecognized, in) - containsTag = true - } - } - - if (inArray.nonEmpty) { - while (inArray.nonEmpty && inArray.head.isEmpty) { - section.currentIndentRaw += 1 - inArray = inArray.tail - } - val elem = inArray.head - if (elem.matches("^\\b[A-Z]{2,}\\b")) { - tryFindingTagInAvailableTags(elem) - } - } - - containsTag - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Code //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** code - used to manage code in documentation. */ - final object code { - def onPushingInline(in: String): Unit = - logger.trace { - val code = in.substring(1).dropRight(1) - result.current = Some(Elem.CodeBlock.Inline(code)) - result.push() - } - - def onPushingMultiline(in: String): Unit = - logger.trace { - val dummyLine = Elem.CodeBlock.Line(0, "") - do { - result.pop() - } while (result.current.get == Elem.Newline) - result.current match { - case Some(code @ (_: Elem.CodeBlock)) => - val newElem = Elem.CodeBlock.Line(indent.current, in) - if (code.elems.head == dummyLine) { - result.current = Some(Elem.CodeBlock(newElem)) - } else { - result.current = Some(Elem.CodeBlock(code.elems.append(newElem))) - } - case Some(_) | None => result.push() - } - result.push() - } - - val inlineCodeTrigger = '`' - val inlinePattern: Pattern = - inlineCodeTrigger >> not(inlineCodeTrigger).many >> inlineCodeTrigger - } - - val notNewLine: Pattern = not(newline).many1 - lazy val CODE: State = state.define("Code") - - ROOT || code.inlinePattern || code.onPushingInline(currentMatch) - CODE || newline || { state.end(); state.begin(NEWLINE) } - CODE || notNewLine || code.onPushingMultiline(currentMatch) - CODE || eof || { state.end(); documentation.onEOF() } - - ////////////////////////////////////////////////////////////////////////////// - //// Formatter /////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** formatter - used to manage text formatters. - * - * stack - holds applied formatters until they're closed. - */ - final object formatter { - var stack: List[Elem.Formatter.Type] = Nil - - def onPushing(typ: Elem.Formatter.Type): Unit = - logger.trace { - val unclosedFormattersToCheck = decideWhichToCheckIfUnclosed(typ) - if (stack.contains(typ)) { - unclosedFormattersToCheck.foreach(checkForUnclosed) - val listOfFormattedAST: List[Elem] = getElemsFromStack(typ) - result.pop() - result.current = Some(Elem.Formatter(typ, listOfFormattedAST)) - stack = stack.tail - result.push() - } else { - addEmptyToStack(typ) - } - } - - def getElemsFromStack(typ: Elem.Formatter.Type): List[Elem] = - logger.trace { - var listOfFormattedAST: List[Elem] = Nil - while ( - result.stack.nonEmpty && result.stack.head != Elem.Formatter(typ) - ) { - result.pop() - result.current match { - case Some(value) => listOfFormattedAST +:= value - case _ => - } - } - listOfFormattedAST - } - - def addEmptyToStack(typ: Elem.Formatter.Type): Unit = - logger.trace { - stack +:= typ - result.current = Some(Elem.Formatter(typ)) - result.push() - } - - def decideWhichToCheckIfUnclosed( - typ: Elem.Formatter.Type - ): List[Elem.Formatter.Type] = - logger.trace { - typ match { - case Elem.Formatter.Strikeout => - List(Elem.Formatter.Bold, Elem.Formatter.Italic) - case Elem.Formatter.Italic => - List(Elem.Formatter.Bold, Elem.Formatter.Strikeout) - case Elem.Formatter.Bold => - List(Elem.Formatter.Italic, Elem.Formatter.Strikeout) - case _ => throw new Error("Trying to use non-existing formatter") - } - } - - def checkForUnclosed(typ: Elem.Formatter.Type): Unit = - logger.trace { - if (stack.nonEmpty) { - if (stack.head == typ) { - val listOfFormattedAST: List[Elem] = getElemsFromStack(typ) - result.pop() - result.current = - Some(Elem.Formatter.Unclosed(typ, listOfFormattedAST)) - stack = stack.tail - result.push() - } - } - } - - val boldTrigger: Char = Elem.Formatter.Bold.marker - val italicTrigger: Char = Elem.Formatter.Italic.marker - val strikeoutTrigger: Char = Elem.Formatter.Strikeout.marker - } - - ROOT || formatter.boldTrigger || formatter - .onPushing(Elem.Formatter.Bold) - - ROOT || formatter.italicTrigger || formatter - .onPushing(Elem.Formatter.Italic) - - ROOT || formatter.strikeoutTrigger || formatter - .onPushing(Elem.Formatter.Strikeout) - - ////////////////////////////////////////////////////////////////////////////// - //// Header ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** header - used to create section headers in Documentation. */ - final object header { - def create(): Unit = - logger.trace { - section.current match { - case Some(_) => loopThroughStackToFindHeader() - case None => - result.pop() - result.current match { - case Some(_: Section.Header) => loopThroughStackToFindHeader() - case _ => result.push() - } - } - } - - def loopThroughStackToFindHeader(): Unit = - logger.trace { - var listForHeader: List[Elem] = Nil - do { - result.pop() - listForHeader +:= result.current.get - } while (result.current.get != Elem.Newline && result.stack.nonEmpty) - if (result.current.get == Elem.Newline) { - result.push() - listForHeader = listForHeader.tail - } - result.current = Some(Section.Header(listForHeader.reverse)) - result.push() - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Links /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** link - used to create links in Documentation. - * - * there are 2 possible link types - Image and normal URL. - */ - final object link { - def onCreatingURL(): Unit = - logger.trace { - if (currentMatch.contains("]") && currentMatch.contains("(")) { - val in = currentMatch.substring(1).dropRight(1).split(']') - val name = in(0) - val url = in(1).substring(1) - pushURL(name, url) - } else { - onInvalidLink() - } - } - - def pushURL(name: String, url: String): Unit = - logger.trace { - result.current = Some(Elem.Link.URL(name, url)) - result.push() - } - - def onCreatingImage(): Unit = - logger.trace { - if (currentMatch.contains("]") && currentMatch.contains("(")) { - val in = currentMatch.substring(2).dropRight(1).split(']') - val name = in(0) - val url = in(1).substring(1) - pushImage(name, url) - } else { - onInvalidLink() - } - } - - def pushImage(name: String, url: String): Unit = - logger.trace { - result.current = Some(Elem.Link.Image(name, url)) - result.push() - } - - def onInvalidLink(): Unit = - logger.trace { - result.current = Some(Elem.Link.Invalid(currentMatch)) - result.push() - } - - def onInvalidLinkNewline(): Unit = - logger.trace { - result.current = Some(Elem.Link.Invalid(currentMatch.dropRight(1))) - result.push() - indent.onPushingNewLine() - } - - def onInvalidLinkEOF(): Unit = - logger.trace { - onInvalidLink() - documentation.onEOF() - } - - val urlNameTrigger: String = "[" - val imageNameTrigger: String = Elem.Link.Image().marker.get + urlNameTrigger - val imagePattern: Pattern = imageNameTrigger >> not(')').many1 >> ')' - val urlPattern: Pattern = urlNameTrigger >> not(')').many1 >> ')' - val invalidPatternNewline: Pattern = - (imageNameTrigger | urlNameTrigger) >> not( - ')' - ).many1 >> newline - val invalidPatternEOF: Pattern = (imageNameTrigger | urlNameTrigger) >> not( - ')' - ).many1 >> eof - } - - ROOT || link.imagePattern || link.onCreatingImage() - ROOT || link.urlPattern || link.onCreatingURL() - ROOT || link.invalidPatternNewline || link.onInvalidLinkNewline() - ROOT || link.invalidPatternEOF || link.onInvalidLinkEOF() - - ////////////////////////////////////////////////////////////////////////////// - //// Indent Management & New line //////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** indent - used to manage text and block indentation. - * - * stack - holds indents for code blocks and lists. - */ - final object indent { - var stack: List[Int] = Nil - def current: Int = - stack match { - case h :: _ => h - case Nil => 0 - } - - def onIndent(): Unit = - logger.trace { - val diff = currentMatch.length - current - if (list.isInList) { - if (diff > list.minIndent) { - tryToFindCodeInStack() - stack +:= currentMatch.length - state.begin(CODE) - } else { - text.push(currentMatch) - } - } else { - if ( - currentMatch.length > section.currentIndentRaw && result.stack.nonEmpty - ) { - tryToFindCodeInStack() - stack +:= currentMatch.length - state.begin(CODE) - } else { - section.currentIndentRaw = currentMatch.length - } - } - } - - def tryToFindCodeInStack(): Unit = - logger.trace { - result.pop() - result.stack.head match { - case _: Elem.CodeBlock => - case _ => - result.push() - val dummyLine = Elem.CodeBlock.Line(0, "") - result.current = Some(Elem.CodeBlock(dummyLine)) - } - result.push() - } - - def onIndentForListCreation(indent: Int, typ: Elem.List.Type): Unit = - logger.trace { - val diff = indent - list.current - - if (!list.isInList) { - section.checkForUnclosedFormattersOnEOS() - onPushingNewLine() - - stack +:= indent - list.startNewList(indent, typ) - } else if (diff == 0) { - list.endListItem() - list.startListItem() - } else if (diff > 0) { - stack +:= indent - list.endListItem() - list.startNewList(indent, typ) - } else { - if (indent > list.prev) { - list.endListItem() - list.startMisalignedListItem(indent, typ) - } else { - do { - list.endListItem() - list.endSublist() - } while (indent < list.current) - list.startListItem() - } - } - } - - def onPushingNewLine(): Unit = - logger.trace { - result.current = Some(Elem.Newline) - result.push() - } - - def onEmptyLine(): Unit = - logger.trace { - if (list.isInList) { - list.endListItem(endList = true) - } - onPushingNewLine() - section.onEOS() - } - - def onIndentPattern(): Unit = - logger.trace { - state.end() - if (result.stack.nonEmpty) { - indent.onPushingNewLine() - } - indent.onIndent() - } - - def onEOFPattern(): Unit = - logger.trace { - state.end() - indent.onPushingNewLine() - documentation.onEOF() - } - - val emptyLine: Pattern = whitespace.opt >> newline - val indentPattern: Pattern = whitespace.opt.many - val EOFPattern: Pattern = indentPattern >> eof - } - - lazy val NEWLINE: State = state.define("Newline") - - ROOT || newline || state.begin(NEWLINE) - NEWLINE || indent.EOFPattern || indent.onEOFPattern() - NEWLINE || indent.indentPattern || indent.onIndentPattern() - - ////////////////////////////////////////////////////////////////////////////// - //// Lists /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** list - used to create lists for documentation. - * - * there are 2 possible types of lists - ordered and unordered. - */ - final object list { - - /** The minimum list indentation consisting of a list symbol and a space - * character. - */ - val minIndent: Int = 2 - - var stack: List[Int] = Nil - def current: Int = - stack match { - case h :: _ => h - case Nil => 0 - } - def prev: Int = - stack match { - case _ :: p :: _ => p - case _ => 0 - } - - def isInList: Boolean = stack.nonEmpty - - def startNewList(indent: Int, listType: Elem.List.Type): Unit = - logger.trace { - list.stack +:= indent - result.current = Some(Elem.List.empty(indent, listType)) - result.push() - } - - def startListItem(): Unit = - logger.trace { - result.pop() - result.current match { - case Some(l: Elem.List) => - result.current = Some(l.addItem()) - result.push() - case elem => - throw new IllegalStateException( - s"Illegal startListItem state [current=$elem]" - ) - } - } - - def startMisalignedListItem(indent: Int, typ: Elem.List.Type): Unit = - logger.trace { - result.pop() - result.current match { - case Some(l: Elem.List) => - val item = Elem.MisalignedItem(indent, typ, List()) - result.current = Some(l.addItem(item)) - result.push() - case elem => - throw new IllegalStateException( - s"Illegal startMisalignedListItem state [current=$elem]" - ) - } - } - - def endListItem(endList: Boolean = false): Unit = - logger.trace { - section.checkForUnclosedFormattersOnEOS() - val elems = stackUnwind() - result.current match { - case Some(l: Elem.List) => - if (endList) { - list.stack = list.stack.tail - } - result.current = Some(l.append(elems)) - result.push() - case elem => - throw new IllegalStateException( - s"Illegal endListItem state [current=$elem]" - ) - } - } - - def endSublist(): Unit = - logger.trace { - result.current match { - case None => - result.pop() - result.current match { - case Some(sublist: Elem.List) => - result.pop() - result.current match { - case Some(l: Elem.List) => - list.stack = list.stack.tail - result.current = Some(l.addItem(sublist)) - result.push() - case elem => - throw new IllegalStateException( - s"Illegal endSublist stack [List,$elem,...]" - ) - } - case elem => - throw new IllegalStateException( - s"Illegal endSublist stack [$elem,...]" - ) - } - case elem => - throw new IllegalStateException( - s"Illegal endSublist state [current=$elem]" - ) - } - } - - def addLastItem(): Unit = - logger.trace { - val elems = stackUnwind() - result.current match { - case Some(l: Elem.List) => - list.stack = list.stack.tail - if (elems.last == Elem.Newline) { - result.current = Some(l.append(elems.init)) - result.push() - result.current = Some(Elem.Newline) - result.push() - } else { - result.current = Some(l.append(elems)) - result.push() - } - while (stack.nonEmpty) { - list.endListItem() - list.endSublist() - } - case elem => - throw new IllegalStateException( - s"Illegal addLastItem state [current=$elem]" - ) - } - } - - /** Get all elements from the stack that were added after the [[Elem.List]] - * node was pushed. - */ - def stackUnwind(): List[Elem] = { - @scala.annotation.tailrec - def go(elems: List[Elem]): List[Elem] = { - result.pop() - result.current match { - case Some(_: Elem.List) | None => - elems - case Some(elem) => - go(elem :: elems) - } - } - - val init = result.current.toList - go(init) - } - - def onOrdered(): Unit = - logger.trace { - state.end() - val listIndent = currentMatch - .takeWhile(_ != orderedListTrigger) - .length - indent.onIndentForListCreation(listIndent, Elem.List.Ordered) - } - - def onUnordered(): Unit = - logger.trace { - state.end() - val listIndent = currentMatch - .takeWhile(_ != unorderedListTrigger) - .length - indent.onIndentForListCreation(listIndent, Elem.List.Unordered) - } - - val orderedListTrigger: Char = Elem.List.Ordered.marker - val unorderedListTrigger: Char = Elem.List.Unordered.marker - - val orderedPattern: Pattern = - indent.indentPattern >> orderedListTrigger >> space - val unorderedPattern: Pattern = - indent.indentPattern >> unorderedListTrigger >> space - } - - NEWLINE || list.orderedPattern || list.onOrdered() - NEWLINE || list.unorderedPattern || list.onUnordered() - - ////////////////////////////////////////////////////////////////////////////// - //// Section ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** section - used to manage sections in Documentation. - * - * there are 2 possible types of sections - marked and raw. - * there are 3 possible types of marked sections: - * - important - * - info - * - example - * - * stack - holds every section in document. - * current - holds current section type. - * currentIndentRaw - holds indent for Raw. - * indentBeforeM & indentAfterM - holds appropriate indents for Marked. - */ - final object section { - var stack: List[Section] = Nil - var current: Option[Section.Marked.Type] = None - var currentIndentRaw: Int = 0 - var indentBeforeMarker: Int = 0 - var indentAfterMarker: Int = 0 - - //// Section Beginning ///// - def onNew(typ: Option[Section.Marked.Type]): Unit = - logger.trace { - result.pop() - current = typ - } - - def onNewMarked(typ: Section.Marked.Type): Unit = - logger.trace { - createMarkedSectionIndent(typ) - onNew(Some(typ)) - currentIndentRaw += currentMatch.length - } - - def createMarkedSectionIndent(typ: Section.Marked.Type): Unit = - logger.trace { - /* NOTE - * We are adding here '_' in front and end in case there was no - * indent on one side or another, and then remove this added char - * from calculation. - * We also add currentIndentRaw as for some reason - * it may be the left indent - */ - val in = "_" + currentMatch + "_" - val inArr = in.split(typ.marker) - indentBeforeMarker = currentIndentRaw + inArr.head.length - 1 - indentAfterMarker = inArr.tail.head.length - 1 - } - - def onNewRaw(): Unit = - logger.trace { - indent.onEmptyLine() - onNew(None) - } - - def onNewRawWithHeader(): Unit = - logger.trace { - state.end() - onNewRaw() - result.current = Some(Section.Header()) - result.push() - } - - def isBeginning(): Boolean = - logger.trace { - result.stack.isEmpty || result.stack.head.isInstanceOf[Section.Header] - } - - //// End of Section //// - def checkForUnclosedFormattersOnEOS(): Unit = - logger.trace { - formatter.checkForUnclosed(Elem.Formatter.Bold) - formatter.checkForUnclosed(Elem.Formatter.Italic) - formatter.checkForUnclosed(Elem.Formatter.Strikeout) - } - - def checkForUnclosedListsOnEOS(): Unit = - if (list.isInList) { - list.addLastItem() - } - - def reverseStackOnEOS(): Unit = - logger.trace { - result.stack = result.stack.reverse - } - - def push(): Unit = - logger.trace { - result.stack match { - case Nil => - /* NOTE - * We don't want to push an empty section into stack - * in case of parsing for example empty file - * Then we want to get back Doc(None) and not Doc(Section()) - */ - case _ => - section.current match { - case Some(marker) => - section.stack +:= Section.Marked( - indentBeforeMarker, - indentAfterMarker, - marker, - result.stack - ) - case None => - section.stack +:= Section.Raw(currentIndentRaw, result.stack) - } - } - } - - def pushReadySection(s: Section): Unit = section.stack = section.stack :+ s - - def pop(): Option[Section] = - logger.trace { - section.stack match { - case ::(head, next) => { - section.stack = next - Some(head) - } - case Nil => None - } - } - - def cleanupOnEOS(): Unit = - logger.trace { - result.current = None - result.stack = Nil - formatter.stack = Nil - } - - def onEOS(): Unit = - logger.trace { - checkForUnclosedFormattersOnEOS() - checkForUnclosedListsOnEOS() - reverseStackOnEOS() - header.create() - push() - cleanupOnEOS() - } - - val importantTrigger: Char = Section.Marked.Important.marker - val infoTrigger: Char = Section.Marked.Info.marker - val exampleTrigger: Char = Section.Marked.Example.marker - - val importantPattern: Pattern = - indent.indentPattern >> importantTrigger >> indent.indentPattern - val infoPattern: Pattern = - indent.indentPattern >> infoTrigger >> indent.indentPattern - val examplePattern: Pattern = - indent.indentPattern >> exampleTrigger >> indent.indentPattern - } - - NEWLINE || indent.emptyLine || section.onNewRaw() - - NEWLINE || indent.emptyLine >> indent.emptyLine || section - .onNewRawWithHeader() - - ROOT || section.importantPattern || section - .onNewMarked(Section.Marked.Important) - - ROOT || section.infoPattern || section - .onNewMarked(Section.Marked.Info) - - ROOT || section.examplePattern || section - .onNewMarked(Section.Marked.Example) - - ////////////////////////////////////////////////////////////////////////////// - //// Documentation /////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** documentation - used to manage every action in case of end of file. - * - * Prepares data to be ready to output to user, also depicts type of - * documentation - is it invoked from Parser as Multi Line or Single Line or - * is it just ran as DocParser - for example in test suite. - */ - final object documentation { - def reverseSectionsStackOnEOF(): Unit = - logger.trace { - section.stack = section.stack.reverse - val baseIndent = - if (section.stack.nonEmpty) section.stack.head.indent else 0 - if (section.stack.length > 2) { - transformOverlyIndentedRawIntoCode(baseIndent) - } - } - - def transformOverlyIndentedRawIntoCode( - baseIndent: Int - ): Unit = { - var newStack = List[Section]() - var currentIndent = baseIndent - while (section.stack.nonEmpty) { - var current = section.pop().get - if ( - current.indent > currentIndent && - current.isInstanceOf[Section.Raw] - ) { - val codeIndent = current.indent - var stackOfCodeSections: List[Section] = List(current) - while ( - section.stack.nonEmpty && - section.stack.head.indent >= codeIndent && - section.stack.head.isInstanceOf[Section.Raw] - ) { - current = section.pop().get - stackOfCodeSections :+= current - } - val codeLines = stackOfCodeSections.flatMap { - _.repr - .build() - .split("\n") - .map { line => - val (indent, text) = line.span(_ == ' ') - Doc.Elem.CodeBlock.Line(indent.length, text) - } - } - if (codeLines.nonEmpty) { - val l1CodeLines = List1(codeLines.head, codeLines.tail) - val codeBlock = Doc.Elem.CodeBlock(l1CodeLines) - val newElems = newStack.head.elems :+ codeBlock - val newSection = newStack.head match { - case marked: Doc.Section.Marked => - marked.copy(elems = newElems) - case raw: Doc.Section.Raw => - raw.copy(elems = newElems) - } - newStack = newStack.drop(1) - newStack +:= newSection - } - } else { - currentIndent = current.indent - newStack +:= current - } - } - section.stack = newStack.reverse - } - - def reverseTagsStackOnEOF(): Unit = - logger.trace { - tags.stack = tags.stack.reverse - } - - def createDoc(): Unit = - logger.trace { - val tags: Option[Tags] = createTags() - val synopsis: Option[Synopsis] = createSynopsis() - val body: Option[Body] = createBody() - result.doc = Some(Doc(tags, synopsis, body)) - } - - def createTags(): Option[Tags] = - logger.trace { - tags.stack match { - case Nil => None - case x :: xs => Some(Tags(List1(x, xs))) - } - } - - def createSynopsis(): Option[Synopsis] = - logger.trace { - section.stack match { - case Nil => None - case x :: _ => Some(Synopsis(x)) - } - } - - def createBody(): Option[Body] = - logger.trace { - section.stack match { - case Nil => None - case _ :: xs => - xs match { - case Nil => None - case y :: ys => Some(Body(List1(y, ys))) - } - } - } - - def onEOF(): Unit = - logger.trace { - section.onEOS() - reverseSectionsStackOnEOF() - reverseTagsStackOnEOF() - createDoc() - } - - def isBeginning(): Boolean = - logger.trace { - result.stack.isEmpty && section.stack.isEmpty - } - } - - ROOT || eof || documentation.onEOF() - ROOT || any || text.push(currentMatch) -} diff --git a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/ParserDef.scala b/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/ParserDef.scala deleted file mode 100644 index 2aa52d44403b..000000000000 --- a/lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/ParserDef.scala +++ /dev/null @@ -1,878 +0,0 @@ -package org.enso.syntax.text.spec - -import org.enso.flexer -import org.enso.flexer.Reader -import org.enso.flexer.State -import org.enso.flexer.automata.Pattern -import org.enso.flexer.automata.Pattern._ -import org.enso.syntax.text.{AST, Shape} - -import scala.annotation.tailrec - -case class ParserDef() extends flexer.Parser[AST.Module] { - import ParserDef2._ - - final def unwrap[T](opt: Option[T]): T = - opt match { - case None => throw new Error("Internal Error") - case Some(t) => t - } - - ///////////// - //// API //// - ///////////// - - override def run(input: Reader): Result[AST.Module] = { - state.begin(block.MODULE) - super.run(input) - } - - /////////////////////////////////// - //// Basic Char Classification //// - /////////////////////////////////// - - val lowerLetter: Pattern = range('a', 'z') - val upperLetter: Pattern = range('A', 'Z') - val digit: Pattern = range('0', '9') - val alphaNum: Pattern = digit | lowerLetter | upperLetter - val space: Pattern = ' '.many1 - val newline: Pattern = '\n' - val emptyLine: Pattern = ' '.many >> newline - - //////////////// - //// Result //// - //////////////// - - override def getResult() = - result.current.flatMap { - case AST.Module.any(mod) => Some(mod) - case _ => None - } - - final object result { - - var current: Option[AST] = None - var stack: List[Option[AST]] = Nil - - def push(): Unit = - logger.trace { - logger.log(s"Pushed: $current") - stack +:= current - current = None - } - - def pop(): Unit = - logger.trace { - current = stack.head - stack = stack.tail - logger.log(s"New result: ${current.map(_.show()).getOrElse("None")}") - } - - def app(fn: String => AST): Unit = - app(fn(currentMatch)) - - def app(ast: AST): Unit = - logger.trace { - current = Some(current match { - case None => ast - case Some(r) => AST.App.Prefix(r, off.use(), ast) - }) - } - - def last(): Option[AST] = { - @tailrec - def go(ast: AST): AST = - ast match { - case AST.App.Prefix.any(t) => go(t.arg) - case t => t - } - current.map(go) - } - - def takeLast(): Option[AST] = { - current.map(c => { - val app = c.asInstanceOf[AST.App.Prefix] - current = Some(app.func) - app.arg - }) - } - } - - //////////////// - //// Offset //// - //////////////// - - final object off { - var current: Int = 0 - var stack: List[Int] = Nil - - def push(): Unit = - logger.trace { - stack +:= current - current = 0 - } - - def pop(): Unit = - logger.trace { - current = stack.head - stack = stack.tail - logger.log(s"New offset: $current") - } - - def use(): Int = - logger.trace { - val offset = current - current = 0 - offset - } - - def on(): Unit = on(0) - - def on(shift: Int): Unit = - logger.trace { - val diff = currentMatch.length + shift - current += diff - logger.log(s"lastOffset + $diff = $current") - } - } - - //////////////////// - //// IDENTIFIER //// - //////////////////// - - final object ident { - - var current: Option[AST.Ident] = None - - def on(cons: String => AST.Ident): Unit = - logger.trace_ { - on(cons(currentMatch)) - } - - def on(ast: AST.Ident): Unit = - logger.trace { - current = Some(ast) - state.begin(SFX_CHECK) - } - - def submit(): Unit = - logger.trace { - result.app(unwrap(current)) - current = None - } - - def onErrSfx(): Unit = - logger.trace { - val ast = AST.Ident.InvalidSuffix(unwrap(current), currentMatch) - result.app(ast) - current = None - state.end() - } - - def onNoErrSfx(): Unit = - logger.trace { - submit() - state.end() - } - - def finalizer(): Unit = - logger.trace { - if (current.isDefined) submit() - } - - val char: Pattern = alphaNum | '_' - val body: Pattern = char.many >> '\''.many - val _var: Pattern = lowerLetter >> body - val cons: Pattern = upperLetter >> body - val annotation: Pattern = "@" >> cons - val breaker: String = "^`!@#$%^&*()-=+[]{}|;:<>,./ \t\r\n\\" - val errSfx: Pattern = noneOf(breaker).many1 - - val SFX_CHECK = state.define("Identifier Suffix Check") - } - - ROOT || ident._var || ident.on(AST.Var(_)) - ROOT || ident.cons || ident.on(AST.Cons(_)) - ROOT || "_" || ident.on(AST.Blank()) - ROOT || ident.annotation || ident.on(AST.Annotation(_)) - ident.SFX_CHECK || ident.errSfx || ident.onErrSfx() - ident.SFX_CHECK || always || ident.onNoErrSfx() - - ////////////////// - //// Operator //// - ////////////////// - - final object opr { - def on(cons: String => AST.Ident): Unit = - logger.trace { - on(cons(currentMatch)) - } - - def onGrp(cons: String => AST.Ident): Unit = - logger.trace { - ident.current = Some(cons(currentMatch)) - ident.onNoErrSfx() - } - - def onNoMod(cons: String => AST.Ident): Unit = - logger.trace { - onNoMod(cons(currentMatch)) - } - - def on(ast: AST.Ident): Unit = - logger.trace { - ident.current = Some(ast) - state.begin(MOD_CHECK) - } - - def onNoMod(ast: AST.Ident): Unit = - logger.trace { - ident.current = Some(ast) - state.begin(SFX_CHECK) - } - - def onMod(): Unit = - logger.trace { - val opr = AST.Mod(unwrap(ident.current).asInstanceOf[AST.Opr].name) - ident.current = Some(opr) - } - - def onDottedOpr(): Unit = { - logger.trace { - ident.current = Some(AST.Ident.Opr(".")) - ident.submit() - ident.current = Some(AST.Ident.Var(currentMatch.substring(1))) - ident.submit() - } - } - - val char: Pattern = anyOf(";!$%&*+-/<>?^~|:\\") - val errChar: Pattern = char | "=" | "," | "." - val errSfx: Pattern = errChar.many1 - val body: Pattern = char.many1 - val opsEq: Pattern = "=" | "==" | ">=" | "<=" | "/=" | "#=" | "!=" - val dot: Pattern = "." - val dottedOps: Pattern = dot >> (body | opsEq) - val opsDot: Pattern = dot | ".." | "..." | "," - val opsGrp: Pattern = anyOf("()[]{}") - val opsCmm: Pattern = "#" | "##" - val opsNoMod: Pattern = opsEq | opsDot | opsCmm - - val SFX_CHECK = state.define("Operator Suffix Check") - val MOD_CHECK = state.define("Operator Modifier Check") - MOD_CHECK.parent = SFX_CHECK - } - - ROOT || opr.body || opr.on(AST.Opr(_)) - ROOT || opr.dottedOps || opr.onDottedOpr() - ROOT || opr.opsNoMod || opr.onNoMod(AST.Opr(_)) - ROOT || opr.opsGrp || opr.onGrp(AST.Opr(_)) - opr.MOD_CHECK || "=" || opr.onMod() - opr.SFX_CHECK || opr.errSfx || ident.onErrSfx() - opr.SFX_CHECK || always || ident.onNoErrSfx() - - //////////////// - //// NUMBER //// - //////////////// - - final object num { - - var part1: String = "" - var part2: String = "" - - def reset(): Unit = - logger.trace { - part1 = "" - part2 = "" - } - - def submit(): Unit = - logger.trace { - val base = if (part1 == "") None else Some(part1) - result.app(AST.Number(base, part2)) - reset() - } - - def onDanglingBase(): Unit = - logger.trace { - state.end() - result.app(AST.Number.DanglingBase(part2)) - reset() - } - - def onDecimal(): Unit = - logger.trace { - part2 = currentMatch - state.begin(PHASE2) - } - - def onExplicitBase(): Unit = - logger.trace { - state.end() - part1 = part2 - part2 = currentMatch.substring(1) - submit() - } - - def onNoExplicitBase(): Unit = - logger.trace { - state.end() - submit() - } - - val decimal: Pattern = digit.many1 - - val PHASE2: State = state.define("Number Phase 2") - } - - ROOT || num.decimal || num.onDecimal() - num.PHASE2 || "_" >> alphaNum.many1 || num.onExplicitBase() - num.PHASE2 || "_" || num.onDanglingBase() - num.PHASE2 || always || num.onNoExplicitBase() - - ////////////// - //// Text //// - ////////////// - - class TextState( - var offset: Int, - var spaces: Int, - var lines: List[AST.Text.Block.Line[AST.Text.Segment.Fmt]], - var emptyLines: List[Int], - var lineBuilder: List[AST.Text.Segment.Fmt] - ) - - final object text { - - import AST.Text.Block.Line - - val Segment = AST.Text.Segment - - var stack: List[TextState] = Nil - var text: TextState = _ - - def push(): Unit = - logger.trace { - stack +:= text - } - - def pop(): Unit = - logger.trace { - text = stack.head - stack = stack.tail - } - - def onInvalidQuote(): Unit = - logger.trace { - result.app(AST.Text.InvalidQuote(currentMatch)) - } - - def onInlineBlock(): Unit = - logger.trace { - result.app(AST.Text.InlineBlock(currentMatch)) - } - - def finish( - raw: List[Line[Segment.Raw]] => AST, - fmt: List[Line[Segment.Fmt]] => AST - ): Unit = - logger.trace { - submitLine() - val isFMT = state.current.parent.contains(FMT) - val body = text.lines.reverse - val t = - if (isFMT) fmt(body) - else raw(body.asInstanceOf[List[Line[Segment.Raw]]]) - pop() - state.end() - result.app(t) - } - - def submit(segment: Segment.Fmt): Unit = - logger.trace { - text.lineBuilder +:= segment - } - - def submit(): Unit = - logger.trace { - finish( - t => AST.Text.Raw(t.head.text: _*), - t => AST.Text(t.head.text: _*) - ) - } - - def submitMissingQuote(): Unit = - logger.trace { - rewind() - submitUnclosed() - } - - def submitInvalidQuote(): Unit = - logger.trace { - submitUnclosed() - onInvalidQuote() - } - - def submitUnclosed(): Unit = - logger.trace { - val Text = AST.Text.Unclosed - finish(t => Text.Raw(t.head.text: _*), t => Text(t.head.text: _*)) - } - - def onEndOfBlock(): Unit = - logger.trace { - if (text.lineBuilder.isEmpty) - block.emptyLines = text.emptyLines ++ block.emptyLines - val (s, o) = (text.spaces, text.offset) - finish(t => AST.Text.Raw(s, o, t: _*), t => AST.Text(s, o, t: _*)) - off.push() - rewind() - } - - def onBegin(grp: State): Unit = - logger.trace { - push() - state.begin(grp) - text = new TextState(0, 0, Nil, Nil, Nil) - } - - def onBeginBlock(grp: State): Unit = - logger.trace { - val offset = if (state.current == block.FIRSTCHAR) { - state.end() - block.current.offset - } else - OFFSET_OF_FIRST_LINE_FOUND - if (currentMatch.last == '\n') { - onBegin(grp) - text.offset = offset - text.spaces = currentMatch.length - BLOCK_QUOTE_SIZE - 1 - state.begin(NEWLINE) - } else { - val spaces = currentMatch.length - BLOCK_QUOTE_SIZE - result.app( - if (grp == FMT_BLCK) AST.Text(spaces = spaces, offset) - else AST.Text.Raw(spaces = spaces, offset) - ) - onEOF() - } - } - - def submitPlainSegment(): Unit = - logger.trace { - text.lineBuilder = text.lineBuilder match { - case Shape.SegmentPlain(t) :: _ => - Segment.Plain(t + currentMatch) :: text.lineBuilder.tail - case _ => Segment.Plain(currentMatch) :: text.lineBuilder - } - } - - def onEscape(code: Segment.Escape): Unit = - logger.trace { - submit(Shape.SegmentEscape(code)) - } - - def onEscape(code: Segment.RawEscape): Unit = - logger.trace { - submit(Shape.SegmentRawEscape(code)) - } - - def onEscapeU21(): Unit = - logger.trace { - val code = currentMatch.drop(3).dropRight(1) - onEscape(Segment.Escape.Unicode.U21(code)) - } - - def onEscapeU16(): Unit = - logger.trace { - val code = currentMatch.drop(2) - onEscape(Segment.Escape.Unicode.U16(code)) - } - - def onEscapeU32(): Unit = - logger.trace { - val code = currentMatch.drop(2) - onEscape(Segment.Escape.Unicode.U32(code)) - } - - def onEscapeInt(): Unit = - logger.trace { - val int = currentMatch.drop(1).toInt - onEscape(Segment.Escape.Number(int)) - } - - def onEscapeInvalid(): Unit = - logger.trace { - val chr = currentMatch.charAt(1) - onEscape(Segment.Escape.Invalid(chr)) - } - - def onEscapeUnfinished(): Unit = - logger.trace { - onEscape(Segment.Escape.Unfinished) - } - - def onEscapeSlash(): Unit = - logger.trace { - onEscape(Segment.Escape.Slash) - } - def onEscapeFormatQuote(): Unit = logger.trace { - onEscape(Segment.Escape.Quote) - } - - def onEscapeQuote(): Unit = - logger.trace { - onEscape(Segment.Escape.Quote) - } - - def onEscapeRawQuote(): Unit = - logger.trace { - onEscape(Segment.Escape.RawQuote) - } - - def onInterpolateBegin(): Unit = - logger.trace { - result.push() - off.push() - state.begin(INTERPOLATE) - } - - def onInterpolateEnd(): Unit = - logger.trace { - if (state.isInside(INTERPOLATE)) { - state.endTill(INTERPOLATE) - submit(Segment.Expr(result.current)) - result.pop() - off.pop() - state.end() - } else { - onUnrecognized() - } - } - - def submitLine(): Unit = - logger.trace { - if ( - state.current == FMT_LINE || state.current == RAW_LINE || text.lineBuilder.nonEmpty - ) { - val Line = Shape.TextBlockLine - text.lines +:= Line(text.emptyLines.reverse, text.lineBuilder.reverse) - text.lineBuilder = Nil - text.emptyLines = Nil - } - } - - def onEndOfLine(): Unit = - logger.trace { - state.begin(NEWLINE) - submitLine() - } - - def onNewLine(): Unit = - logger.trace { - state.end() - if (text.offset == OFFSET_OF_FIRST_LINE_FOUND) - text.offset = currentMatch.length - val leadingSpaces = currentMatch.length - text.offset - if (leadingSpaces < 0) { - onEndOfBlock() - state.begin(block.NEWLINE) - } else if (leadingSpaces != 0) - text.lineBuilder +:= Segment.Plain(" " * leadingSpaces) - } - - def onEmptyLine(): Unit = - logger.trace { - text.emptyLines :+= currentMatch.length - 1 - } - - def onEOFNewLine(): Unit = - logger.trace { - state.end() - onEndOfBlock() - state.begin(block.NEWLINE) - } - - val BLOCK_QUOTE_SIZE = 3 - val OFFSET_OF_FIRST_LINE_FOUND = -1 - - val fmtBlock = "'''" >> space.opt >> (eof | newline) - val rawBlock = "\"\"\"" >> space.opt >> (eof | newline) - val fmtChar = noneOf("'`\\\n") - val escapeChar = noneOf("'\\\"`\\\\\\n\\r{}") - val escape_int = "\\" >> num.decimal - val escape_u21 = "\\u{" >> repeat(escapeChar, 0, 8) >> "}" - val escape_u16 = "\\u" >> repeat(escapeChar, 0, 4) - val escape_u32 = "\\U" >> repeat(escapeChar, 0, 8) - val fmtSeg = fmtChar.many1 - val rawSeg = noneOf("\"\n").many1 - val fmtBSeg = noneOf("\n\\`").many1 - val rawBSeg = noneOf("\n").many1 - - val FMT: State = state.define("Formatted Text") - val FMT_LINE: State = state.define("Formatted Line Of Text") - val RAW_LINE: State = state.define("Raw Line Of Text") - val FMT_BLCK: State = state.define("Formatted Block Of Text") - val RAW_BLCK: State = state.define("Raw Block Of Text") - val NEWLINE: State = state.define("Text Newline") - val INTERPOLATE: State = state.define("Interpolate") - INTERPOLATE.parent = ROOT - FMT_LINE.parent = FMT - FMT_BLCK.parent = FMT - } - - ROOT || '`' || text.onInterpolateEnd() - ROOT || "'''" >> "'".many1 || text.onInvalidQuote() - ROOT || "'" || text.onBegin(text.FMT_LINE) - ROOT || text.fmtBlock || text.onBeginBlock(text.FMT_BLCK) - ROOT || "'''" || text.onInlineBlock() - ROOT || "\"\"\"" >> "\"".many1 || text.onInvalidQuote() - ROOT || '"' || text.onBegin(text.RAW_LINE) - ROOT || text.rawBlock || text.onBeginBlock(text.RAW_BLCK) - ROOT || "\"\"\"" || text.onInlineBlock() - - block.FIRSTCHAR || text.fmtBlock || text.onBeginBlock(text.FMT_BLCK) - block.FIRSTCHAR || text.rawBlock || text.onBeginBlock(text.RAW_BLCK) - - text.FMT || '`' || text.onInterpolateBegin() - AST.Text.Segment.Escape.Character.codes.foreach { code => - val char = s"text.Segment.Escape.Character.$code" - text.FMT || s"\\$code" run s"text.onEscape($char)" - } - AST.Text.Segment.Escape.Control.codes.foreach { code => - val ctrl = s"text.Segment.Escape.Control.$code" - text.FMT || s"\\$code" run s"text.onEscape($ctrl)" - } - - text.FMT || text.escape_u21 || text.onEscapeU21() - text.FMT || text.escape_u16 || text.onEscapeU16() - text.FMT || text.escape_u32 || text.onEscapeU32() - text.FMT || text.escape_int || text.onEscapeInt() - text.FMT || "\\'" || text.onEscapeFormatQuote() - text.FMT || "\\\\" || text.onEscapeSlash() - text.FMT || "\\" >> any || text.onEscapeInvalid() - text.FMT || "\\" || text.onEscapeUnfinished() - - text.FMT_LINE || "'" || text.submit() - text.FMT_LINE || "'".many1 || text.submitInvalidQuote() - text.FMT_LINE || text.fmtSeg || text.submitPlainSegment() - text.FMT_LINE || eof || text.submitMissingQuote() - text.FMT_LINE || newline || text.submitMissingQuote() - - text.FMT_BLCK || text.fmtBSeg || text.submitPlainSegment() - text.FMT_BLCK || eof || text.onEndOfBlock() - text.FMT_BLCK || newline || text.onEndOfLine() - - text.RAW_LINE || '"' || text.submit() - text.RAW_LINE || '"'.many1 || text.submitInvalidQuote() - text.RAW_LINE || text.rawSeg || text.submitPlainSegment() - text.RAW_LINE || eof || text.submitMissingQuote() - text.RAW_LINE || newline || text.submitMissingQuote() - - text.RAW_BLCK || text.rawBSeg || text.submitPlainSegment() - text.RAW_BLCK || eof || text.onEndOfBlock() - text.RAW_BLCK || newline || text.onEndOfLine() - - text.NEWLINE || space.opt || text.onNewLine() - text.NEWLINE || space.opt >> newline || text.onEmptyLine() - text.NEWLINE || space.opt >> eof || text.onEOFNewLine() - - ////////////// - /// Blocks /// - ////////////// - - // because of bug in macroContext.eval it cannot be part of object block - class BlockState( - val isOrphan: Boolean, - var isValid: Boolean, - var offset: Int, - var emptyLines: List[Int], - var firstLine: Option[AST.Block.Line.NonEmpty], - var lines: List[AST.Block.OptLine] - ) - - final object block { - - var stack: List[BlockState] = Nil - var emptyLines: List[Int] = Nil - var current: BlockState = new BlockState(false, true, 0, Nil, None, Nil) - - def push(newIndent: Int, orphan: Boolean): Unit = - logger.trace { - stack +:= current - current = - new BlockState(orphan, true, newIndent, emptyLines.reverse, None, Nil) - emptyLines = Nil - } - - def pop(): Unit = - logger.trace { - current = stack.head - stack = stack.tail - } - - def build(): AST.Block = - logger.trace { - submitLine() - AST.Block( - current.isOrphan, - AST.Block.Continuous, - current.offset, - current.emptyLines, - unwrap(current.firstLine), - current.lines.reverse - ) - } - - def submit(): Unit = - logger.trace { - val block = build() - result.pop() - off.pop() - pop() - val block2: AST.Block = result.last() match { - case Some(AST.Opr.any(_)) => - block.replaceType(AST.Block.Discontinuous) - case _ => block - } - result.app(block2) - off.push() - logger.endGroup() - } - - def submitModule(): Unit = - logger.trace { - val body = current.firstLine match { - case None => current.lines.reverse - case Some(line) => line.toOptional +: current.lines.reverse - } - val line :: lines = (current.emptyLines - .map(AST.Block.Line(None, _)): List[AST.Block.OptLine]) ++ body - val module = AST.Module(line, lines) - result.current = Some(module) - logger.endGroup() - } - - def submitLine(): Unit = - logger.trace { - result.current match { - case None => - case Some(r) => - off.pop() - current.firstLine match { - case None => - current.firstLine = Some(AST.Block.Line.Required(r, off.use())) - case Some(_) => - current.lines +:= AST.Block.Line(result.current, off.use()) - } - } - emptyLines.reverse.foreach(current.lines +:= AST.Block.OptLine(_)) - emptyLines = Nil - result.current = None - } - - def onEmptyLine(): Unit = - logger.trace { - off.on(-1) - emptyLines +:= off.use() - } - - def onModuleBegin(): Unit = - logger.trace { - current.emptyLines = emptyLines.reverse - emptyLines = Nil - rewind() - off.push() - state.end() - state.begin(NEWLINE) - } - - def onBegin(newIndent: Int): Unit = - logger.trace { - val isOrphan = result.current.isEmpty - result.push() - push(newIndent, isOrphan) - logger.beginGroup() - } - - def onEOFLine(): Unit = - logger.trace { - state.end() - submitLine() - off.on() - current.lines +:= AST.Block.OptLine(off.use()) - off.pop() - onEOF() - } - - def onEndLine(): Unit = - logger.trace { - off.push() - state.begin(NEWLINE) - } - - def onNewLine(): Unit = - logger.trace { - state.end() - off.on() - if (off.current == current.offset) - submitLine() - else if (off.current > current.offset) - onBegin(off.use()) - else - onEnd(off.use()) - state.begin(FIRSTCHAR) - } - - def onEnd(newIndent: Int): Unit = - logger.trace { - while (newIndent < current.offset) submit() - if (newIndent > current.offset) { - logger.log("Block with invalid indentation") - onBegin(newIndent) - current.isValid = false - } else { - off.push() - submitLine() - } - } - - val MODULE = state.define("Module") - val NEWLINE = state.define("Newline") - val FIRSTCHAR = state.define("First Char") - } - - ROOT || newline || block.onEndLine() - block.NEWLINE || emptyLine || block.onEmptyLine() - block.NEWLINE || space.opt >> eof || block.onEOFLine() - block.NEWLINE || space.opt || block.onNewLine() - block.MODULE || emptyLine || block.onEmptyLine() - block.MODULE || space.opt || block.onModuleBegin() - block.FIRSTCHAR || always || state.end() - - //////////////// - /// Defaults /// - //////////////// - - final def onUnrecognized(): Unit = - logger.trace { - result.app(AST.Invalid.Unrecognized(_)) - } - - final def onEOF(): Unit = - logger.trace { - ident.finalizer() - off.push() - block.submitLine() - block.onEnd(0) - block.submitModule() - } - - ROOT || space || off.on() - ROOT || eof || onEOF() - ROOT || any || onUnrecognized() -} - -object ParserDef2 { - type Result[T] = flexer.Parser.Result[T] -} diff --git a/lib/scala/syntax/specialization/js/src/main/scala/org/enso/syntax/text/Main.scala b/lib/scala/syntax/specialization/js/src/main/scala/org/enso/syntax/text/Main.scala deleted file mode 100644 index d1a031b828bd..000000000000 --- a/lib/scala/syntax/specialization/js/src/main/scala/org/enso/syntax/text/Main.scala +++ /dev/null @@ -1,44 +0,0 @@ -package org.enso.syntax.text - -import io.circe.syntax._ -import org.enso.flexer.Reader -import org.enso.syntax.text.Parser.ParserError - -import scala.scalajs.js.annotation._ - -object Parse { - @JSExportTopLevel("parse") - def parse(program: String, idsJson: String): String = { - val idmap = Parser - .idMapFromJson(idsJson) - .left - .map { error => - throw new ParserError("Could not deserialize idmap.", error) - } - .merge - new Parser().run(new Reader(program), idmap).toJson().noSpacesSortKeys - } - - @JSExportTopLevel("parse_with_metadata") - def parse_with_metadata(program: String): String = { - new Parser().runWithMetadata(program).asJson.noSpacesSortKeys - } - - @JSExportTopLevel("doc_parser_generate_html_source") - def doc_parser_generate_html_source(program: String): String = { - val parser = new Parser() - val module = parser.run(program) - val dropMeta = parser.dropMacroMeta(module) - val doc = docparser.DocParserRunner.createDocs(dropMeta) - val htmlCode = - docparser.DocParserHTMLGenerator.generateHTMLForEveryDocumented(doc) - htmlCode - } - - @JSExportTopLevel("doc_parser_generate_html_from_doc") - def doc_parser_generate_html_from_doc(code: String): String = { - val doc = DocParser.runMatched(code) - val htmlCode = docparser.DocParserHTMLGenerator.generateHTMLPureDoc(doc) - htmlCode - } -} diff --git a/lib/scala/syntax/specialization/lexer-bench/src/bench/java/org/enso/syntax/LexerBench.java b/lib/scala/syntax/specialization/lexer-bench/src/bench/java/org/enso/syntax/LexerBench.java deleted file mode 100644 index 2153da6afdc9..000000000000 --- a/lib/scala/syntax/specialization/lexer-bench/src/bench/java/org/enso/syntax/LexerBench.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.enso.syntax; - -import org.openjdk.jmh.annotations.*; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.Blackhole; - -import java.util.concurrent.TimeUnit; - -@BenchmarkMode(Mode.AverageTime) -@Fork(1) -@Warmup(iterations = 5) -@Measurement(iterations = 10) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -public class LexerBench { - - @State(Scope.Thread) - public static class BenchState { - @Param({"1024" /* 1KB */, "102400" /* 100KB */, "1048576" /* 1MB */, "10485760" /* 10MB */}) - public int bytesSize; - - public String myInput; - - @Setup(Level.Trial) - public void doSetup(BenchmarkParams params) { - var benchNameSegments = params.getBenchmark().split("\\."); - var benchName = benchNameSegments[benchNameSegments.length - 1]; - var benchInput = LexerBenchFixtures.benchmarks().get(benchName).get(); - this.myInput = LexerBenchFixtures.replicate(benchInput, bytesSize, false); - } - } - - - // === Literals === - - @Benchmark - public void literalNumberInteger(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalNumberIntegerExplicitBase(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalNumberDecimal(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalNumberDecimalExplicitBase(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalNumberErrorBase(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalTextFormatLine(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalTextFormatInlineBlock(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalTextFormatBlock(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalTextRawLine(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalRawInlineBlock(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void literalRawBlock(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - - // === Names === - - @Benchmark - public void nameLineOf(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void nameInvalidSuffix(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - - // === Operators === - - @Benchmark - public void operatorLineOf(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void operatorDotCall(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void operatorInvalidSuffix(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - - // === Blocks === - - @Benchmark - public void blockTopLevel(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void blockNested(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void blockDeeplyNested(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - - // === Comments === - - @Benchmark - public void commentLine(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void commentInLine(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void commentDoc(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - - // === Combined === - - @Benchmark - public void combinedSimple(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } - - @Benchmark - public void combinedComplex(Blackhole blackhole, BenchState state) { - blackhole.consume(LexerBenchFixtures.runLexer(state.myInput)); - } -} diff --git a/lib/scala/syntax/specialization/lexer-bench/src/bench/scala/org/enso/syntax/LexerBenchFixtures.scala b/lib/scala/syntax/specialization/lexer-bench/src/bench/scala/org/enso/syntax/LexerBenchFixtures.scala deleted file mode 100644 index 00a6f96a5b73..000000000000 --- a/lib/scala/syntax/specialization/lexer-bench/src/bench/scala/org/enso/syntax/LexerBenchFixtures.scala +++ /dev/null @@ -1,223 +0,0 @@ -package org.enso.syntax - -import java.nio.charset.StandardCharsets - -import org.enso.flexer -import org.enso.flexer.Reader -import org.enso.syntax.text.AST -import org.enso.syntax.text.spec.{ParserDef, ParserDef2} - -object LexerBenchFixtures { - - private val newEngine = flexer.Parser.compile(ParserDef()) - - // === Lexer Runner === - - /** Execute the lexer on the provided `input`. - * - * @param input the source code - * @return the result of lexing `input` - */ - def runLexer(input: String): ParserDef2.Result[AST.Module] = { - val engine = newEngine() - val reader = new Reader(input) - engine.run(reader) - } - - // === Utilities === - - /** Replicate the provided `input` out to the provided `size` in bytes - * (according to utf-8). - * - * @param input the text to replicate - * @param size the size to replicate `input` to - * @param addNewline whether or not a newline should be added after each - * repetition of `input` - * @return `input`, repeated enough times to make the size >= `size` - */ - def replicate( - input: String, - size: Int, - addNewline: Boolean = false - ): String = { - val inputSize = input.getBytes(StandardCharsets.UTF_8).length - val times = 1 + size / inputSize - val inputNewline = if (addNewline) input + "\n" else input + " " - inputNewline.repeat(times) - } - - /** Replace all CRLF line endings in the input by LF. - * - * @param input the input text - * @return `input` with all `\r\n` replaced by `\n` - */ - def preprocess(input: String): String = { - input.replace("\r\n", "\n") - } - - // === Benchmarks === - - val benchmarks: Map[String, String] = Map( - // Literals - ("literalNumberInteger", Literal.Number.integer), - ("literalNumberIntegerExplicitBase", Literal.Number.integerExplicitBase), - ("literalNumberDecimal", Literal.Number.decimal), - ("literalNumberDecimalExplictBase", Literal.Number.decimalExplicitBase), - ("literalNumberErrorBase", Literal.Number.errorBase), - ("literalTextFormatLine", Literal.Text.formatLine), - ("literalTextFormatInlineBlock", Literal.Text.formatInlineBlock), - ("literalTextFormatBlock", Literal.Text.formatBlock), - ("literalTextRawLine", Literal.Text.rawLine), - ("literalTextRawInlineBlock", Literal.Text.rawInlineBlock), - ("literalTextRawBlock", Literal.Text.rawBlock), - // Names - ("nameLineOf", Name.lineOf), - ("nameInvalidSuffix", Name.invalidSuffix), - // Operators - ("operatorLineOf", Operator.lineOf), - ("operatorDotCall", Operator.dotCall), - ("operatorInvalidSuffix", Operator.invalidSuffix), - // Blocks - ("blockTopLevel", Block.topLevel), - ("blockNested", Block.nested), - ("blockDeeplyNested", Block.deeplyNested), - // Comments - ("commentLine", Comment.line), - ("commentInLine", Comment.inLine), - ("commentDoc", Comment.doc), - // Combined - ("combinedSimple", Combined.simple), - ("combinedComplex", Combined.complex) - ) - - // === Inputs === - - object Literal { - object Number { - val integer: String = preprocess("12345") - val integerExplicitBase: String = preprocess("16_a4fd31") - val decimal: String = preprocess("1.3141") - val decimalExplicitBase: String = preprocess("10_1.000999") - val errorBase: String = preprocess("10.2_2") - } - - object Text { - val formatLine: String = - "'dearest creature in \\n creation studying english pronunciation'" - - val formatInlineBlock: String = - "''' An inline block. It's a very good inline block carl \\u{AB}" - - val formatBlock: String = - """''' Here is my block of format text. I can `interpolate + things` like that. - | It goes on and on and on for `times` times because I feel like it. - | - | Complex interpolated expression `x -> y ~> x | y` woo! - |""".stripMargin - - val rawLine: String = - "\"dearest creature in '''' creation studying english pronunciation\"" - - val rawInlineBlock: String = - "\"\"\" An inline block. It's a very good inline block carl " - - val tQuote = "\"\"\"" - val rawBlock: String = - s"""$tQuote Here is my block of raw text. `Interpolations` are nothing special here. - | It goes on and on and on for I can escape \" though. - | - | It also supports blank lines! - |""".stripMargin - } - } - - object Name { - val lineOf: String = - "Referent_Ident var_ident JavaType _ @annotation ticked_ident' number_1" - - val invalidSuffix: String = "some_var'iable some_varД" - } - - object Operator { - val lineOf: String = "+ - * -> ~> <~ <- ! & | /" - val dotCall: String = ".== . != .<*> .*> .|>" - val invalidSuffix: String = ".... +==" - } - - object Block { - val topLevel: String = "foo\nbar\nbaz" - val nested: String = "foo\\nbar\\n baz\\n quux" - val deeplyNested: String = - """foo - |bar - | baz - | quux - | bim - | bam - | oh - |no - |""".stripMargin - } - - object Comment { - val line: String = - "# foo bar baz I have a really long line comment here that goes on and on" - - val inLine: String = "a + b # A useless comment: add a to b" - - val doc: String = - """## I have a really big doc comment here - | That just keeps prattling on and on and on. - | - | With blank lines - | - | Forever - | - | and - | ever - | - | and - | - | - | - | - | ever - |documented - |""".stripMargin - } - - object Combined { - val simple: String = - """ - |import Base.Meta - | - |## Decompose the value using runtime reflection and print its decomposition. - |Main.print_decomp a b = - | y = a + b - | decomp = Meta.decompose y - | Io.println decomp - |""".stripMargin - - val complex: String = - """import Base.Meta - | - |## Frobnicate the doodads by constructing a new type operator through runtime reflection such that - | it can be passed to another language. - | - | ! WARNING - | Type-checking code like this is virtually impossible, and it is treated as `Dynamic` inside - | Enso code. - |Main.foo a b = - | y = x -> z -> - | ty = a.gen_type (~>) (<-) b - | ty (z x) - | decomp = Meta.decompose (y a b) - | Io.println decomp - | - |## Execute the main function of this project. - |main = - | func = Meta.reify (here.foo "My_Name" "my_field") - | Io.println(func) - |""".stripMargin - } -} diff --git a/lib/scala/syntax/specialization/shared/src/bench/scala/org/enso/syntax/DocParserBench.scala b/lib/scala/syntax/specialization/shared/src/bench/scala/org/enso/syntax/DocParserBench.scala deleted file mode 100644 index d9d803f5f7ae..000000000000 --- a/lib/scala/syntax/specialization/shared/src/bench/scala/org/enso/syntax/DocParserBench.scala +++ /dev/null @@ -1,62 +0,0 @@ -package org.enso.syntax - -import org.enso.syntax.text.DocParser -import org.enso.syntax.text.DocParser.Result -import org.enso.syntax.text.ast.Doc -import org.scalameter.api._ -import org.scalameter.execution.LocalExecutor -import org.scalameter.picklers.Implicits._ - -import scala.annotation.nowarn -import scala.math.pow - -object DocParserBench extends Bench.OfflineRegressionReport { - - override def executor = new LocalExecutor(warmer, aggregator, measurer) - - val range = 0 - @nowarn("cat=w-flag-numeric-widen") - def exp(i: Int): Gen[Int] = - Gen.exponential("size")(pow(2, i - range).toInt, pow(2, i).toInt, 2) - - def gen(range: Gen[Int], f: Int => String): Gen[String] = - for { i <- range } yield f(i) - - val tests = List( - "formatters" -> gen(exp(14), i => "*foo bar*\n" * i), - "unclosed" -> gen(exp(14), i => "*_foobar*\n" * i), - "combined" -> gen(exp(14), i => "*_~fo0~_*\n" * i), - "normal" -> gen(exp(14), i => "test12345\n" * i), - "link" -> gen(exp(14), i => "[foo](bo)\n" * i), - "tags" -> gen(exp(14), i => "ADDED\nfoo\n" * i), - "list" -> gen( - exp(13), - i => """foo - | - A - | - B - | - C - |""".stripMargin * i - ), - "list_nested" -> gen( - exp(12), - i => """foo - | - A - | - B - | * CA - | * CB - | - D - |""".stripMargin * i - ), - "sections" -> gen( - exp(13), - i => "Foo\n\n!B\n\n?C \n\n>D \n\n" * i - ) - ) - - def run(str: String): Result[Doc] = DocParser.run(str) - performance of "DocParser" in { - tests.foreach { case (name, gen) => - measure method name in (using(gen) in run) - } - } -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/CommentRemover.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/CommentRemover.scala deleted file mode 100644 index 118065542903..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/CommentRemover.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.enso.syntax.text - -/** A dirty hack that replaces all disable comments with whitespace. - * Documentation comments are left untouched. - * - * @param input the original input file - */ -class CommentRemover(input: String) { - - def run: String = { - var currentState: CommentRemover.State = CommentRemover.Base - val it = input.iterator.buffered - val result = new StringBuilder(input.length) - - while (it.hasNext) { - val char: Char = it.next() - - if (char != '\n' && currentState == CommentRemover.InComment) { - result.addOne(' ') - } else if (currentState == CommentRemover.InTextLine) { - if (char == '\'' || char == '"') { - result.addOne(char) - currentState = CommentRemover.Base - } else if (char == '\\' && it.hasNext && it.head == '\'') { - result.addOne(char) - result.addOne(it.next()) - } else if (char == '\n') { - result.addOne(char) - currentState = CommentRemover.Base - } else { - result.addOne(char) - } - } else if ( - currentState == CommentRemover.Base && (char == '"' || char == '\'') - ) { - currentState = CommentRemover.InTextLine - result.addOne(char) - } else if (currentState == CommentRemover.Base && char == '#') { - if (it.hasNext && it.head == '#') { - result.addOne('#') - currentState = CommentRemover.InDocComment - } else { - result.addOne(' ') - currentState = CommentRemover.InComment - } - } else if (char == '\n') { - result.addOne('\n') - currentState = CommentRemover.Base - } else { - result.addOne(char) - } - } - result.toString() - } -} - -object CommentRemover { - sealed trait State - case object Base extends State - case object InDocComment extends State - case object InComment extends State - case object InTextLine extends State -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/DocParser.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/DocParser.scala deleted file mode 100644 index 4a1fcd031fe5..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/DocParser.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.enso.syntax.text - -import org.enso.flexer -import org.enso.flexer.Reader -import org.enso.syntax.text.ast.Doc -import org.enso.syntax.text.spec.DocParserDef -import flexer.Parser.{Result => res} - -//////////////////////////////////////////////////////////////////////////////// -//// Doc Parser //////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -/** This is the class used to invoke Documentation Parser. - * - * It is used to create structured documentation from the blocks of commented - * text created by the main Enso parser. - * - * It has been built on the same foundation as Parser, so in order not to - * duplicate information, please refer to Parser documentation. - */ -class DocParser { - import DocParser._ - private val engine = newEngine() - private val errMsg = "Internal Documentation Parser Error" - - /** Used to match result of [[run]] function to possibly retrieve Doc - * - * @param input - input string to Doc Parser - * @return - If it was able to retrieve Doc, then retrieved data, else - * exception with error message [[errMsg]] - */ - def runMatched(input: String): Doc = - run(input) match { - case res(_, res.Success(v)) => v - case _ => throw new Exception(errMsg) - } - - /** Used to initialize Doc Parser with input string to get parsed Doc - * - * @param input - input string to Doc Parser - * @return - unmatched result possibly containing Doc - */ - def run(input: String): Result[Doc] = engine.run(new Reader(input)) -} - -object DocParser { - type Result[T] = flexer.Parser.Result[T] - private val newEngine = flexer.Parser.compile(DocParserDef()) - - /** Doc Parser running methods, as described above, in class [[docparser]] - */ - def runMatched(input: String): Doc = { - val lines = input.split("\n") - var indentSecondLine = 0 - if (lines.tail.nonEmpty) { - val s = lines.tail.head - indentSecondLine = s.indexOf(s.trim()) - } - val in = " " * indentSecondLine + lines.mkString("\n") - new DocParser().runMatched(in) - } - def run(input: String): Result[Doc] = new DocParser().run(input) -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/InHoisting.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/InHoisting.scala deleted file mode 100644 index a0ebf9fb5a06..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/InHoisting.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.enso.syntax.text - -import cats.implicits._ -import org.enso.syntax.text.AST.App - -/** This preprocessor step is responsible for hoisting occurrences of `in` as a - * variable identifier to occurrences of an _operator_ identifier, letting it - * behave properly in the syntax. - */ -case object InHoisting { - private val inName: String = "in" - - /** Executes the hoisting procedure on the provided token stream. - * - * @param tokenStream the input token stream - * @return `tokenStream` with all occurrences of `Var("in")` replaced with - * `Opr("in")` - */ - def run(tokenStream: AST.Module): AST.Module = { - tokenStream.setLines( - tokenStream.lines.map(l => l.copy(elem = l.elem.map(process))) - ) - } - - /** Maps over the token stream, replacing occurrences of `Var("in")` with - * `Opr("in")` at the same location. - * - * @param ast the ast element to process - * @return `ast`, with occurrences of "in" replaced - */ - def process(ast: AST): AST = ast match { - case App.Prefix.any(app) => - App.Prefix(process(app.func), app.off, process(app.arg)) - case AST.Ident.Var.any(v) => - if (v.name == inName) { - v.copy( - shape = Shape.Opr("in") - ) - } else { - v - } - case AST.Block.any(block) => - val newFirstLine = block.firstLine.map(process) - val newLines = block.lines.map(_.map(_.map(process))) - - block.replaceFirstLine(newFirstLine).replaceLines(newLines) - case n => n.map(process) - } -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/Parser.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/Parser.scala deleted file mode 100644 index b6e9a6551b1d..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/Parser.scala +++ /dev/null @@ -1,716 +0,0 @@ -package org.enso.syntax.text - -import java.util.UUID - -import cats.Foldable -import org.enso.data.List1 -import org.enso.data.Span -import org.enso.flexer -import org.enso.flexer.Reader -import org.enso.syntax.text.AST.Block.Line -import org.enso.syntax.text.AST.Block.OptLine -import org.enso.syntax.text.AST.Macro.Match.SegmentOps -import org.enso.syntax.text.AST.App -import org.enso.syntax.text.ast.meta.Builtin -import org.enso.syntax.text.prec.Macro -import org.enso.syntax.text.spec.ParserDef -import cats.implicits._ -import io.circe -import io.circe.Encoder -import io.circe.Json -import io.circe.generic.auto._ -import io.circe.parser._ - -//////////////////////////////// - -class InternalError(reason: String, cause: Throwable = None.orNull) - extends Exception(s"Internal error $reason", cause) - -//////////////// -//// Parser //// -//////////////// - -/** This is the main Parser class. - * - * ==The Macro System== - * - * The parser bases on a very sophisticated Macro mechanism, allowing users for - * unparalleled control and flexibility. The macro systems allows advanced - * users to create new syntax or new domain-specific languages. In a similar - * fashion to Lisp, Enso macros can freely transform the syntactic structure of - * the program. In short, anything that Enso can do to a data structure, Enso - * macros can do to code. In contrast, in most other languages, the parser's - * output is purely internal to the language implementation and cannot be - * manipulated by the programmer. - * - * Macro resolution steps: - * - * 1. Parser is executed by using the [[Parser#run]] function. It reads source - * code and outputs a token stream [[AST.Stream]]. The token stream contains a - * very narrow range of possible elements: [[AST.Blank]], [[AST.Var]], - * [[AST.Cons]], [[AST.Opr]], [[AST.Number]], [[AST.Text]], and [[AST.Block]], - * which contains lines of streams of these elements. Every other AST structure - * is build by the macro system. Please note that the stream in this step is - * encoded by using [[AST.App]] on subsequent elements. - * - * 2. Parser prepares [[Builtin.registry]] containing predefined set of macro - * [[AST.Macro.Definition]], which define such constructs as comments, parensed - * expressions, imports, new data definitions, if-then-else mixfix functions, - * or even foreign languages support. During this step parser will be also - * asking interpreter to fill the registry with definitions from other modules. - * Each [[AST.Macro.Definition]] contains macro segment descriptions and a - * finalizer, a function transforming matched tokens to final AST. Finalizer is - * used only if all macro segments were matched correctly. - * - * 3. The token stream is partitioned according to registered macros segments. - * Each macro contains multiple segments. A segment contains of an identifier, - * like "if" or "then" and a macro [[Pattern]]. Patterns are not used in this - * step. The AST stream is partitioned solely by segment identifiers. Macros - * can overlap, for example, [[Builtin.registry]] contains both "if-then" and - * "if-then-else" macro. When it is impossible to decide which macro to choose, - * like for the input "(if a) b", [[AST.Macro.Ambiguous]] is returned. - * Otherwise, each macro segment is matched against corresponding [[Pattern]] - * and [[AST.Macro.Match]] is returned and stored back in the [[AST.Stream]]. - * Please note, that even if pattern matching fails, the [[AST.Macro.Match]] - * will be the result. It will contain information about failed patterns. - * - * 4. The segment [[Pattern]] is similar to regular expression. It contains - * around 10 building blocks, such as [[Pattern.Nothing]], which does not - * consume any input, or [[Pattern.Tok]], allowing matching a specific token, - * like the "if" keyword. The result, [[Pattern.Match]] is stored in - * [[AST.Macro.Match]]. The [[Pattern.Match.Err]] is used to mark unsuccessful - * pattern match fragments, while the [[Pattern.Match.Tok]] is used to provide - * additional help messages to the end-user. Please note that it is impossible - * for the pattern match mechanism to break even on malformed user - * [[AST.Macro.Definition]]. Each definition contains a pre-process step inside - * of [[AST.Macro.Definition]] constructor, which modifies the user provided - * rules with checks if the pattern succeed and in case the pattern was used - * between segments, if it consumed all tokens. In case either of validators - * fail, all tokens are consumed and marked as an invalid match. - * - * 5. A very special pattern is the [[Pattern.Build]] construction, which tells - * the pattern match mechanism that it should build a single [[AST]] expression - * out of matched tokens. For example, a pattern - * [[Pattern.Build(Pattern.Cls[AST.Opr])]] will match an operator token and - * build a side-section AST from it. The [[Pattern.Build]] blocks are resolved - * during the pattern match step. After this step is finished and - * [[AST.Macro.Match]] or [[AST.Macro.Ambiguous]] is stored back in the - * [[AST.Stream]], nothing more happens, parsing is done! It is important to - * note, that there is a special module parsing macro, which runs - * [[Pattern.Build]] on every line. - * - * ==Pattern Build Mechanism== - * - * The resolution of [[Pattern.Build]] is as interesting as the macro system. - * It contains of the following stages: - * - * 1. First, the [[AST.Stream]] is partitioned byt the [[Distance]] processor - * according to the spacing information. All non-spaced tokens are grouped - * together and processed first. After their processing is done and each group - * will be transformed to a single [[AST]], it is put back to the original - * [[AST.Stream]] and the whole stream is processed the same way (described in - * the following points). - * - * 2. Each token of a chosen stream is then processed by the - * [[https://en.wikipedia.org/wiki/Shunting-yard_algorithm Shunting-yard - * algorithm]]. Basically, it re-shuffles the [[AST]] stream to combination of - * [[AST.App]], [[AST.App.Left]], [[AST.App.Right]], and [[AST.App.Sides]], - * according to the operator precedence. Please note that the precedence of - * user defined operators is fixed in Enso and depends on the shape of the - * operator. For example, all "arrows" like "<-", "<-<", or "<=<", have the - * same precedence. The associativity is inferred by the operator direction, - * where both "=" and "," operators are considered right-associative. See - * [[Operator]] and [[Prec]] for more information. - * - * ==Finalizers== - * - * A careful reader will notice that there was no description of how finalizers - * (mentioned in the first section) are used. Finalizers are user-provided AST - * transformations which are applied to valid AST Macro matches. After - * finalizer is applied, the spacing information might be lost. - * - * ==Space-unaware AST=== - * - * That's because they are NOT used during parsing. A very important design - * decision is that Enso AST contains all information allowing for printing the - * code back from the AST, while keeping all whitespaces as they were before - * parsing. This is why each space-aware AST, like [[AST.App]] records all - * positional information. For convenient usage, all space-aware [[AST]] - * definitions end with "Of", like [[AST.App.PrefixOf]] and have a counterpart - * without "Of" allowing for pattern matching without thinking about the - * spacing information. Because macro system is end-user extensible, we cannot - * assume that the end-user will care about recording valid spacing when - * transforming [[AST]] to another form. That's why there are also - * space-unaware [[AST]] structures, which are handy to work with by automated - * tools like the interpreter, while all the spacing information is stored only - * in the basic set of tokens and [[AST.Macro]] tokens. Each AST node has a - * [[AST.map]] function for mapping over sub-nodes, which allows easy building - * of AST traversals. The [[Parser#resolveMacros]] is such a traversal, which - * applies [[AST.Macro.Definition.Resolver]] to each [[AST.Macro.Match]] found - * in the AST, while loosing a lot of positional information. - */ -case class SourceFile(ast: AST, metadata: Json) - -object SourceFile { - val METATAG = "\n\n\n#### METADATA ####\n" - - implicit def MWMEncoder: Encoder[SourceFile] = - module => - Json.obj("ast" -> module.ast.toJson(), "metadata" -> module.metadata) -} - -class Parser { - import Parser._ - private val engine = newEngine() - - def splitMeta(code: String): (String, IDMap, Json) = { - import SourceFile._ - code.split(METATAG) match { - case Array(input) => (input, Seq(), Json.obj()) - case Array(input, rest) => - val meta = rest.split('\n') - if (meta.length < 2) { - throw new ParserError(s"Expected two lines after METADATA.") - } - val idmap = idMapFromJson(meta(0)).left.map { error => - throw new ParserError("Could not deserialize idmap.", error) - }.merge - val metadata = decode[Json](meta(1)).left.map { error => - throw new ParserError("Could not deserialize metadata.", error) - }.merge - (input, idmap, metadata) - case arr: Array[_] => - throw new ParserError( - s"Could not not deserialize metadata (found ${arr.length - 1} metadata sections)" - ) - } - } - - /** Parse contents of the program source file, - * where program code may be followed by idmap and metadata. - */ - def runWithMetadata(program: String): SourceFile = { - val (input, idmap, metadata) = splitMeta(program) - SourceFile(run(new Reader(input), idmap), metadata) - } - - /** Parse contents of the program source file, attaching any IDs defined - * in the metadata section and dropping macros resolution data. - * - * @param input the code parse. - * @return the AST resulting from parsing input. - */ - def runWithIds(input: String): AST.Module = { - val (code, idmap, _) = splitMeta(input) - val ast = run(code) - val noMacros = dropMacroMeta(ast) - attachIds(noMacros, idmap) - } - - private def attachIds(module: AST.Module, ids: IDMap): AST.Module = { - val idMap: Map[Location, AST.ID] = ids.map { case (span, id) => - (Location(span.index.value, span.index.value + span.size.value), id) - }.toMap - - def go(ast: AST): AST = { - val id = ast.location.flatMap(idMap.get) - ast.setID(id).map(go) - } - - module.map(go) - } - - /** Parse simple string with empty IdMap into AST. */ - def run(input: String): AST.Module = - run(new Reader(new CommentRemover(input).run), Nil) - - /** Parse input with provided IdMap into AST */ - def run(input: Reader, idMap: IDMap): AST.Module = { - val tokenStream = engine.run(input).map(InHoisting.run) - - val spanned = tokenStream.map(attachModuleLocations) - val resolved = spanned.map(Macro.run) match { - case flexer.Parser.Result(_, flexer.Parser.Result.Success(mod)) => - val mod2 = annotateModule(idMap, mod) - resolveMacros(mod2).asInstanceOf[AST.Module] - case _ => throw ParsingFailed - } - resolved - } - - /** Processes an input [[AST.Module]], attaching absolute span information - * to it and all its children. - */ - def attachModuleLocations(ast: AST.Module): AST.Module = { - val toplevelOffset = 0 - var currentOffset = toplevelOffset - val newLines: List1[OptLine] = ast.lines.map { line => - val locatedElem = line.elem.map(attachLocations(_, currentOffset)) - val locatedLine = Line(locatedElem, line.off) - val expressionSpan = line.elem.map(_.span).getOrElse(0) - val lineOffset = line.off - val newLineOffset = 1 - currentOffset += expressionSpan + lineOffset + newLineOffset - locatedLine - } - val unspannedModule = ast.setLines(newLines) - unspannedModule.setLocation(Location(toplevelOffset, ast.span)) - } - - /** Processes an AST block, attaching absolute span information to it - * and all children. - * - * @param ast the AST block to mark with absolute positions - * @param startOffset the position in the file this AST is located at - * @return an AST properly marked with absolute span information - */ - def attachBlockLocations(ast: AST.Block, startOffset: Int): AST.Block = { - val blockBeginOffset = 1 - val newLineOffset = 1 - val emptyLinesNewLinesOffset = ast.emptyLines.length - val emptyLinesSpacingOffset = ast.emptyLines.sum - val firstLineOffset = startOffset + blockBeginOffset + - emptyLinesNewLinesOffset + emptyLinesSpacingOffset - var currentOffset = firstLineOffset - currentOffset += ast.indent - val locatedFirstLine: AST.Block.Line = - ast.firstLine.map(attachLocations(_, currentOffset)) - currentOffset += locatedFirstLine.elem.span + locatedFirstLine.off + newLineOffset - val locatedLines = ast.lines.map { line => - if (line.elem.isDefined) { - currentOffset += ast.indent - } - val locatedLine = line.map(_.map(attachLocations(_, currentOffset))) - val elemSpan = locatedLine.elem.map(_.span).getOrElse(0) - currentOffset += elemSpan + locatedLine.off + newLineOffset - locatedLine - } - val unspannedBlock = ast - .replaceFirstLine(locatedFirstLine) - .replaceLines(locatedLines) - unspannedBlock.setLocation( - Location(startOffset, startOffset + ast.span) - ) - } - - /** Attaches absolute span information to arbitrary AST. - * - * [[App.Prefix]] nodes are treated specially, since at the stage this - * method is run, we expect a token-stream-like AST, where App nodes act as - * list conses. - * - * @param ast the AST to attach span information to - * @param startOffset the absolute offset this AST was encountered at - * @return a version of the input AST with the absolute positioning info - * populated - */ - def attachLocations(ast: AST, startOffset: Int): AST = - ast match { - case App.Prefix.any(app) => - val locatedFn = attachLocations(app.func, startOffset) - val locatedArg = - attachLocations(app.arg, startOffset + locatedFn.span + app.off) - App.Prefix(locatedFn, app.off, locatedArg) - case AST.Block.any(block) => attachBlockLocations(block, startOffset) - case _ => - ast.setLocation(Location(startOffset, startOffset + ast.span)) - } - - def annotateModule( - idMap: IDMap, - mod: AST.Module - ): AST.Module = { - var ids = idMap.sorted.toList - mod.traverseWithOff { (off, ast) => - val key = Span(off, ast) - - while (ids.nonEmpty && ids.head._1 < key) ids = ids.tail - - ids match { - case (k, id) :: _ if k == key => - ids = ids.tail - ast.setID(id) - case _ => - ast.withNewID() - } - } - } - - /** Although this function does not use any Parser-specific API now, it will - * use such in the future when the interpreter will provide information about - * defined macros other than [[Builtin.registry]]. - */ - def resolveMacros(ast: AST): AST = - ast match { - case AST.Macro.Match.any(ast) => - val resolvedAST = ast.map(resolveMacros) - Builtin.registry.get(resolvedAST.path) match { - case None => throw MissingMacroDefinition - case Some(spec) => - val id = resolvedAST.id.getOrElse(UUID.randomUUID) - val segments = resolvedAST.segs.toList().map(_.wrapped) - val ctx = AST.Macro.Resolver.Context(resolvedAST.pfx, segments, id) - resolvedAST.copy(shape = resolvedAST.shape.copy[AST](resolved = { - Option(resolveMacros(spec.resolver(ctx))) - })) - } - case _ => ast.map(resolveMacros) - } - - /* Note: [Type safety] - * ~~~~~~~~~~~~~~~~~~~ - * This function promises to return AST with the same shape as it - * received, however compiler cannot verify this due to type erasure. - * - * As we are only using copy/map function and never change shape to use - * different variants, we can say it is safe and coerce the types. - */ - - /** Automatically derives source location for an AST node, based on its - * children's locations - * - * @param ast the AST to derive location for - * @return AST with derived location - */ - def deriveLocation(ast: AST): AST = - if (ast.location.isEmpty) { - val location = ast match { - case AST.App.Section.Right(opr, right) => - opr.location |+| right.location - case _ => ast.foldMap(_.location) - } - ast.setLocation(location) - } else { - ast - } - - /** Derives location (see [[deriveLocation]]) for a node and all its - * descendants. - * - * @param ast the AST to derive location for - * @return AST with derived location - */ - def deriveLocations(ast: AST): AST = { - val withLocatedChildren = ast.map(deriveLocations) - deriveLocation(withLocatedChildren) - } - - /** Drops macros metadata keeping only resolved macros in the AST. - * WARNING: this transformation drops the relative information about AST - * spacing, while the absolute positioning is preserved. - */ - def dropMacroMeta(ast: AST.Module): AST.Module = { - def go: AST => AST = { - case AST.Macro.Match.any(t) => { - val prefix = t.pfx.toList.flatMap(_.toStream.map(_.wrapped)) - val segments = - t.segs.toList().flatMap(_.wrapped.toStream.map(_.wrapped)) - val originalSegments = (prefix ++ segments).map(deriveLocations) - val originalSpan = - Foldable[List].foldMap(originalSegments)(_.location) - val resolved = t.resolved.map(_.setLocation(originalSpan)) - go(resolved.orNull) - } - case t => deriveLocation(t.map(go)) - } - ast.map(go) - } - -} - -object Parser { - - type IDMap = Seq[(Span, AST.ID)] - - private val newEngine = flexer.Parser.compile(ParserDef()) - - def apply(): Parser = new Parser() - - def idMapFromJson(json: String): Either[circe.Error, IDMap] = - decode[IDMap](json) - - //// Exceptions //// - - case object ParsingFailed extends ParserError("parsing failed") - case object MissingMacroDefinition - extends ParserError("macro definition not found") - class ParserError(reason: String, cause: Throwable = None.orNull) - extends InternalError(s"in parser $reason", cause) -} - -//////////////////////////////////////////////////////////////////////////////// -//// Interactive Testing Utilities ///////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -////////////// -//// Main //// -////////////// - -object Main extends scala.App { - - println("--- START ---") - - val parser = new Parser() - - val in_def_maybe = - """## Foo bar baz - | bax - |def Maybe a - | ## test - | def Just val:a - | def Nothing - """.stripMargin - - val in_arr1 = "a = b -> c d" - - val in3 = "(a) b = c" - val in4 = "if a then (b)" - val in2 = "(a) b = c]" - val inp2 = "a (b (c)) x" - - val inp = - """ - |## - | DEPRECATED - | REMOVED - replaced by Foo Bar - | ADDED - | MODIFIED - | UPCOMING - | ALAMAKOTA a kot ma Ale - | This is a test of Enso Documentation Parser. This is a short synopsis. - | - | Here you can write the body of documentation. On top you can see tags - | added to this piece of code. You can customise your text with _Italic_ - | ~Strikethrough~ or *Bold*. ~_*Combined*_~ is funny - | - | - | There are 3 kinds of sections - | - Important - | - Info - | - Example - | * You can use example to add multiline code to your documentation - | - | ! Important - | Here is a small test of Important Section - | - | ? Info - | Here is a small test of Info Section - | - | > Example - | Here is a small test of Example Section - | Import Foo - | def Bar a - |type Maybe a - | ## test attached to Just - | type Just val:a - | ##DEPRECATED - | foo bar baz - | type Nothing - | - | ## The pow function calculates power of integers. - | pow x y = x ** y - |""".stripMargin - val inCdeprecated = - """## ADDED in 2.0 - | MODIFIED in 2.1 - | UNSTABLE - | Optional values. - | - | Type `Option` represents an optional value: every `Option` is either `Some` - | and contains a value, or `None`, and does not. Option types are very common - | in Enso code, as they have a number of uses: - | - Initial values. - | - Return values for functions that are not defined - | over their entire input range (partial functions). - | - Return value for otherwise reporting simple errors, where `None` is returned on error. - | - Optional struct fields. - | - Optional function arguments. - | `Option`s are commonly paired with pattern matching to query the presence of - | a value and take action, always accounting for the None case. - |type Option a - | - | ## ADVANCED - | The `Some` type indicates a presence of a value. - | type Some a - | - | ## MODIFIED - | The `None` type indicates a lack of a value. - | - | It is a very common type and is used by such types as `Maybe` or `List`. - | Also, `None` is the return value of functions which do not return an - | explicit value. - | type None - | - | ## DEPRECATED - | PRIVATE - | UNSTABLE - | TEXTONLY - | The `Nothing` is previous `None`. - | type Nothing - | - | ## The pow function calculates power of integers. - | - | ! Important - | This function, if used wildly, will break space-time continuum. - | pow x y = x ** y - | - |## TEXTONLY - | PRIVATE - | This is a testing framework for `Option`. - | - | ? Info - | It doesn't do too much in current state. - |type Option_Testing - | type Foo - | type Bar - |""".stripMargin - - val inC = - """from Standard.Base import all - | - |## A type representing computations that may fail. - |type Maybe - | - | ## No contained value. - | Nothing - | - | ## A value. - | - | Arguments: - | - value: The contained value in the maybe. - | type Some value - | - | ## Applies the provided function to the contained value if it exists, - | otherwise returning the provided default value. - | - | Arguments: - | - default: The value to return if `this` is Nothing. This value is lazy - | and hence will not execute any provided computation unless it is used. - | - function: The function to execute on the value inside the `Some`, if it - | is a just. - | - | > Example - | Apply a function over a Some value to get 4. - | (Some 2).maybe 0 *2 - | maybe : Any -> (Any -> Any) -> Any - | maybe ~default function = case this of - | Nothing -> default - | Some val -> function val - | - | ## Check if the maybe value is `Some`. - | - | > Example - | Check if `Nothing` is `Some`. - | Nothing.is_some - | is_some : Boolean - | is_some = case this of - | Nothing -> False - | Some _ -> True - | - | ## Check if the maybe value is `Nothing`. - | - | > Example - | Check if `Nothing` is `Nothing`. - | Nothing.is_nothing - | is_nothing : Boolean - | is_nothing = this.is_some.not - | - |""".stripMargin - - val testWithMultilineComment = - """ - |## ALIAS New File - | - | Creates a new file object, pointing to the given path. - | - | Arguments: - | - path: The path to the file that you want to create, or a file itself. The - | latter is a no-op. - | - | > Example - | Create a new file pointing to the `data.csv` file in the project directory. - | - | import Standard.Base.System.File - | import Standard.Examples - | - | example_new = File.new Examples.csv_path - |new : (Text | File) -> File - |new path = case path of - | Text -> File (Prim_Io.get_file path) - | _ -> path - |""".stripMargin - - println("--- PARSING ---") - - val mod = parser.run(testWithMultilineComment) - - println(Debug.pretty(mod.toString)) - - println("=========================") - println(Debug.pretty(parser.dropMacroMeta(mod).toString)) - val rmod = parser.resolveMacros(mod) - if (mod != rmod) { - println("\n---\n") - println(Debug.pretty(rmod.toString)) - } - - println("------") - println(mod.show() == testWithMultilineComment) - println("------") - println(mod.show()) - println("------") - - /** Invoking the Enso Documentation Parser */ - println("===== DOCUMENTATION =====") - val droppedMeta = parser.dropMacroMeta(mod) - val doc = docparser.DocParserRunner.createDocs(droppedMeta) - - println(Debug.pretty(doc.toString)) - println("------") - println(doc.show()) - val htmlCode = - docparser.DocParserHTMLGenerator.generateHTMLForEveryDocumented(doc) - println("========== HTML ===========") - println(htmlCode) - println("===========================") - - println("===== PURE DOCUMENTATION PARSER AND GENERATOR (W/O AST CONN) =====") - val inpOnlyDoc = - """DEPRECATED - |REMOVED - replaced by Foo Bar - |ADDED - |MODIFIED - |UPCOMING - |ALAMAKOTA a kot ma Ale - |This is a test of Enso Documentation Parser. This is a short synopsis. - | - |Here you can write the body of documentation. On top you can see tags - |added to this piece of code. You can customise your text with _Italic_ - |~Strikethrough~ or *Bold*. ~_*Combined*_~ is funny - | - | - |There are 3 kinds of sections - | - Important - | - Info - | - Example - | * You can use example to add multiline code to your documentation - | - |! Important - | Here is a small test of Important Section - | - |? Info - | Here is a small test of Info Section - | - |> Example - | Here is a small test of Example Section - | Import Foo - | def Bar a - | Foo x y - |""".stripMargin - val doc2 = DocParser.runMatched(inpOnlyDoc) - val htmlCode2 = docparser.DocParserHTMLGenerator.generateHTMLPureDoc(doc2) - println(htmlCode2) - - AST.main() - -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserHTMLGenerator.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserHTMLGenerator.scala deleted file mode 100644 index 238d110676d5..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserHTMLGenerator.scala +++ /dev/null @@ -1,342 +0,0 @@ -package org.enso.syntax.text.docparser - -import org.enso.syntax.text.AST -import org.enso.syntax.text.Shape.Block.Line -import org.enso.syntax.text.ast.Doc -import scalatags.Text.TypedTag -import scalatags.Text.{all => HTML} -import HTML._ - -//////////////////////////////////////////////////////////////////////////////// -//// Doc Parser HTML Generator ///////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -/** This is Doc Parser HTML Generator. - * - * Essentially it enables Doc Parser to create pretty HTML files from - * documented code. - */ -object DocParserHTMLGenerator { - - /** This method is used for generation of HTML files from parsed and - * reformatted [[AST.Documented]] - * - * @param ast - parsed AST.Module and reformatted using Doc Parser - */ - def generateHTMLForEveryDocumented(ast: AST): String = { - var allDocs = new String - val extMethodsHeader = HTML.h2( - HTML.div(HTML.`class` := "ml-20 flex")( - HTML.raw( - "" - ), - HTML.p("Extension Methods") - ) - ) - var extensionMethods = new String - ast.map { elem => - elem match { - case AST.Documented.any(documented) => - val file = onHTMLRendering(documented) - documented.ast match { - case AST.App.Infix.any(_) => - extensionMethods += HTML - .div(HTML.`class` := "ml-20 mb-20")(file.code) - .toString() - case _ => - allDocs += HTML - .div(HTML.`class` := "mb-20")(file.code) - .toString() - } - case AST.Def.any(tp) => - tp.body match { - case Some(body) => allDocs += generateHTMLForEveryDocumented(body) - case None => () - } - case _ => - allDocs += generateHTMLForEveryDocumented(elem) - } - elem - } - if (extensionMethods.length > 0) { - extensionMethods = extMethodsHeader.render + extensionMethods - } - "
" + allDocs + extensionMethods + "
" - } - - /** Function to generate HTML File from pure doc comment w/o connection to AST - * - * @param doc - Doc from Doc Parser - * @return - HTML Code from Doc - */ - def generateHTMLPureDoc(doc: Doc, title: String = ""): String = - HTML - .html( - HTML.body( - HTML.div(HTML.`class` := "enso docs")(doc.htmlWithTitle(title)) - ) - ) - .toString() - - ////////////////////////////////////////////////////////////////////////////// - //// HTML Rendering of Documentation ///////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** Used to create HTML files from Doc with or without title after Doc Parser - * Runner finished it's job - * - * @param documented - documented made by Doc Parser Runner from AST and Doc - * @return - HTML code with file name - */ - def onHTMLRendering(documented: AST.Documented): htmlFile = { - val htmlCode = DocumentedToHtml(documented.ast, documented.doc) - val astLines = documented.ast.show().split("\n") - val fileName = - astLines.head - .replaceAll("/", "") - .replaceAll(" ", "_") - .split("=") - .head - htmlFile(htmlCode, fileName) - } - case class htmlFile(code: TypedTag[String], name: String) - - /** This function is used to get HTML content of Doc and try to render AST, - * by finding if it also contains Documented to retrieve Doc and it's AST, - * or simply call show() method on other element of AST. - * - * @param ast - AST from Parser - * @param doc - Doc from Doc Parser - * @return - HTML Code from Doc and contents of [[AST.Def]] or - * [[AST.App.Infix]] - */ - def DocumentedToHtml(ast: AST, doc: Doc): TypedTag[String] = { - val astHTML = createHTMLFromAST(ast, doc.tags) - val astName = HTML.div(astHTML.header) - astHTML.body match { - case Some(body) => - // Case when producing main page - HTML.div(HTML.`class` := "main ml-20")( - astName, - doc.htmlWoTagsMain, - body - ) - case None => - // Case when listing atoms or methods - HTML.div( - astName, - doc.htmlWoTags - ) - } - } - - /** This case class is used to hold HTML-rendered AST - * - * @param header - header of AST - name of module/method with parameters - * @param body - body of AST - All of AST's documented submodules/methods - */ - case class astHtmlRepr( - header: TypedTag[String], - body: Option[TypedTag[String]] - ) - object astHtmlRepr { - def apply(header: TypedTag[String], body: TypedTag[String]): astHtmlRepr = - new astHtmlRepr(header, Some(body)) - def apply(header: TypedTag[String]): astHtmlRepr = - new astHtmlRepr(header, None) - def apply(): astHtmlRepr = - new astHtmlRepr(HTML.div(), None) - } - - /** Function invoked by [[DocumentedToHtml]] to create HTML from AST in - * [[AST.Documented]] on every matching element - * - * @param ast - AST - * @return - HTML Code - */ - def createHTMLFromAST( - ast: AST, - tags: Option[Doc.Tags] = None - ): astHtmlRepr = { - ast match { - case AST.Def.any(d) => - d.body match { - case Some(body) => - body match { - case AST.Block.any(b) => - createDefWithBody(d.name, d.args, b, tags) - case _ => - astHtmlRepr(createAtomHtmlRepr(d.name, d.args, tags)) - } - case None => astHtmlRepr(createAtomHtmlRepr(d.name, d.args, tags)) - } - case AST.App.Infix.any(i) => - if (i.larg.show().split(" ").nonEmpty) { - astHtmlRepr(createInfixHtmlRepr(i, tags)) - } else { - astHtmlRepr() - } - case _ => astHtmlRepr() - } - } - - /** Helper function for [[createHTMLFromAST]] to generate appropriate code - * from [[AST.Def]] with traversing through body and creating HTML code - * on elements in it - * - * @param name - Def Name - * @param args - Def Arguments - * @param body - Def body - * @return - HTML code generated from Def - */ - def createDefWithBody( - name: AST.Cons, - args: List[AST], - body: AST.Block, - tags: Option[Doc.Tags] - ): astHtmlRepr = { - val firstLine = Line(Option(body.firstLine.elem), body.firstLine.off) - val atomsHeader = HTML.h2( - HTML.div(HTML.`class` := "flex")( - HTML.raw( - "" - ), - HTML.p("Atoms") - ) - ) - - val extMethodsHeader = HTML.h2( - HTML.div(HTML.`class` := "flex")( - HTML.raw( - "" - ), - HTML.p("Methods") - ) - ) - - val allLines = firstLine :: body.lines - val generatedCode = renderHTMLOnLine(allLines) - val atoms = generatedCode.filter( - _.toString().contains("class=\"atom flex\"") - ) - val methods = generatedCode.filter( - _.toString().contains("class=\"method flex\"") - ) - - val head = createDocTitle(name, args, tags) - var methodsLines = HTML.div() - var atomsLines = HTML.div() - if (methods.nonEmpty) { - methodsLines = - HTML.div(HTML.`class` := "methods")(extMethodsHeader, methods) - } - if (atoms.nonEmpty) { - atomsLines = HTML.div(HTML.`class` := "atoms")(atomsHeader, atoms) - } - val lines = HTML.div(atomsLines, methodsLines) - astHtmlRepr(head, lines) - } - - def createDocTitle( - name: AST.Cons, - args: List[AST], - tags: Option[Doc.Tags] - ): TypedTag[String] = { - val nameStr = name.show() - val argsStr = args.map(_.show()) - val tagsHtml = tags.getOrElse(Doc.Elem.Text("")).html - - HTML.h1( - HTML.p( - HTML.span(HTML.`class` := "name")(nameStr), - " ", - HTML.span(HTML.`class` := "parameter")(argsStr) - ), - tagsHtml - ) - } - - /** Function to generate Atoms from it's name and tags. - * - * @param name - Atom Name - * @param args - Atom Arguments - * @param tags - Atom Tags - * @return - HTMl doc of atom. - */ - def createAtomHtmlRepr( - name: AST.Cons, - args: List[AST], - tags: Option[Doc.Tags] - ): TypedTag[String] = { - val nameStr = name.show() - val argsStr = args.map(_.show() + " ") - val tagsHtml = tags.getOrElse(Doc.Elem.Text("")).html - - HTML.div(HTML.`class` := "atom flex")( - HTML.p( - HTML.span(HTML.`class` := "name")(nameStr), - " ", - HTML.span(HTML.`class` := "parameter")(argsStr) - ), - tagsHtml - ) - } - - /** Helper function for [[createHTMLFromAST]] to generate appropriate HTML - * code from [[AST.App.Infix]] - * - * @param infix - AST Infix - * @return - HTML code generated from Infix - */ - def createInfixHtmlRepr( - infix: AST.App.Infix, - tags: Option[Doc.Tags] - ): TypedTag[String] = { - val nameStr = infix.larg.show().split(" ").head - val argsStr = infix.larg.show().split(" ").tail.mkString(" ") - val tagsHtml = tags.getOrElse(Doc.Elem.Text("")).html - - HTML.div(HTML.`class` := "method flex")( - HTML.p( - HTML.span(HTML.`class` := "name")(nameStr), - " ", - HTML.span(HTML.`class` := "argument")(argsStr) - ), - tagsHtml - ) - } - - /** Helper function for [[createDefWithBody]] to traverse through body's lines - * and try to generate HTML code from [[AST.Documented]] parts of it. It also - * tries to find nested [[AST.Def]] and [[AST.App.Infix]] inside of body - * - * @param lines - lines inside of Def body - * @return - HTML code generated from contents of lines - */ - def renderHTMLOnLine(lines: List[AST.Block.OptLine]): List[TypedTag[String]] = - lines match { - case Line(Some(AST.Documented.any(doc)), _) :: rest => - val docHtml = DocumentedToHtml(doc.ast, doc.doc) - HTML.div(docHtml) :: renderHTMLOnLine(rest) - case _ :: rest => renderHTMLOnLine(rest) - case other => - other match { - case Nil => List() - case _ :: rest => renderHTMLOnLine(rest) - } - } - - /** Function invoked by [[DocumentedToHtml]] to create HTML.Head part of file - * - * @param title - HTML page title - * @return - HTML Head Code - */ - def createHTMLHead(title: String): TypedTag[String] = { - val metaEquiv = HTML.httpEquiv := "Content-Type" - val metaCont = HTML.content := "text/html" - val metaChar = HTML.charset := "UTF-8" - val meta = HTML.meta(metaEquiv)(metaCont)(metaChar) - val fileTitle = scalatags.Text.tags2.title(title) - HTML.head(meta)(fileTitle) - } -} diff --git a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserRunner.scala b/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserRunner.scala deleted file mode 100644 index 937420fdfc96..000000000000 --- a/lib/scala/syntax/specialization/shared/src/main/scala/org/enso/syntax/text/docparser/DocParserRunner.scala +++ /dev/null @@ -1,220 +0,0 @@ -package org.enso.syntax.text.docparser - -import org.enso.data.List1 -import org.enso.syntax.text.{AST, DocParser} -import org.enso.syntax.text.Shape.Block.Line -import org.enso.syntax.text.ast.Doc - -//////////////////////////////////////////////////////////////////////////////// -//// Doc Parser Runner ///////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// - -/** Doc Parser Runner binds together Enso Parser with Doc Parser. - * - * When Parser finishes its job it invokes runner with AST created by it after - * resolving macros. Then Runner does it's job - running Doc Parser on every - * [[AST.Comment]], combined with connecting [[Doc]] with AST in - * [[AST.Documented]] node, which gets AST from [[AST.Def]] and - * [[AST.App.Infix]] - */ -object DocParserRunner { - - ////////////////////////////////////////////////////////////////////////////// - //// Created Doc's in right places with appropriate AST ////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - /** This function invokes the documentation parser on every instance of - * [[AST.Comment]]. - * - * It matches on [[AST.Module]] or [[AST.Def]] in order to traverse - * through their lines to create [[AST.Documented]] from [[AST.Comment]] if - * found, with AST from [[AST.App.Infix]] or [[AST.Def]] - * - * @param ast - module with possibility to create Documentation from comments - * @return - modified data containing possibly Documentation(s) with AST - */ - def createDocs(ast: AST): AST = { - ast match { - case AST.Module.any(m) => createDocsFromModule(m) - case AST.Def.any(d) => - d.body match { - case Some(body) => - body match { - case AST.Block.any(b) => createDocsFromDefBody(d.name, d.args, b) - case _ => d - } - case None => d - } - case other => other - } - } - - /** This is a helper function for [[createDocs]] to traverse through - * [[AST.Module]] and create Docs from comments with appropriate [[AST]] - */ - def createDocsFromModule(m: AST.Module): AST.Module = { - val emptyLine = List1(AST.Block.OptLine()) - val transformedLines = - List1(attachDocToSubsequentAST(m.lines.toList)).getOrElse(emptyLine) - AST.Module(transformedLines) - } - - /** This is a helper function for [[createDocs]] to traverse through - * [[AST.Def]] and create Docs from comments inside [[AST.Def]] with - * appropriate [[AST]] - */ - def createDocsFromDefBody( - name: AST.Cons, - args: List[AST], - b: AST.Block - ): AST.Def = { - val firstLine = Line(Option(b.firstLine.elem), b.firstLine.off) - val linesToTransform = firstLine :: b.lines - val transformedLines = attachDocToSubsequentAST(linesToTransform) - val TLHeadElem = transformedLines.head.elem.get - val TLHeadOff = transformedLines.head.off - val head = AST.Block.Line(TLHeadElem, TLHeadOff) - val lines = transformedLines.tail - val body = AST.Block(b.ty, b.indent, b.emptyLines, head, lines) - AST.Def(name, args, Some(body)) - } - - /** This is a helper function for creating docs with AST. - * Essentially it traverses through lines and tries to find a pattern on them - * - * @param lines - AST lines - * @return - lines with possibly Doc with added AST - */ - def attachDocToSubsequentAST( - lines: List[AST.Block.OptLine] - ): List[AST.Block.OptLine] = - lines match { - case line1 :: tail => - line1 match { - case Line(Some(AST.Comment.any(com)), off) => - tail match { - case line2 :: rest => - line2 match { - case Line(Some(AST.App.Infix.any(ast)), _) => - commentWithInfixForDocumented( - com, - off, - ast, - rest - ) - case Line(Some(AST.Def.any(ast)), _) => - commentWithDefForDocumented(com, off, ast, rest) - case Line(None, _) => - var restTrav = rest - var emp = 1 - val emptyLine = Line(None, 0) - while (restTrav.nonEmpty && restTrav.head == emptyLine) { - emp += 1 - restTrav = restTrav.tail - } - val rTail = restTrav.tail - val rHead = restTrav.head - rHead match { - case Line(Some(AST.App.Infix.any(ast)), _) => - commentWithInfixForDocumented( - com, - off, - ast, - rTail, - emp - ) - case Line(Some(AST.Def.any(ast)), _) => - commentWithDefForDocumented( - com, - off, - ast, - rTail, - emp - ) - case _ => - line1 :: line2 :: attachDocToSubsequentAST(rest) - } - case other => - line1 :: attachDocToSubsequentAST(other :: rest) - } - case Nil => line1 :: Nil - } - case other => other :: attachDocToSubsequentAST(tail) - } - case Nil => Nil - } - - /** Creates Docs from comments found in parsed data - * - * @param comment - Comment found in AST. - * @return - Documentation. - */ - def createDocFromComment(comment: AST.Comment): Doc = { - val in = comment.lines.mkString("\n") - DocParser.runMatched(in) - } - - /** Function for creating documented lines in [[attachDocToSubsequentAST]] - * method with [[AST.App.Infix]] as Documented AST - * - * @param com - comment found in AST - * @param off - line offset - * @param ast - [[AST.App.Infix]] to go with comment into Documented - * @param rest - lines after documented - * @param emptyLines - Empty lines in between Doc and AST - * @return - [[AST.Documented]] - */ - def commentWithDefForDocumented( - com: AST.Comment, - off: Int, - ast: AST.Def, - rest: List[AST.Block.OptLine], - emptyLines: Int = 0 - ): List[AST.Block.OptLine] = { - val docFromAst = createDocs(ast) - val docLine = - createDocumentedLine(com, emptyLines, docFromAst, off) - docLine :: attachDocToSubsequentAST(rest) - } - - /** Function for creating documented lines in [[attachDocToSubsequentAST]] - * method with [[AST.Def]] as Documented AST - * - * @param com - comment found in AST - * @param off - line offset - * @param ast - [[AST.Def]] to go with comment into Documented - * @param rest - lines after documented - * @param emptyLines - Empty lines in between Doc and AST - * @return - [[AST.Documented]] - */ - def commentWithInfixForDocumented( - com: AST.Comment, - off: Int, - ast: AST.App.Infix, - rest: List[AST.Block.OptLine], - emptyLines: Int = 0 - ): List[AST.Block.OptLine] = { - val docLine = createDocumentedLine(com, emptyLines, ast, off) - docLine :: attachDocToSubsequentAST(rest) - } - - /** Function for creating documented lines in [[attachDocToSubsequentAST]] - * method - * - * @param comment - comment found in AST - * @param emptyLines - Empty lines in between Doc and AST - * @param off - line offset - * @param ast - AST to go with comment into Documented - * @return - [[AST.Documented]] - */ - def createDocumentedLine( - comment: AST.Comment, - emptyLines: Int, - ast: AST, - off: Int - ): Line[Some[AST.Documented]] = { - val doc = createDocFromComment(comment) - val documented = Some(AST.Documented(doc, emptyLines, ast)) - Line(documented, off) - } -} diff --git a/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/DocParserTests.scala b/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/DocParserTests.scala deleted file mode 100644 index a821fd504388..000000000000 --- a/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/DocParserTests.scala +++ /dev/null @@ -1,1689 +0,0 @@ -package org.enso.syntax.text - -import org.enso.syntax.text.ast.Doc -import org.enso.syntax.text.ast.Doc._ -import org.enso.syntax.text.ast.Doc.Elem._ -import org.enso.Logger -import org.enso.flexer.Parser.Result -import org.enso.flexer.Reader -import org.enso.syntax.text.ast.Doc.Tags.Tag -import org.scalatest.Assertion -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -class DocParserTests extends AnyFlatSpec with Matchers { - val logger = new Logger() - - def assertExpr( - input: String, - result: Doc, - assertShow: Boolean = true - ): Assertion = { - val output = DocParser.run(input) - output match { - case Result(_, Result.Success(value)) => - if (assertShow) { - assert(value.show() == new Reader(input).toString()) - } - assert(value == result) - case _ => - fail(s"Parsing documentation failed, consumed ${output.offset} chars") - } - } - - implicit class TestString(input: String) { - def parseDocumentation(str: String): String = { - val escape = (str: String) => str.replace("\n", "\\n") - s"parse `${escape(str)}`" - } - - private val testBase = it should parseDocumentation(input) - - def ?=(out: Doc): Unit = testBase in { - assertExpr(input, out) - } - def ?==(out: Doc): Unit = testBase in { - assertExpr(input, out, assertShow = false) - } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Formatters ////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "Foo" ?= Doc(Synopsis(Section.Raw("Foo"))) - "Foo\uD83D\uDC98" ?= Doc(Synopsis(Section.Raw("Foo", "\uD83D\uDC98"))) - "*Foo*" ?= Doc(Synopsis(Section.Raw(Formatter(Formatter.Bold, "Foo")))) - "_Foo_" ?= Doc(Synopsis(Section.Raw(Formatter(Formatter.Italic, "Foo")))) - "~Foo~" ?= Doc( - Synopsis(Section.Raw(Formatter(Formatter.Strikeout, "Foo"))) - ) - "`Foo`" ?= Doc(Synopsis(Section.Raw(CodeBlock.Inline("Foo")))) - "~*Foo*~" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Strikeout, Formatter(Formatter.Bold, "Foo")) - ) - ) - ) - "~_Foo_~" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Strikeout, Formatter(Formatter.Italic, "Foo")) - ) - ) - ) - "_~Foo~_" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Italic, Formatter(Formatter.Strikeout, "Foo")) - ) - ) - ) - "_*Foo*_" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Italic, Formatter(Formatter.Bold, "Foo")) - ) - ) - ) - "*_Foo_*" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Bold, Formatter(Formatter.Italic, "Foo")) - ) - ) - ) - "*~Foo~*" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Bold, Formatter(Formatter.Strikeout, "Foo")) - ) - ) - ) - "_~*Foo*~_" ?= Doc( - Synopsis( - Section.Raw( - Formatter( - Formatter.Italic, - Formatter(Formatter.Strikeout, Formatter(Formatter.Bold, "Foo")) - ) - ) - ) - ) - "`import foo`" ?= Doc( - Synopsis( - Section.Raw(CodeBlock.Inline("import foo")) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Unclosed formatters ///////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "_*Foo*" ?= Doc( - Synopsis( - Section.Raw( - Formatter.Unclosed(Formatter.Italic, Formatter(Formatter.Bold, "Foo")) - ) - ) - ) - "~*Foo*" ?= Doc( - Synopsis( - Section.Raw( - Formatter - .Unclosed(Formatter.Strikeout, Formatter(Formatter.Bold, "Foo")) - ) - ) - ) - "***Foo" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Bold), - Formatter.Unclosed(Formatter.Bold, "Foo") - ) - ) - ) - "*_Foo_" ?= Doc( - Synopsis( - Section.Raw( - Formatter.Unclosed(Formatter.Bold, Formatter(Formatter.Italic, "Foo")) - ) - ) - ) - "~_Foo_" ?= Doc( - Synopsis( - Section.Raw( - Formatter - .Unclosed(Formatter.Strikeout, Formatter(Formatter.Italic, "Foo")) - ) - ) - ) - "___Foo" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Italic), - Formatter.Unclosed(Formatter.Italic, "Foo") - ) - ) - ) - "*~Foo~" ?= Doc( - Synopsis( - Section.Raw( - Formatter - .Unclosed(Formatter.Bold, Formatter(Formatter.Strikeout, "Foo")) - ) - ) - ) - "_~Foo~" ?= Doc( - Synopsis( - Section.Raw( - Formatter - .Unclosed(Formatter.Italic, Formatter(Formatter.Strikeout, "Foo")) - ) - ) - ) - "~~~Foo" ?= Doc( - Synopsis( - Section.Raw( - Formatter(Formatter.Strikeout), - Formatter.Unclosed(Formatter.Strikeout, "Foo") - ) - ) - ) - " foo *bar* _baz *bo*_" ?= Doc( - Synopsis( - Section.Raw( - 1, - "foo ", - Formatter(Formatter.Bold, "bar"), - " ", - Formatter(Formatter.Italic, "baz ", Formatter(Formatter.Bold, "bo")) - ) - ) - ) - """foo *bar - |*""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis(Section.Raw("foo ", Formatter(Formatter.Bold, "bar", Newline))) - ) - - """foo _foo - |_foo2""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section - .Raw("foo ", Formatter(Formatter.Italic, "foo", Newline), "foo2") - ) - ) - - """foo *foo - |*foo2""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section - .Raw("foo ", Formatter(Formatter.Bold, "foo", Newline), "foo2") - ) - ) - - """foo ~foo - |~foo2""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section - .Raw("foo ", Formatter(Formatter.Strikeout, "foo", Newline), "foo2") - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Segments //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "!Important" ?= Doc( - Synopsis( - Section.Marked(Section.Marked.Important, Section.Header("Important")) - ) - ) - " ! Important" ?= Doc( - Synopsis( - Section - .Marked(1, 1, Section.Marked.Important, Section.Header("Important")) - ) - ) - " ! Important" ?= Doc( - Synopsis( - Section - .Marked(3, 1, Section.Marked.Important, Section.Header("Important")) - ) - ) - " ! Important" ?= Doc( - Synopsis( - Section - .Marked(1, 4, Section.Marked.Important, Section.Header("Important")) - ) - ) - """ ! Important - | This is important.""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - "This is important." - ) - ) - ) - """! Synopsis - | This _is_ important""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Marked( - 0, - 1, - Section.Marked.Important, - Section.Header("Synopsis"), - Doc.Elem.Newline, - "This ", - Formatter(Formatter.Italic, "is"), - " important" - ) - ) - ) - """Synopsis - |This _is1_ important""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw( - "Synopsis", - Doc.Elem.Newline, - "This ", - Formatter(Formatter.Italic, "is1"), - " important" - ) - ) - ) - """ Synopsis - | - | ! Important - | This is important.""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - "This is important." - ) - ) - ) - """ Synopsis - | - | ! Important - | This is important. - | - | And this""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - "This is important.", - Doc.Elem.Newline - ), - Section.Raw(3, "And this") - ) - ) - """ Synopsis - | - | !Important - | This is a code - | - | And this is not""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 0, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(4, "This is a code")), - Doc.Elem.Newline - ), - Section.Raw(2, "And this is not") - ) - ) - """ Synopsis - | - | !Important - | This is a code - | - | Other: And this is a section""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 0, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(4, "This is a code")), - Doc.Elem.Newline - ), - Section.Raw(1, "Other: And this is a section") - ) - ) - """ Synopsis - | - | ! Important - | - | This is a multiline code - | - | ? Info""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?== Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(4, "This is a multiline code")) - ), - Section.Marked( - 1, - 1, - Section.Marked.Info, - Section.Header("Info") - ) - ) - ) - """Synopsis - | - |! Important - | This is important - | - | And this is a code""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?== Doc( - Synopsis( - Section.Raw("Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 0, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - "This is important", - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(4, "And this is a code")) - ) - ) - ) - """ Synopsis - | - | ! Important - | - | This is a multiline code - | - | Other: And this *is* a section""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?== Doc( - Synopsis( - Section.Raw(1, "Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(4, "This is a multiline code")) - ), - Section.Raw( - 1, - "Other: And this ", - Formatter(Formatter.Bold, "is"), - " a section" - ) - ) - ) - "?Info" ?= Doc( - Synopsis(Section.Marked(Section.Marked.Info, Section.Header("Info"))) - ) - ">Example" ?= Doc( - Synopsis( - Section.Marked(Section.Marked.Example, Section.Header("Example")) - ) - ) - """?Info - | - |!Important""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Marked(Section.Marked.Info, Section.Header("Info"), Newline) - ), - Body( - Section.Marked(Section.Marked.Important, Section.Header("Important")) - ) - ) - """?Info - | - |!Important - | - |>Example""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Marked(Section.Marked.Info, Section.Header("Info"), Newline) - ), - Body( - Section.Marked( - Section.Marked.Important, - Section.Header("Important"), - Newline - ), - Section.Marked(Section.Marked.Example, Section.Header("Example")) - ) - ) - """Synopsis - | - | ! Important - | This is important - | And this is a code - | - |> Example - | This is example - | More code""".stripMargin.replaceAll( - System.lineSeparator(), - "\n" - ) ?= Doc( - Synopsis( - Section.Raw("Synopsis", Doc.Elem.Newline) - ), - Body( - Section.Marked( - 1, - 1, - Section.Marked.Important, - Section.Header("Important"), - Doc.Elem.Newline, - "This is important", - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(5, "And this is a code")), - Doc.Elem.Newline - ), - Section.Marked( - 0, - 1, - Section.Marked.Example, - Section.Header("Example"), - Doc.Elem.Newline, - "This is example", - Doc.Elem.Newline, - CodeBlock(CodeBlock.Line(6, "More code")) - ) - ) - ) - """Foo *Foo* ~*Bar~ `foo bar baz bo` - | - | - |Hello Section - | - |!important - | - |?info - | - |>Example""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "Foo ", - Formatter(Formatter.Bold, "Foo"), - " ", - Formatter( - Formatter.Strikeout, - Formatter.Unclosed(Formatter.Bold, "Bar") - ), - " ", - CodeBlock.Inline("foo bar baz bo"), - Newline - ) - ), - Body( - Section.Raw(Section.Header("Hello Section"), Newline), - Section - .Marked( - Section.Marked.Important, - Section.Header("important"), - Newline - ), - Section.Marked(Section.Marked.Info, Section.Header("info"), Newline), - Section.Marked(Section.Marked.Example, Section.Header("Example")) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Lists /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "ul:\n - Foo\n - Bar" ?= Doc( - Synopsis( - Section.Raw( - "ul:", - Newline, - List(2, List.Unordered, "Foo", "Bar") - ) - ) - ) - "ol:\n * Foo\n * Bar" ?= Doc( - Synopsis( - Section.Raw( - "ol:", - Newline, - List(2, List.Ordered, "Foo", "Bar") - ) - ) - ) - """List - |- First - |- Second - |- Third""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 0, - List.Unordered, - "First", - "Second", - "Third" - ) - ) - ) - ) - """List - | - First unordered item - | - Second unordered item - | - Third unordered item - |""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First unordered item", - "Second unordered item", - "Third unordered item" - ), - Newline - ) - ) - ) - """List - |- _First_ - |- *Second*""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 0, - List.Unordered, - Formatter(Formatter.Italic, "First"), - Formatter(Formatter.Bold, "Second") - ) - ) - ) - ) - """List - |- _First_ list `item` - |- *Second* list ~item""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 0, - List.Unordered, - ListItem( - Formatter(Formatter.Italic, "First"), - " list ", - CodeBlock.Inline("item") - ), - ListItem( - Formatter(Formatter.Bold, "Second"), - " list ", - Formatter.Unclosed(Formatter.Strikeout, "item") - ) - ) - ) - ) - ) - """List - |- _First_ list `item` - |- *Second* list ~item - |""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 0, - List.Unordered, - ListItem( - Formatter(Formatter.Italic, "First"), - " list ", - CodeBlock.Inline("item") - ), - ListItem( - Formatter(Formatter.Bold, "Second"), - " list ", - Formatter.Unclosed(Formatter.Strikeout, "item", Newline) - ) - ) - ) - ) - ) - """ List - | - unclosed_formatter - | - second""".stripMargin.stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - 1, - "List", - Newline, - List( - 3, - List.Unordered, - ListItem( - "unclosed", - Formatter.Unclosed(Formatter.Italic, "formatter") - ), - "second" - ) - ) - ) - ) - """ unclosed_formatter - | - first - | - second - |""".stripMargin.stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - 1, - "unclosed", - Formatter.Unclosed(Formatter.Italic, "formatter"), - Newline, - List( - 1, - List.Unordered, - "first", - "second" - ), - Newline - ) - ) - ) - """ unclosed_formatter - | - first - | - second_unclosed - | - sub""".stripMargin.stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - 1, - "unclosed", - Formatter.Unclosed(Formatter.Italic, "formatter"), - Newline, - List( - 1, - List.Unordered, - ListItem("first"), - ListItem("second", Formatter.Unclosed(Formatter.Italic, "unclosed")), - List( - 3, - List.Unordered, - "sub" - ) - ) - ) - ) - ) - """List - | - First - | - Second - | * First1 - | * Second1 - | - Third""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - "Second", - List( - 4, - List.Ordered, - "First1", - "Second1" - ), - "Third" - ) - ) - ) - ) - """List - | - First - | - First1 - | - First2 - | - First3""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - List( - 4, - List.Unordered, - "First1", - List( - 6, - List.Unordered, - "First2", - List( - 8, - List.Unordered, - "First3" - ) - ) - ) - ) - ) - ) - ) - """List - | - First - | - First1 - | - First2 - | - First3 - |""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - List( - 4, - List.Unordered, - "First1", - List( - 6, - List.Unordered, - "First2", - List( - 8, - List.Unordered, - ListItem("First3", Newline) - ) - ) - ) - ) - ) - ) - ) - """List - | - First - | - First1 - | - First2 - | - First3 - | - Second""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - List( - 4, - List.Unordered, - "First1", - List( - 6, - List.Unordered, - "First2", - List( - 8, - List.Unordered, - "First3" - ) - ) - ), - "Second" - ) - ) - ) - ) - - """List - | - First unordered item - | - Second unordered item - | * First ordered sub item - | * Second ordered sub item - | - Third unordered item""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First unordered item", - "Second unordered item", - List( - 4, - List.Ordered, - "First ordered sub item", - "Second ordered sub item" - ), - "Third unordered item" - ) - ) - ) - ) - """List - | - First unordered item - | - Second unordered item - | * First ordered sub item - | * Second ordered sub item - | - Third unordered item""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First unordered item", - "Second unordered item", - List( - 4, - List.Ordered, - "First ordered sub item", - " Second ordered sub item" - ), - "Third unordered item" - ) - ) - ) - ) - """List - | - First unordered item - | - Second unordered item - | * First ordered sub item - | * Second ordered sub item - | - Third unordered item - | * First ordered sub item - | * Second ordered sub item - | - First unordered sub item - | - Second unordered sub item - | * Third ordered sub item - | - Fourth unordered item""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First unordered item", - "Second unordered item", - List( - 4, - List.Ordered, - "First ordered sub item", - "Second ordered sub item" - ), - "Third unordered item", - List( - 4, - List.Ordered, - "First ordered sub item", - "Second ordered sub item", - List( - 6, - List.Unordered, - "First unordered sub item", - "Second unordered sub item" - ), - "Third ordered sub item" - ), - "Fourth unordered item" - ) - ) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Wrong indent //////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - """List - | - First - | * Aligned - | * Misaligned - | * Misaligned _styled_ - | * Correct - | - Second""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - List( - 5, - List.Ordered, - "Aligned", - Elem.MisalignedItem(4, List.Ordered, "Misaligned"), - Elem.MisalignedItem( - 3, - List.Ordered, - "Misaligned ", - Formatter(Formatter.Italic, "styled") - ), - "Correct" - ), - "Second" - ) - ) - ) - ) - - """List - | - First - | - First1 - | - Second1 - | - First2 - | - Second""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "List", - Newline, - List( - 2, - List.Unordered, - "First", - List( - 4, - List.Unordered, - "First1", - Elem.MisalignedItem(3, List.Unordered, "Second1"), - List( - 6, - List.Unordered, - "First2" - ) - ), - "Second" - ) - ) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Links /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "[Hello](Http://Google.com)" ?= Doc( - Synopsis( - Section.Raw( - Link.URL( - "Hello", - "Http://Google.com" - ) - ) - ) - ) - "![Media](http://foo.com)" ?= Doc( - Synopsis( - Section.Raw( - Link.Image( - "Media", - "http://foo.com" - ) - ) - ) - ) - "![foo)" ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "![foo)" - ) - ) - ) - ) - "[foo)" ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "[foo)" - ) - ) - ) - ) - "[foo]bo)" ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "[foo]bo)" - ) - ) - ) - ) - "![foo]google" ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "![foo]google" - ) - ) - ) - ) - - "[foo]google" ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "[foo]google" - ) - ) - ) - ) - - """[foo]bo) - |basdbasd""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - Link.Invalid( - "[foo]bo)" - ), - Newline, - "basdbasd" - ) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Tags //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - val allPossibleTags: Set[Tag.Type] = - Tags.Tag.Type.codes.-(Tags.Tag.Unrecognized) - - allPossibleTags.foreach(t => - s"${t.toString.toUpperCase()}\nFoo" ?= Doc( - Tags(Tags.Tag(t)), - Synopsis(Section.Raw("Foo")) - ) - ) - "DEPRECATED in 1.0\nFoo" ?= Doc( - Tags(Tags.Tag(Tags.Tag.Type.Deprecated, " in 1.0")), - Synopsis(Section.Raw("Foo")) - ) - "DEPRECATED in 1.0\nMODIFIED\nFoo" ?= Doc( - Tags( - Tags.Tag(Tags.Tag.Type.Deprecated, " in 1.0"), - Tags.Tag(Tags.Tag.Type.Modified) - ), - Synopsis(Section.Raw("Foo")) - ) - """ ALAMAKOTA - | DEPRECATED - | PRIVATE - | foo bar""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Tags( - Tags.Tag(3, Tags.Tag.Unrecognized, "ALAMAKOTA"), - Tags.Tag(1, Tags.Tag.Type.Deprecated), - Tags.Tag(1, Tags.Tag.Type.Private) - ), - Synopsis(Section.Raw(1, "foo bar")) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Multiline code ////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - """afsfasfsfjanfjanfa - |jfnajnfjadnbfjabnf - | siafjaifhjiasjf - | fasfknfanfijnf""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "afsfasfsfjanfjanfa", - Newline, - "jfnajnfjadnbfjabnf", - Newline, - CodeBlock( - CodeBlock.Line(3, "siafjaifhjiasjf"), - CodeBlock.Line(3, "fasfknfanfijnf") - ) - ) - ) - ) - """afsfasfsfjanfjanfa - |jfnajnfjadnbfjabnf - | siafjaifhjiasjf - | fasfknfanfijnf - | fasfknfanfijnf""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "afsfasfsfjanfjanfa", - Newline, - "jfnajnfjadnbfjabnf", - Newline, - CodeBlock( - CodeBlock.Line(3, "siafjaifhjiasjf"), - CodeBlock.Line(5, "fasfknfanfijnf"), - CodeBlock.Line(3, "fasfknfanfijnf") - ) - ) - ) - ) - """afsfasfsfjanfjanfa - |jfnajnfjadnbfjabnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "afsfasfsfjanfjanfa", - Newline, - "jfnajnfjadnbfjabnf", - Newline, - CodeBlock( - CodeBlock.Line(3, "fasfknfanfijnf"), - CodeBlock.Line(5, "fasfknfanfijnf"), - CodeBlock.Line(10, "fasfknfanfijnf"), - CodeBlock.Line(5, "fasfknfanfijnf"), - CodeBlock.Line(3, "fasfknfanfijnf") - ) - ) - ) - ) - """afsfasfsfjanfjanfa - |jfnajnfjadnbfjabnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf - | fasfknfanfijnf""".stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - "afsfasfsfjanfjanfa", - Newline, - "jfnajnfjadnbfjabnf", - Newline, - CodeBlock( - CodeBlock.Line(3, "fasfknfanfijnf"), - CodeBlock.Line(5, "fasfknfanfijnf"), - CodeBlock.Line(2, "fasfknfanfijnf"), - CodeBlock.Line(5, "fasfknfanfijnf"), - CodeBlock.Line(3, "fasfknfanfijnf") - ) - ) - ) - ) - - ////////////////////////////////////////////////////////////////////////////// - //// Unclassified tests ////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - """ - | - bar - | baz - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - Newline, - List(1, List.Unordered, ListItem("bar", Newline, " ", "baz")), - Newline - ) - ) - ) - """ - | - bar - | baz - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - Newline, - List( - 1, - List.Unordered, - ListItem("bar", Newline, CodeBlock(CodeBlock.Line(5, "baz"))) - ), - Newline - ) - ) - ) - """ - | - bar - | baz - | - bar - | baz - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - Newline, - List( - 1, - List.Unordered, - ListItem("bar", Newline, " ", "baz"), - ListItem("bar", Newline, " ", "baz") - ), - Newline - ) - ) - ) - """ - | - bar - | baz - | - bar - | baz - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - Newline, - List( - 1, - List.Unordered, - ListItem("bar", Newline, CodeBlock(CodeBlock.Line(5, "baz"))), - ListItem("bar", Newline, " ", "baz") - ), - Newline - ) - ) - ) - - """ This does foo: - | - bar - | baz - | Another raw text. - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - 1, - "This does foo:", - Newline, - List( - 1, - List.Unordered, - ListItem( - "bar", - Newline, - " ", - "baz", - Newline, - " ", - "Another raw text." - ) - ), - Newline - ) - ) - ) - """ This does foo: - | - bar - | baz - | - | Another raw text. - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Raw( - 1, - "This does foo:", - Newline, - List(1, List.Unordered, ListItem("bar", Newline, " ", "baz")), - Newline - ) - ), - Body( - Section.Raw( - 1, - "Another raw text.", - Newline - ) - ) - ) - - """ > Example - | This does foo: - | - bar - | baz - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Synopsis( - Section.Marked( - 1, - 1, - Section.Marked.Example, - Section.Header("Example"), - Newline, - "This does foo:", - Newline, - List(4, List.Unordered, ListItem("bar", Newline, " ", "baz")), - Newline - ) - ) - ) - - """ DEPRECATED das sfa asf - |REMOVED fdsdf - |Construct and manage a graphical, event-driven user interface for your iOS or - |tvOS app. - | - | foo *foo*""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Tags( - Tags.Tag(3, Tags.Tag.Type.Deprecated, " das sfa asf"), - Tags.Tag(0, Tags.Tag.Type.Removed, " fdsdf") - ), - Synopsis( - Section.Raw( - "Construct and manage a graphical, event-driven user interface for your iOS or", - Newline, - "tvOS app.", - Newline - ) - ), - Body(Section.Raw(1, "foo ", Formatter(Formatter.Bold, "foo"))) - ) - - """ DEPRECATED das sfa asf - |REMOVED fdsdf - |Construct and manage a graphical, event-driven user interface for your iOS or - |tvOS app. - | - | foo *foo""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Tags( - Tags.Tag(3, Tags.Tag.Type.Deprecated, " das sfa asf"), - Tags.Tag(0, Tags.Tag.Type.Removed, " fdsdf") - ), - Synopsis( - Section.Raw( - "Construct and manage a graphical, event-driven user interface for your iOS or", - Newline, - "tvOS app.", - Newline - ) - ), - Body(Section.Raw(1, "foo ", Formatter.Unclosed(Formatter.Bold, "foo"))) - ) - - """ DEPRECATED das sfa asf - | REMOVED - | Foo""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?= Doc( - Tags( - Tags.Tag(4, Tags.Tag.Type.Deprecated, " das sfa asf"), - Tags.Tag(2, Tags.Tag.Type.Removed) - ), - Synopsis(Section.Raw(1, "Foo")) - ) - - """ DEPRECATED das sfa asf - |REMOVED fdsdf - |ALIAS Convert From - |Construct and manage a graphical user interface for your iOS or - |tvOS app. - | - | fooo bar baz - | dsadasfsaf asfasfas - | asfasfa sf - | asfas fasf """.stripMargin - .replaceAll(System.lineSeparator(), "\n") ?= Doc( - Tags( - Tags.Tag(3, Tags.Tag.Type.Deprecated, " das sfa asf"), - Tags.Tag(0, Tags.Tag.Type.Removed, " fdsdf"), - Tags.Tag(0, Tags.Tag.Type.Alias, " Convert From") - ), - Synopsis( - Section.Raw( - "Construct and manage a graphical user interface for your iOS or", - Newline, - "tvOS app.", - Newline - ) - ), - Body( - Section.Raw( - 3, - "fooo bar baz", - Newline, - "dsadasfsaf asfasfas", - Newline, - "asfasfa sf", - Newline, - "asfas fasf " - ) - ) - ) - - """ALIAS New File - | - |Creates a new file object, pointing to the given path. - | - |> Example - | Create a new file pointing to the `data.csv` file in the project directory. - | - | import Standard.Base.System.File - | import Standard.Examples - | - | example_new = - | path = - | Examples.csv_path - | File.new path - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?== Doc( - Tags(Tags.Tag(0, Tags.Tag.Type.Alias, " New File")), - Synopsis(Section.Raw(0, Newline)), - Body( - Section.Raw( - 0, - "Creates a new file object, pointing to the given path.", - Newline - ), - Section.Marked( - 0, - 1, - Section.Marked.Example, - Section.Header("Example"), - Newline, - "Create a new file pointing to the ", - CodeBlock.Inline("data.csv"), - " file in the project directory.", - Newline, - CodeBlock( - CodeBlock.Line(6, "import Standard.Base.System.File"), - CodeBlock.Line(6, "import Standard.Examples"), - CodeBlock.Line(6, "example_new ="), - CodeBlock.Line(10, "path ="), - CodeBlock.Line(14, "Examples.csv_path"), - CodeBlock.Line(10, "File.new path") - ) - ) - ) - ) - - """ UNSTABLE - | - | Creates a new table from a vector of column names and a vector of vectors - | specifying row contents. - | - | Arguments: - | - header: A list of texts specifying the column names - | - rows: A vector of vectors, specifying the contents of each table row. The - | length of each element of `rows` must be equal in length to `header`. - | - | > Example - | Create a table with 3 columns, named `foo`, `bar`, and `baz`, containing - | `[1, 2, 3]`, `[True, False, True]`, and `['a', 'b', 'c']`, respectively. - | - | import Standard.Table - | - | example_from_rows = - | header = [ 'foo' , 'bar' , 'baz' ] - | row_1 = [ 1 , True , 'a' ] - | row_2 = [ 2 , False , 'b' ] - | row_3 = [ 3 , True , 'c' ] - | Table.from_rows header [row_1, row_2, row_3] - | - | Icon: table-from-rows - | Aliases: foo, bar baz, redshift® - |""".stripMargin.replaceAll(System.lineSeparator(), "\n") ?== Doc( - Tags(Tag(1, Tags.Tag.Type.Unstable, None)), - Synopsis(Section.Raw(1, Newline)), - Body( - Section.Raw( - 1, - "Creates a new table from a vector of column names and a vector of vectors", - Newline, - "specifying row contents.", - Newline - ), - Section.Raw( - 1, - "Arguments:", - Newline, - List( - 1, - List.Unordered, - ListItem("header: A list of texts specifying the column names"), - ListItem( - "rows: A vector of vectors, specifying the contents of each table row. The", - Newline, - " ", - "length of each element of ", - CodeBlock.Inline("rows"), - " must be equal in length to ", - CodeBlock.Inline("header"), - "." - ) - ), - Newline - ), - Section.Marked( - 1, - 1, - Section.Marked.Example, - Section.Header("Example"), - Newline, - "Create a table with 3 columns, named ", - CodeBlock.Inline("foo"), - ", ", - CodeBlock.Inline("bar"), - ", and ", - CodeBlock.Inline("baz"), - ", containing", - Newline, - CodeBlock.Inline("[1, 2, 3]"), - Text(", "), - CodeBlock.Inline("[True, False, True]"), - ", and ", - CodeBlock.Inline("['a', 'b', 'c']"), - ", respectively.", - Newline, - CodeBlock( - CodeBlock.Line(7, "import Standard.Table"), - CodeBlock.Line(7, "example_from_rows ="), - CodeBlock.Line(11, "header = [ 'foo' , 'bar' , 'baz' ]"), - CodeBlock.Line(11, "row_1 = [ 1 , True , 'a' ]"), - CodeBlock.Line(11, "row_2 = [ 2 , False , 'b' ]"), - CodeBlock.Line(11, "row_3 = [ 3 , True , 'c' ]"), - CodeBlock.Line(11, "Table.from_rows header [row_1, row_2, row_3]") - ) - ), - Section.Raw( - 1, - "Icon: table-from-rows", - Newline, - "Aliases: foo, bar baz, redshift", - "®", - Newline - ) - ) - ) - -} diff --git a/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/ParserTest.scala b/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/ParserTest.scala deleted file mode 100644 index 3e31f0d54f47..000000000000 --- a/lib/scala/syntax/specialization/shared/src/test/scala/org/enso/syntax/text/ParserTest.scala +++ /dev/null @@ -1,460 +0,0 @@ -package org.enso.syntax.text - -import org.enso.data.{List1, Shifted, Tree} -import org.enso.flexer.Reader -import org.enso.syntax.text.AST.Block.OptLine -import org.enso.syntax.text.AST._ -import org.enso.syntax.text.AST.conversions._ -import org.enso.syntax.text.Shape.SegmentPlain -import org.enso.syntax.text.ast.DSL._ -import org.scalatest._ -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -class ParserTest extends AnyFlatSpec with Matchers { - - def assertSpan(input: String, ast: AST): Assertion = { - val gotSpan = ast.span - val expectedSpan = new Reader(input).toString().length - gotSpan shouldEqual expectedSpan - } - - def assertModule(input: String, result: AST): Assertion = { - val parser = Parser() - val module = parser.run(input) - assertSpan(input, module) - val rmodule = parser.dropMacroMeta(module) - assert(rmodule == result) - assert(module.show() == new Reader(input).toString()) - } - - def assertExpr(input: String, result: AST): Assertion = { - val parser = Parser() - val module = parser.run(input) - assertSpan(input, module) - val rmodule = parser.dropMacroMeta(module) - val tail = module.lines.tail - if (!tail.forall(_.elem.isEmpty)) fail("Multi-line block") - else { - rmodule.lines.head.elem match { - case None => fail("Empty expression") - case Some(e) => - assert(e == result) - assert(module.show() == new Reader(input).toString()) - } - } - } - - def assertIdentity(input: String): Assertion = { - val module = Parser().run(input) - val idmap1 = module.idMap - val idmap2 = Parser().run(new Reader(input), idmap1).idMap - assertSpan(input, module) - assert(module.show() == new Reader(input).toString()) - assert(idmap1 == idmap2) - } - - implicit class TestString(input: String) { - def parseTitle(str: String): String = { - val maxChars = 20 - val escape = (str: String) => str.replace("\n", "\\n") - val str2 = escape(str) - val str3 = - if (str2.length < maxChars) str2 - else str2.take(maxChars) + "..." - s"parse `$str3`" - } - - private val testBase = it should parseTitle(input) - - def ?=(out: AST) = testBase in { assertExpr(input, out) } - def ??=(out: Module) = testBase in { assertModule(input, out) } - def testIdentity() = testBase in { assertIdentity(input) } - } - - ////////////////////////////////////////////////////////////////////////////// - //// Identifiers ///////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "_" ?= "_" - "Name" ?= "Name" - "name" ?= "name" - "name'" ?= "name'" - "name''" ?= "name''" - "name'a" ?= Ident.InvalidSuffix("name'", "a") - "name_" ?= "name_" - "name_'" ?= "name_'" - "name'_" ?= Ident.InvalidSuffix("name'", "_") - "name`" ?= "name" $ Invalid.Unrecognized("`") - "@Annotation" ?= Ident.Annotation("@Annotation") - - ////////////////////////////////////////////////////////////////////////////// - //// Operators /////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - import App.Section._ - import App.{Section => Sect} - - "++" ?= Sides("++") - "==" ?= Sides("==") - ":" ?= Sides(":") - "," ?= Sides(",") - "." ?= Sides(".") - ".." ?= Sides("..") - "..." ?= Sides("...") - ">=" ?= Sides(">=") - "<=" ?= Sides("<=") - "/=" ?= Sides("/=") - ".+" ?= Sect.Right(".", 0, Var("+")) - ".<*>" ?= Sect.Right(".", 0, Var("<*>")) - ".==" ?= Sect.Right(".", 0, Var("==")) - "Foo.+" ?= App.Infix(Cons("Foo"), 0, Opr("."), 0, Var("+")) - "+=" ?= Mod("+") - "-=" ?= Mod("-") - "===" ?= Ident.InvalidSuffix("==", "=") - "...." ?= Ident.InvalidSuffix("...", ".") - ">==" ?= Ident.InvalidSuffix(">=", "=") - "+==" ?= Ident.InvalidSuffix("+", "==") - - ////////////////////////////////////////////////////////////////////////////// - //// Precedence + Associativity ////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "a b" ?= ("a" $_ "b") - "a + b" ?= ("a" $_ "+") $__ "b" - "a + b + c" ?= ("a" $_ "+" $_ "b") $_ "+" $_ "c" - "a , b , c" ?= "a" $_ "," $_ ("b" $_ "," $_ "c") - "a + b * c" ?= "a" $_ "+" $_ ("b" $_ "*" $_ "c") - "a * b + c" ?= ("a" $_ "*" $_ "b") $_ "+" $_ "c" - "a+ b" ?= ("a" $ "+") $$_ "b" - "a +b" ?= "a" $_ ("+" $ "b") - "a+ +b" ?= ("a" $ "+") $$_ ("+" $ "b") - "*a+" ?= ("*" $ "a") $ "+" - "+a*" ?= "+" $ ("a" $ "*") - "+ <$> a <*> b" ?= (Sides("+") $_ "<$>" $_ "a") $_ "<*>" $_ "b" - "+ * ^" ?= Sect.Right("+", 1, Sect.Right("*", 1, Sides("^"))) - "+ ^ *" ?= Sect.Right("+", 1, Sect.Left(Sides("^"), 1, "*")) - "^ * +" ?= Sect.Left(Sect.Left(Sides("^"), 1, "*"), 1, "+") - "* ^ +" ?= Sect.Left(Sect.Right("*", 1, Sides("^")), 1, "+") - "^ + *" ?= App.Infix(Sides("^"), 1, "+", 1, Sides("*")) - "* + ^" ?= App.Infix(Sides("*"), 1, "+", 1, Sides("^")) - "a = b.c.d = 10" ?= "a" $_ "=" $_ (("b" $ "." $ "c" $ "." $ "d") $_ "=" $_ 10) - "v = f x=1 y=2" ?= "v" $_ "=" $_ ("f" $_ ("x" $ "=" $ 1) $_ ("y" $ "=" $ 2)) - "v' = v .x=1" ?= "v'" $_ "=" $_ ("v" $_ ("." $ "x" $ "=" $ 1)) - - ////////////////////////////////////////////////////////////////////////////// - //// Arrows ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "a -> b" ?= "a" $_ "->" $_ "b" - "a -> b -> c" ?= "a" $_ "->" $_ ("b" $_ "->" $_ "c") - "a b -> c d" ?= ("a" $_ "b") $_ "->" $_ ("c" $_ "d") - "a b-> c d" ?= "a" $_ ("b" $_ "->" $_ ("c" $_ "d")) - "a = b -> c d" ?= "a" $_ "=" $_ ("b" $_ "->" $_ ("c" $_ "d")) - "a = b-> c d" ?= "a" $_ "=" $_ ("b" $_ "->" $_ ("c" $_ "d")) - "a + b -> c d" ?= ("a" $_ "+" $_ "b") $_ "->" $_ ("c" $_ "d") - "a + b-> c d" ?= "a" $_ "+" $_ ("b" $_ "->" $_ ("c" $_ "d")) - "a + b-> c = d" ?= "a" $_ "+" $_ ("b" $_ "->" $_ ("c" $_ "=" $_ "d")) - "a = b -> c = d" ?= "a" $_ "=" $_ ("b" $_ "->" $_ ("c" $_ "=" $_ "d")) - - ////////////////////////////////////////////////////////////////////////////// - //// Layout ////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "" ??= Module(OptLine()) - "\n" ??= Module(OptLine(), OptLine()) - " \n " ??= Module(OptLine(2), OptLine(1)) - "\n\n" ??= Module(OptLine(), OptLine(), OptLine()) - " \n \n " ??= Module(OptLine(1), OptLine(2), OptLine(3)) - - ////////////////////////////////////////////////////////////////////////////// - //// Numbers ///////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "7" ?= 7 - "07" ?= Number("07") - "10_7" ?= Number(10, 7) - "16_ff" ?= Number(16, "ff") - "16_" ?= Number.DanglingBase("16") - "7.5" ?= App.Infix(7, 0, Opr("."), 0, 5) - - ////////////////////////////////////////////////////////////////////////////// - //// UTF Surrogates ////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "\uD800\uDF1E" ?= Invalid.Unrecognized("\uD800\uDF1E") - - ////////////////////////////////////////////////////////////////////////////// - //// Text //////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - import Shape.SegmentPlain.txtFromString - def line(s: String, empty: Int*) = - Shape.TextBlockLine(empty.toList, List(txtFromString[AST](s))) - def line(segment: AST.Text.Segment.Fmt, empty: Int*) = - Shape.TextBlockLine(empty.toList, List(segment)) - - "'" ?= Text.Unclosed() - "''" ?= Text() - "'''" ?= Text(0, 0) - "'''a" ?= Text.InlineBlock("'''") $ "a" - "''a" ?= Text() $ "a" - "'a'" ?= Text("a") - "'a" ?= Text.Unclosed("a") - "'a''" ?= Text.Unclosed("a") $ Text.InvalidQuote("''") - "'\"'" ?= Text("\"") - - "''' \n\n X\n\n Y" ?= Text(1, 0, line(" X", 0), line(" Y", 0)) - "a '''\n\n\n X\n\n Y" ?= "a" $_ Text(0, 1, line("X", 0, 0), line("Y", 0)) - - "\"" ?= Text.Unclosed.Raw() - "\"\"" ?= Text.Raw() - "\"\"\"" ?= Text.Raw(0, 0) - "\"\"\"a" ?= Text.InlineBlock("\"\"\"") $ "a" - "\"\"a" ?= Text.Raw() $ "a" - "\"a\"" ?= Text.Raw("a") - "\"a" ?= Text.Unclosed.Raw("a") - "\"a\"\"" ?= Text.Unclosed.Raw("a") $ Text.InvalidQuote("\"\"") - "\"'\"" ?= Text.Raw("'") - - "\"\"\" \n\n X\n\n Y" ?= Text.Raw(1, 0, line(" X", 0), line(" Y", 0)) - "a \"\"\"\n\n\n X\n\n Y" ?= "a" $_ Text.Raw( - 0, - 1, - line("X", 0, 0), - line("Y", 0) - ) - - //// Escapes //// - - val Esc = Text.Segment.Escape - def escape(code: Text.Segment.Escape) = Shape.SegmentEscape[AST](code) - def escape(code: Text.Segment.RawEscape) = Shape.SegmentRawEscape[AST](code) - - Text.Segment.Escape.Character.codes.foreach(i => s"'\\$i'" ?= Text(escape(i))) - Text.Segment.Escape.Control.codes.foreach(i => s"'\\$i'" ?= Text(escape(i))) - - "'\\\\'" ?= Text(escape(Esc.Slash)) - "'\\''" ?= Text(escape(Esc.Quote)) - "'\\" ?= Text.Unclosed(escape(Esc.Unfinished)) - "'\\c'" ?= Text(escape(Esc.Invalid('c'))) - "'\\cd'" ?= Text(escape(Esc.Invalid('c')), "d") - "'\\123d'" ?= Text(escape(Esc.Number(123)), "d") - - "\"\\\\\"" ?= Text.Raw("\\\\") - "\"\\\"" ?= Text.Raw("\\") - "\"\\" ?= Text.Unclosed.Raw(SegmentPlain("\\")) - "\"\\cd\"" ?= Text.Raw("\\cd") - - //// Interpolation //// - - def expr(ast: AST) = Shape.SegmentExpr(Some(ast)) - - "'a`b`c'" ?= Text("a", expr("b"), "c") - "'a`b 'c`d`e' f`g'" ?= { - val bd = "b" $_ Text("c", expr("d"), "e") $_ "f" - Text("a", expr(bd), "g") - } - - "say \n '''\n Hello\n `World`\npal" ??= Module( - OptLine( - "say" $_ Block(2, Text(0, 2, line("Hello"), line(expr("World")))) - ), - OptLine("pal") - ) - - "say '''\n Hello\n `World`\npal" ??= Module( - OptLine("say" $_ Text(0, 2, line("Hello"), line(expr("World")))), - OptLine("pal") - ) - -//// // // Comments -////// expr("#" , Comment) -////// expr("#c" , Comment :: CommentBody("c")) -//// // expr("#c\na" , Comment :: CommentBody("c") :: EOL :: Var("a")) -//// // expr("#c\n a" , Comment :: CommentBody("c") :: EOL :: CommentBody(" a")) -//// // expr(" #c\n a" , Comment :: CommentBody("c") :: EOL :: Var("a")) -//// // expr(" #c\n a" , Comment :: CommentBody("c") :: EOL :: CommentBody(" a")) -//// // expr("a#c" , Var("a") :: Comment :: CommentBody("c")) -//// // expr("a # c" , Var("a") :: Comment :: CommentBody(" c")) -//// // expr("a#" , Var("a") :: Comment) -//// // expr("a#\nb" , Var("a") :: Comment :: EOL :: Var("b")) -//// // expr("a#\n b" , Var("a") :: Comment :: EOL :: CommentBody(" b")) -//// // -//// // // Disabled -//// // expr("a #= b" , Var("a") :: DisabledAssignment :: Var("b")) -//// // -// -// ////////////////////////////////////////////////////////////////////////////// -// //// Comments //////////////////////////////////////////////////////////////// -// ////////////////////////////////////////////////////////////////////////////// -// -//// "foo ##L1" ?= "foo" $___ Comment.SingleLine("L1") -//// "##\n L1\n L2" ?= Comment.MultiLine(0, List("", " L1", " L2")) -//// "##L1\nL2" ??= Module(OptLine(Comment.SingleLine("L1")), OptLine(Cons("L2"))) -//// "foo #a b" ?= "foo" $_ Comment.Disable("a" $_ "b") -// - ////////////////////////////////////////////////////////////////////////////// - //// Flags /////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "x = skip a" ?= "x" $_ "=" $_ "a" - "x = skip a.fn" ?= "x" $_ "=" $_ "a" - "x = skip fn a" ?= "x" $_ "=" $_ "a" - "x = skip (a)" ?= "x" $_ "=" $_ "a" - "x = skip (a.fn)" ?= "x" $_ "=" $_ "a" - "x = skip (a + b)" ?= "x" $_ "=" $_ "a" - "x = skip ((a + b) + c)" ?= "x" $_ "=" $_ "a" - "x = skip ()" ?= "x" $_ "=" $_ Group() -// "a = freeze b c" ?= "a" $_ "#=" $_ ("b" $_ "c") // freeze - - ////////////////////////////////////////////////////////////////////////////// - //// Mixfixes //////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - def amb(head: AST, lst: List[List[AST]]): Macro.Ambiguous = - Macro.Ambiguous( - Shifted.List1(Macro.Ambiguous.Segment(head)), - Tree(lst.map(_ -> ()): _*) - ) - - def amb(head: AST, lst: List[List[AST]], body: SAST): Macro.Ambiguous = - Macro.Ambiguous( - Shifted.List1(Macro.Ambiguous.Segment(head, Some(body))), - Tree(lst.map(_ -> ()): _*) - ) - - def _amb_group_(i: Int)(t: AST): Macro.Ambiguous = - amb("(", List(List(")")), Shifted(i, t)) - - val amb_group = _amb_group_(0)(_) - val amb_group_ = _amb_group_(1)(_) - val amb_group__ = _amb_group_(2)(_) - def group_(): Macro.Ambiguous = amb("(", List(List(")"))) - - def _amb_if(i: Int)(t: AST) = - amb("if", List(List("then"), List("then", "else")), Shifted(i, t)) - - val amb_if = _amb_if(0)(_) - val amb_if_ = _amb_if(1)(_) - val amb_if__ = _amb_if(2)(_) - - "()" ?= Group() - "( )" ?= Group() - "( ( ) )" ?= Group(Group()) - "(a)" ?= Group("a") - "(+a)" ?= Group("+" $ "a") - "(a+)" ?= Group("a" $ "+") - "((a))" ?= Group(Group("a")) - "(((a)))" ?= Group(Group(Group("a"))) - "( ( a ))" ?= Group(Group("a")) - "(a) (b)" ?= Group("a") $_ Group("b") - "(" ?= amb("(", List(List(")"))) - "((" ?= amb_group(group_()) - - """type Maybe a - | type Just val:a - | type Nothing""".stripMargin ?= { - val defJust = Def("Just", List("val" $ ":" $ "a")) - val defNothing = Def("Nothing") - Def( - "Maybe", - List("a"), - Some(Block(4, defJust, defNothing)) - ) - } -// -// """foo -> -// | bar -// |""".stripMargin ?= "foo" $_ "->" $_ Block( -// Block.Discontinuous, -// 4, -// "bar", -// None -// ) -// - "if a then b" ?= Mixfix( - List1[AST.Ident]("if", "then"), - List1[AST]("a", "b") - ) - "if a then b else c" ?= Mixfix( - List1[AST.Ident]("if", "then", "else"), - List1[AST]("a", "b", "c") - ) - - "if a" ?= amb_if_("a": AST) - "(if a) b" ?= Group(amb_if_("a": AST)) $_ "b" - "if (a then b " ?= amb_if_(amb_group("a" $_ "then" $_ "b")) - -// ////////////////////////////////////////////////////////////////////////////// -// //// Foreign ///////////////////////////////////////////////////////////////// -// ////////////////////////////////////////////////////////////////////////////// -// -// "f = foreign Python3\n a" ?= "f" $_ "=" $_ Foreign(1, "Python3", List("a")) -// -// val pyLine1 = "import re" -// val pyLine2 = """re.match(r"[^@]+@[^@]+\.[^@]+", "foo@ds.pl") != None""" -// s"""validateEmail address = foreign Python3 -// | $pyLine1 -// | $pyLine2""".stripMargin ?= ("validateEmail" $_ "address") $_ "=" $_ -// Foreign(4, "Python3", List(pyLine1, pyLine2)) - - ////////////////////////////////////////////////////////////////////////////// - //// Large Input ///////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - -// ("(" * 33000).testIdentity() // FIXME Stack Overflow -// ("OVERFLOW " * 5000).testIdentity() -// ("\uD800\uDF1E " * 10000).testIdentity() - - ////////////////////////////////////////////////////////////////////////////// - //// OTHER (TO BE PARTITIONED)//////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////////// - - "\na \nb \n".testIdentity() - "f = \n\n\n".testIdentity() - " \n\n\n f\nf".testIdentity() - "f = \n\n x ".testIdentity() - " a\n b\n c".testIdentity() - "f =\n\n x\n\n y".testIdentity() - - """ - a - b - c - d - e - f g h - """.testIdentity() - - """ - armageddon adults children mutants = - log ''' - keepBest - `pop1` - `pop2` - `pop3` - unique xs - = xs.at(0.0) +: [1..length xs -1] . filter (isUnique xs) . map xs.at - isUnique xs i - = xs.at(i).score != xs.at(i-1).score - adults++children++mutants . sorted . unique . take (length pop1) . pure - """.testIdentity() - - /////////////////////// - //// Preprocessing //// - /////////////////////// - - "\t" ??= Module(OptLine(4)) - "\r" ??= Module(OptLine(), OptLine()) - "\r\n" ??= Module(OptLine(), OptLine()) - -} -//////////////////////////////////////////////////////////////////////////// -// TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO // -//////////////////////////////////////////////////////////////////////////// - -// [ ] operator blocks -// [ ] warnings in scala code -// [ ] Undefined parsing -// [ ] All block types diff --git a/lib/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala b/lib/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala new file mode 100644 index 000000000000..767d6c698177 --- /dev/null +++ b/lib/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala @@ -0,0 +1,64 @@ +package org.enso.syntax.text + +import org.enso.data.Span +import io.circe +import io.circe.Json +import io.circe.generic.auto._ +import io.circe.parser._ + +import java.util.UUID + +//////////////////////////////// + +class InternalError(reason: String, cause: Throwable = None.orNull) + extends Exception(s"Internal error $reason", cause) + +//////////////// +//// Parser //// +//////////////// + +object SourceFile { + val METATAG = "\n\n\n#### METADATA ####\n" +} + +class Parser { + import Parser._ + + def splitMeta(code: String): (String, IDMap, Json) = { + import SourceFile._ + code.split(METATAG) match { + case Array(input) => (input, Seq(), Json.obj()) + case Array(input, rest) => + val meta = rest.split('\n') + if (meta.length < 2) { + throw new ParserError(s"Expected two lines after METADATA.") + } + val idmap = idMapFromJson(meta(0)).left.map { error => + throw new ParserError("Could not deserialize idmap.", error) + }.merge + val metadata = decode[Json](meta(1)).left.map { error => + throw new ParserError("Could not deserialize metadata.", error) + }.merge + (input, idmap, metadata) + case arr: Array[_] => + throw new ParserError( + s"Could not not deserialize metadata (found ${arr.length - 1} metadata sections)" + ) + } + } +} + +object Parser { + + type IDMap = Seq[(Span, UUID)] + + def apply(): Parser = new Parser() + + def idMapFromJson(json: String): Either[circe.Error, IDMap] = + decode[IDMap](json) + + //// Exceptions //// + + class ParserError(reason: String, cause: Throwable = None.orNull) + extends InternalError(s"in parser $reason", cause) +} diff --git a/project/NativeImage.scala b/project/NativeImage.scala index c1d238132e32..717380da7a91 100644 --- a/project/NativeImage.scala +++ b/project/NativeImage.scala @@ -188,7 +188,7 @@ object NativeImage { Tracked.diffInputs(store, FileInfo.hash)(filesSet) { sourcesDiff: ChangeReport[File] => if (sourcesDiff.modified.nonEmpty) - rebuild("Native Image is not up to date") + rebuild(s"Native Image is not up to date") else if (!artifactFile(artifactName).exists()) rebuild("Native Image does not exist") else diff --git a/project/RecompileParser.scala b/project/RecompileParser.scala index d3f760843afe..1bc33190bf08 100644 --- a/project/RecompileParser.scala +++ b/project/RecompileParser.scala @@ -1,7 +1,5 @@ import sbt._ import sbt.Keys._ -import sbtcrossproject.CrossProject -import sbtcrossproject.CrossPlugin.autoImport._ object RecompileParser { @@ -9,10 +7,10 @@ object RecompileParser { * `syntaxDefinition` is changed. Should be attached to the `compile` task as * a dependency. */ - def run(syntaxDefinition: CrossProject) = + def run(syntaxDefinition: Project) = Def.taskDyn { val parserCompile = - (syntaxDefinition.jvm / Compile / compileIncremental).value + (syntaxDefinition / Compile / compileIncremental).value if (parserCompile.hasModified) { Def.task { streams.value.log.info("Parser changed, forcing recompilation.") diff --git a/project/plugins.sbt b/project/plugins.sbt index 535e42a518b0..7dbd6735aaa8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,11 +1,9 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") -addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.3") -addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.2.0") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.10.1") -addSbtPlugin("com.typesafe.sbt" % "sbt-license-report" % "1.2.0") -addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.7.0") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") -addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") +addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.3") +addSbtPlugin("com.typesafe.sbt" % "sbt-license-report" % "1.2.0") +addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.7.0") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") libraryDependencies += "io.circe" %% "circe-yaml" % "0.14.1" libraryDependencies += "commons-io" % "commons-io" % "2.11.0" diff --git a/tools/legal-review/Base/report-state b/tools/legal-review/Base/report-state index 5eac38682aa2..ddbce83cc740 100644 --- a/tools/legal-review/Base/report-state +++ b/tools/legal-review/Base/report-state @@ -1,3 +1,3 @@ 2E31BAAC203253896E596DD20144363C2C54799F303D675D2B2D7E45743CC7F3 -4D6C4A246DC46C5A878347CB17ED2807834D4772D4D1EA813BE7A5D3FC33BB50 +9EE0B9E749EEDAA6A1ABDC69F52BEDD1BF1E8F91FB9A196BFC09DEB2F57D85B9 0 diff --git a/tools/legal-review/Database/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.12.6/files-ignore b/tools/legal-review/Database/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.12.6/files-ignore index c3349a9e9ede..82f5404b1ffd 100644 --- a/tools/legal-review/Database/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.12.6/files-ignore +++ b/tools/legal-review/Database/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.12.6/files-ignore @@ -1 +1,2 @@ #license +/FasterXML/jackson-dataformats-binary/blob/2.15/LICENSE diff --git a/tools/legal-review/Database/report-state b/tools/legal-review/Database/report-state index 64be9312e2ea..2d8a2833ff1b 100644 --- a/tools/legal-review/Database/report-state +++ b/tools/legal-review/Database/report-state @@ -1,3 +1,3 @@ 87AFCC58BE0E7EE9CF656C3750BE38C23B6DEC25E1918CB469E77287E9F654C0 -FA6CED83FBDF5A14CE6F21E398ECE68EA9AE98FDBF6F92C4B00D9F2CD3159CF8 +99D73D03217F8E7D41288717A740C13DB765186A8246C4C6813E24ED23142A4C 0 diff --git a/tools/legal-review/Google_Api/report-state b/tools/legal-review/Google_Api/report-state index adc928d50b78..d0b4540dc531 100644 --- a/tools/legal-review/Google_Api/report-state +++ b/tools/legal-review/Google_Api/report-state @@ -1,3 +1,3 @@ 91501165A015BCF6A393EF77FABE0098B3B62040E155A7F0F90053FC04179CD5 -DE6F86DC6C33E246D92FD2F5548E184B6B44BDDA172814EB7BAE3710C5C2F74C +08C1BF0537AC8146E59134D2FA815E79345CE9706FE951198D5585BA31964340 0 diff --git a/tools/legal-review/Image/report-state b/tools/legal-review/Image/report-state index a7ec298df340..c7f2c5d58713 100644 --- a/tools/legal-review/Image/report-state +++ b/tools/legal-review/Image/report-state @@ -1,3 +1,3 @@ 53E78E605D84E066AB36B506248124E1E412C964A67728C6D5CE0180EA3B0AD9 -F45A653DBF23487883EB2314DAA7F86DDC966873E336AEE24DDF213262110D80 +308CFBC112907BAC523DBD32479726A6674371822C2438020A073250E05037B0 0 diff --git a/tools/legal-review/Table/report-state b/tools/legal-review/Table/report-state index 565a0f0f3b28..a23b2d018bfa 100644 --- a/tools/legal-review/Table/report-state +++ b/tools/legal-review/Table/report-state @@ -1,3 +1,3 @@ 840031EDBA6D7166EE1BABF8D1AB65F7219F5258683A2D487D12D3D4B8387BD7 -4BC5787A7330388C3B8BF8C5955FEFB57E57CB47DFAA243180AF0DA066E3D0D6 +E9D17B826DDD13DD281EB7B162F73B5458984A61D1E7DE7C2B00A03B3F156A56 0 diff --git a/tools/legal-review/engine/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore b/tools/legal-review/engine/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore index c3349a9e9ede..82f5404b1ffd 100644 --- a/tools/legal-review/engine/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore +++ b/tools/legal-review/engine/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore @@ -1 +1,2 @@ #license +/FasterXML/jackson-dataformats-binary/blob/2.15/LICENSE diff --git a/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/custom-license b/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/custom-license deleted file mode 100644 index 6b1d0bfabc3c..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/custom-license +++ /dev/null @@ -1 +0,0 @@ -LICENSE diff --git a/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/files-keep b/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/files-keep deleted file mode 100644 index aac49611ba17..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.geny_2.13-0.6.10/files-keep +++ /dev/null @@ -1 +0,0 @@ -/com-lihaoyi/geny/blob/main/LICENSE diff --git a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-ignore b/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-ignore deleted file mode 100644 index 699945dbcb0a..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-ignore +++ /dev/null @@ -1 +0,0 @@ -Represents a side comment; text like a disclaimer or copyright, which is not diff --git a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-keep b/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-keep deleted file mode 100644 index d881ec537678..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/copyright-keep +++ /dev/null @@ -1 +0,0 @@ -Defines the footer for a page or section. It often contains a copyright diff --git a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/custom-license b/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/custom-license deleted file mode 100644 index 35252fda76e8..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/custom-license +++ /dev/null @@ -1 +0,0 @@ -LICENSE.txt diff --git a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/files-add/LICENSE.txt b/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/files-add/LICENSE.txt deleted file mode 100644 index 2145e7996cd3..000000000000 --- a/tools/legal-review/engine/com.lihaoyi.scalatags_2.13-0.11.1/files-add/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2018 Li Haoyi (haoyi.sg@gmail.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/copyright-add b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/copyright-add new file mode 100644 index 000000000000..d390f204f1b3 --- /dev/null +++ b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/copyright-add @@ -0,0 +1,15 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/custom-license b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/custom-license deleted file mode 100644 index 6b1d0bfabc3c..000000000000 --- a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/custom-license +++ /dev/null @@ -1 +0,0 @@ -LICENSE diff --git a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-ignore b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-ignore index c3349a9e9ede..d1c1ec7cede9 100644 --- a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-ignore +++ b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-ignore @@ -1 +1,2 @@ +/circe/circe-yaml/blob/main/LICENSE #license diff --git a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-keep b/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-keep deleted file mode 100644 index 6fd04cbb4280..000000000000 --- a/tools/legal-review/engine/io.circe.circe-yaml_2.13-0.14.1/files-keep +++ /dev/null @@ -1 +0,0 @@ -/circe/circe-yaml/blob/master/LICENSE diff --git a/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-ignore b/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-ignore new file mode 100644 index 000000000000..cfc27f100295 --- /dev/null +++ b/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-ignore @@ -0,0 +1,11 @@ +Copyright (c) 2007 Timothy Wall +Copyright (c) 2007-2008 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2012 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2013 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2015 Timothy Wall, All Rights Reserved +Copyright (c) 2009 Timothy Wall, All Rights Reserved +Copyright (c) 2011 Timothy Wall, All Rights Reserved +Copyright (c) 2012 Timothy Wall, All Rights Reserved +Copyright (c) 2018 Matthias Bläsing +Copyright (c) 2019 Matthias Bläsing, All Rights Reserved +Copyright 2007 Timothy Wall diff --git a/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-keep b/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-keep new file mode 100644 index 000000000000..ab004d769c85 --- /dev/null +++ b/tools/legal-review/engine/net.java.dev.jna.jna-5.5.0/copyright-keep @@ -0,0 +1,3 @@ +Copyright (c) 2007 Timothy Wall, All Rights Reserved +Copyright (c) 2007 Wayne Meissner, All Rights Reserved +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved diff --git a/tools/legal-review/engine/report-state b/tools/legal-review/engine/report-state index 081f9e1dbc79..0d6f3e76d9ad 100644 --- a/tools/legal-review/engine/report-state +++ b/tools/legal-review/engine/report-state @@ -1,3 +1,3 @@ -1B53BC9C81329A6179E621F07FD2E05E655D0A2BA5B92ED15057676D34EE4737 -E6F042346A7544959A9B053257F92C1885133D5ABC00D736474FF76C5F02F5EC +5F7F10B5303656FC01C4D26138CF2035606E7A272ADFD5419121097F009BE008 +7C1A29901335B64467B939EFDE5AC11ECC703ECF503DAEE810B20EF9F48827EE 0 diff --git a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/copyright-add b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/copyright-add new file mode 100644 index 000000000000..d390f204f1b3 --- /dev/null +++ b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/copyright-add @@ -0,0 +1,15 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/custom-license b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/custom-license deleted file mode 100644 index 6b1d0bfabc3c..000000000000 --- a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/custom-license +++ /dev/null @@ -1 +0,0 @@ -LICENSE diff --git a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-ignore b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-ignore index c3349a9e9ede..d1c1ec7cede9 100644 --- a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-ignore +++ b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-ignore @@ -1 +1,2 @@ +/circe/circe-yaml/blob/main/LICENSE #license diff --git a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-keep b/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-keep deleted file mode 100644 index 6fd04cbb4280..000000000000 --- a/tools/legal-review/launcher/io.circe.circe-yaml_2.13-0.14.1/files-keep +++ /dev/null @@ -1 +0,0 @@ -/circe/circe-yaml/blob/master/LICENSE diff --git a/tools/legal-review/launcher/report-state b/tools/legal-review/launcher/report-state index 2ef9ee074dc5..1ee2c0e65b70 100644 --- a/tools/legal-review/launcher/report-state +++ b/tools/legal-review/launcher/report-state @@ -1,3 +1,3 @@ 5CB5929D58A0BB32D644A378015717936213AC7B9E0707646AF26BAC06E2F8C8 -92FE6963CA39BC46E85F6D30712C97010A0D287EBD80FAA62FFD5F3AD5CD4636 +A6CEFDD85FE55ABB13D9767164E2CB8FBCBA618FA1547D76BE245D7964A48E64 0 diff --git a/tools/legal-review/project-manager/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore b/tools/legal-review/project-manager/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore index c3349a9e9ede..82f5404b1ffd 100644 --- a/tools/legal-review/project-manager/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore +++ b/tools/legal-review/project-manager/com.fasterxml.jackson.dataformat.jackson-dataformat-cbor-2.13.3/files-ignore @@ -1 +1,2 @@ #license +/FasterXML/jackson-dataformats-binary/blob/2.15/LICENSE diff --git a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/copyright-add b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/copyright-add new file mode 100644 index 000000000000..d390f204f1b3 --- /dev/null +++ b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/copyright-add @@ -0,0 +1,15 @@ +/* + * Copyright 2016 circe + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/custom-license b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/custom-license deleted file mode 100644 index 6b1d0bfabc3c..000000000000 --- a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/custom-license +++ /dev/null @@ -1 +0,0 @@ -LICENSE diff --git a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-ignore b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-ignore index c3349a9e9ede..d1c1ec7cede9 100644 --- a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-ignore +++ b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-ignore @@ -1 +1,2 @@ +/circe/circe-yaml/blob/main/LICENSE #license diff --git a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-keep b/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-keep deleted file mode 100644 index 6fd04cbb4280..000000000000 --- a/tools/legal-review/project-manager/io.circe.circe-yaml_2.13-0.14.1/files-keep +++ /dev/null @@ -1 +0,0 @@ -/circe/circe-yaml/blob/master/LICENSE diff --git a/tools/legal-review/project-manager/report-state b/tools/legal-review/project-manager/report-state index 413ada307b49..e75a25ee4eee 100644 --- a/tools/legal-review/project-manager/report-state +++ b/tools/legal-review/project-manager/report-state @@ -1,3 +1,3 @@ 27E62459A95B059D0BA10416285FF20C38FEB28E12343177457C5A7912BB22CD -ECC53C4E1635B01E3D89BAB15C533F92C7DC87D5BAABFC399D28CEC364C9DC4F +5D78EC2970BDDC32CF6BBDFA72CEFF618C97C893886687DC96CFB2919995861F 0