From 7cad9c0567c27e365f7c3606aeba8b9ec4e7c848 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Wed, 16 Oct 2024 19:32:36 +0200 Subject: [PATCH 1/2] re-add syntect --- Cargo.lock | 81 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 17 +++++--- src/html/comrak_adapters.rs | 66 ++++++++++++++++++++++++++++++ src/html/jsdoc.rs | 4 ++ src/html/mod.rs | 9 +++++ tests/html_test.rs | 4 ++ 6 files changed, 173 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1784990..c2953e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,15 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -376,6 +385,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.4.0" @@ -580,6 +598,7 @@ dependencies = [ "regex", "serde", "serde_json", + "syntect", "termcolor", "tokio", "tree-sitter-bash", @@ -614,9 +633,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.83.2" +version = "0.83.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e043fe1a12c83bb603f03f56e84ba9f63b1ba240c4e54a10cab76896a1faf65b" +checksum = "8c62ce152f24a4c0580e7a91431f75de48281157cf645459de8e9d7268dd95b2" dependencies = [ "anyhow", "async-trait", @@ -799,6 +818,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1322,6 +1351,28 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oorandom" version = "11.1.4" @@ -1503,6 +1554,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "plotters" version = "0.3.7" @@ -2252,6 +2309,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syntect" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1" +dependencies = [ + "bincode", + "bitflags 1.3.2", + "flate2", + "fnv", + "once_cell", + "onig", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "thiserror", + "walkdir", +] + [[package]] name = "tap" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 9c5742a1..192d5207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,14 @@ termcolor = "1.4.1" html-escape = { version = "0.2.13", optional = true } comrak = { version = "0.28.0", optional = true, default-features = false } handlebars = { version = "6.1", optional = true, features = ["string_helpers"] } +syntect = { version = "5.2.0", optional = true, default-features = false, features = [ + "parsing", + "default-syntaxes", + "default-themes", + "html", + "dump-load", + "regex-onig", +] } ammonia = { version = "4.0.0", optional = true } tree-sitter-highlight = { version = "0.22.6", optional = true } @@ -67,13 +75,10 @@ pretty_assertions = "1.4.0" insta = { version = "1.39.0", features = ["json"] } [features] -default = ["html", "rust"] +default = ["html", "rust", "ammonia", "tree-sitter"] rust = [] -ammonia = ["dep:ammonia"] -html = [ - "html-escape", - "comrak", - "handlebars", +html = ["html-escape", "comrak", "handlebars"] +tree-sitter = [ "tree-sitter-highlight", "tree-sitter-javascript", "tree-sitter-typescript", diff --git a/src/html/comrak_adapters.rs b/src/html/comrak_adapters.rs index 79fd9f1d..2ebcd135 100644 --- a/src/html/comrak_adapters.rs +++ b/src/html/comrak_adapters.rs @@ -1,5 +1,17 @@ +// Copied and modified from https://github.com/kivikakk/comrak/blob/main/src/plugins/syntect.rs + +//! Adapter for the Syntect syntax highlighter plugin. + #![allow(clippy::print_stderr)] +#[cfg(any( + not(any(feature = "syntect", feature = "tree-sitter")), + all(feature = "syntect", feature = "tree-sitter") +))] +compile_error!( + "Either feature \"syntect\" or \"tree-sitter\" must be enabled, not both or neither." +); + use comrak::adapters::HeadingAdapter; use comrak::adapters::HeadingMeta; use comrak::adapters::SyntaxHighlighterAdapter; @@ -11,7 +23,13 @@ use std::sync::Arc; use std::sync::Mutex; #[derive(Debug)] +/// Syntect syntax highlighter plugin. pub struct HighlightAdapter { + #[cfg(feature = "syntect")] + pub syntax_set: syntect::parsing::SyntaxSet, + #[cfg(feature = "syntect")] + pub theme_set: syntect::highlighting::ThemeSet, + #[cfg(feature = "tree-sitter")] pub language_cb: fn(&str) -> Option<&'static tree_sitter_highlight::HighlightConfiguration>, pub show_line_numbers: bool, @@ -76,6 +94,54 @@ impl HighlightAdapter { } impl SyntaxHighlighterAdapter for HighlightAdapter { + #[cfg(all(feature = "syntect", not(feature = "tree-sitter")))] + fn write_highlighted( + &self, + output: &mut dyn Write, + lang: Option<&str>, + code: &str, + ) -> std::io::Result<()> { + let lang = match lang { + Some(l) if !l.is_empty() => l, + _ => "Plain Text", + }; + + let syntax = + self + .syntax_set + .find_syntax_by_token(lang) + .unwrap_or_else(|| { + self + .syntax_set + .find_syntax_by_first_line(code) + .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text()) + }); + + let theme = &self.theme_set.themes["InspiredGitHub"]; + let mut highlighter = syntect::easy::HighlightLines::new(syntax, theme); + + match self.highlight_html( + syntect::util::LinesWithEndings::from(code), + |lines, line| { + let regions = highlighter.highlight_line(line, &self.syntax_set)?; + + syntect::html::append_highlighted_html_for_styled_line( + ®ions, + syntect::html::IncludeBackground::No, + lines, + )?; + + Ok(()) + }, + ) { + Ok(highlighted_code) => output.write_all(highlighted_code.as_bytes())?, + Err(_) => output.write_all(code.as_bytes())?, + } + + self.write_button(output, code) + } + + #[cfg(all(feature = "tree-sitter", not(feature = "syntect")))] fn write_highlighted( &self, output: &mut dyn Write, diff --git a/src/html/jsdoc.rs b/src/html/jsdoc.rs index 09427366..8446f6f1 100644 --- a/src/html/jsdoc.rs +++ b/src/html/jsdoc.rs @@ -84,6 +84,10 @@ lazy_static! { AmmoniaRelativeUrlEvaluator(), ))); + #[cfg(feature = "syntect")] + ammonia_builder.add_tag_attributes("span", ["style"]); + + #[cfg(feature = "tree-sitter")] ammonia_builder.add_allowed_classes("span", super::tree_sitter::CLASSES); ammonia_builder diff --git a/src/html/mod.rs b/src/html/mod.rs index 116922e3..1d3380b6 100644 --- a/src/html/mod.rs +++ b/src/html/mod.rs @@ -18,6 +18,7 @@ pub mod partition; mod render_context; mod search; mod symbols; +#[cfg(feature = "tree-sitter")] pub mod tree_sitter; mod types; mod usage; @@ -794,6 +795,14 @@ pub fn setup_highlighter( show_line_numbers: bool, ) -> comrak_adapters::HighlightAdapter { comrak_adapters::HighlightAdapter { + #[cfg(feature = "syntect")] + syntax_set: syntect::dumps::from_uncompressed_data(include_bytes!( + "./default_newlines.packdump" + )) + .unwrap(), + #[cfg(feature = "syntect")] + theme_set: syntect::highlighting::ThemeSet::load_defaults(), + #[cfg(feature = "tree-sitter")] language_cb: tree_sitter::tree_sitter_language_cb, show_line_numbers, } diff --git a/tests/html_test.rs b/tests/html_test.rs index fb5dc54d..e3780b47 100644 --- a/tests/html_test.rs +++ b/tests/html_test.rs @@ -182,6 +182,7 @@ async fn html_doc_files() { ] ); + #[cfg(feature = "tree-sitter")] { insta::assert_snapshot!(files.get("./all_symbols.html").unwrap()); insta::assert_snapshot!(files.get("./index.html").unwrap()); @@ -272,6 +273,7 @@ async fn html_doc_files_rewrite() { ] ); + #[cfg(feature = "tree-sitter")] { insta::assert_snapshot!(files.get("./all_symbols.html").unwrap()); insta::assert_snapshot!(files.get("./index.html").unwrap()); @@ -385,6 +387,7 @@ async fn symbol_group() { } } + #[cfg(feature = "tree-sitter")] insta::assert_json_snapshot!(files); } @@ -483,5 +486,6 @@ async fn module_doc() { module_docs.push(module_doc); } + #[cfg(feature = "tree-sitter")] insta::assert_json_snapshot!(module_docs); } From de3851d0e09c1efe3283cb0f44456ab6012f20d7 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Wed, 16 Oct 2024 19:41:52 +0200 Subject: [PATCH 2/2] fix --- .../html_test__html_doc_files-3.snap | 8 +- .../html_test__html_doc_files-5.snap | 16 +-- .../html_test__html_doc_files-7.snap | 8 +- .../html_test__html_doc_files_rewrite-10.snap | 8 +- .../html_test__html_doc_files_rewrite-12.snap | 18 +-- .../html_test__html_doc_files_rewrite-14.snap | 8 +- .../html_test__html_doc_files_rewrite-15.snap | 8 +- .../html_test__html_doc_files_rewrite-16.snap | 8 +- .../html_test__html_doc_files_rewrite-17.snap | 8 +- .../html_test__html_doc_files_rewrite-2.snap | 104 ++++++++---------- .../html_test__html_doc_files_rewrite-3.snap | 8 +- .../html_test__html_doc_files_rewrite-5.snap | 8 +- .../html_test__html_doc_files_rewrite-6.snap | 8 +- .../html_test__html_doc_files_rewrite-7.snap | 10 +- .../html_test__html_doc_files_rewrite-8.snap | 8 +- .../html_test__html_doc_files_rewrite-9.snap | 8 +- .../html_test__html_doc_files_rewrite.snap | 2 +- tests/snapshots/html_test__module_doc.snap | 2 +- tests/snapshots/html_test__symbol_group.snap | 44 ++++---- 19 files changed, 139 insertions(+), 153 deletions(-) diff --git a/tests/snapshots/html_test__html_doc_files-3.snap b/tests/snapshots/html_test__html_doc_files-3.snap index 14958f25..d87d9db3 100644 --- a/tests/snapshots/html_test__html_doc_files-3.snap +++ b/tests/snapshots/html_test__html_doc_files-3.snap @@ -52,12 +52,12 @@ expression: "files.get(\"./~/Bar.html\").unwrap()"
-

Usage

import { Bar } from ".";
+    
Usage
import { Bar } from ".";
diff --git a/tests/snapshots/html_test__html_doc_files-5.snap b/tests/snapshots/html_test__html_doc_files-5.snap index 116ed1f3..0e48a64d 100644 --- a/tests/snapshots/html_test__html_doc_files-5.snap +++ b/tests/snapshots/html_test__html_doc_files-5.snap @@ -45,12 +45,12 @@ expression: "files.get(\"./~/Foo.html\").unwrap()"
class Foo - 
using time = new FakeTime();
+          
using time = new FakeTime();
@@ -58,12 +58,12 @@ expression: "files.get(\"./~/Foo.html\").unwrap()"
-

Usage

import { Foo } from ".";
+    
Usage
import { Foo } from ".";
diff --git a/tests/snapshots/html_test__html_doc_files-7.snap b/tests/snapshots/html_test__html_doc_files-7.snap index 33950e7d..f06eacdd 100644 --- a/tests/snapshots/html_test__html_doc_files-7.snap +++ b/tests/snapshots/html_test__html_doc_files-7.snap @@ -50,12 +50,12 @@ expression: "files.get(\"./~/Foobar.html\").unwrap()"
-

Usage

import Foobar from ".";
+    
Usage
import Foobar from ".";
diff --git a/tests/snapshots/html_test__html_doc_files_rewrite-10.snap b/tests/snapshots/html_test__html_doc_files_rewrite-10.snap index 55fccf94..072c4894 100644 --- a/tests/snapshots/html_test__html_doc_files_rewrite-10.snap +++ b/tests/snapshots/html_test__html_doc_files_rewrite-10.snap @@ -81,12 +81,12 @@ Type
-

Usage

import { Foo } from ".";
+    
Usage
import { Foo } from ".";
 

 
// This code block is ignored when getting the title of this doc
+          
// This code block is ignored when getting the title of this doc
 const foobar = new Foobar();

Foobar docs

@@ -80,7 +80,7 @@ const foobar = new Foobar(); -See
@@ -88,12 +88,12 @@ See