From fc30e75b54dae158d48750265d412bd67a6bf9f4 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Mon, 2 Jul 2018 15:31:39 +0300 Subject: [PATCH 01/12] initial rework of syntest to be usable internally --- examples/syntest.rs | 218 ++++------------------------------------- src/easy.rs | 232 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 248 insertions(+), 202 deletions(-) diff --git a/examples/syntest.rs b/examples/syntest.rs index 0b09b475..6c56ecce 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -15,16 +15,14 @@ extern crate regex; extern crate getopts; //extern crate onig; -use syntect::parsing::{SyntaxSet, ParseState, ScopeStack, Scope}; -use syntect::highlighting::ScopeSelectors; -use syntect::easy::{ScopeRegionIterator}; +use syntect::parsing::{SyntaxSet}; +use syntect::easy::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions}; use std::path::Path; +use std::io::prelude::*; use std::io::{BufRead, BufReader}; use std::fs::File; -use std::cmp::{min, max}; use std::time::Instant; -use std::str::FromStr; use getopts::Options; use regex::Regex; @@ -36,12 +34,6 @@ pub enum SyntaxTestHeaderError { SyntaxDefinitionNotFound, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SyntaxTestFileResult { - FailedAssertions(usize, usize), - Success(usize), -} - lazy_static! { pub static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm) ^(?P\s*\S+) @@ -49,113 +41,19 @@ lazy_static! { "(?P[^"]+)" \s*(?P\S+)?$ "#).unwrap(); - pub static ref SYNTAX_TEST_ASSERTION_PATTERN: Regex = Regex::new(r#"(?xm) - \s*(?: - (?P<-)|(?P\^+) - )(.*)$"#).unwrap(); -} - -#[derive(Clone, Copy)] -struct OutputOptions { - time: bool, - debug: bool, - summary: bool, -} - -#[derive(Debug)] -struct AssertionRange<'a> { - begin_char: usize, - end_char: usize, - scope_selector_text: &'a str, - is_pure_assertion_line: bool, -} - -#[derive(Debug)] -struct ScopedText { - scope: Vec, - char_start: usize, - text_len: usize, -} - -#[derive(Debug)] -struct RangeTestResult { - column_begin: usize, - column_end: usize, - success: bool, -} - -fn get_line_assertion_details<'a>(testtoken_start: &str, testtoken_end: Option<&str>, line: &'a str) -> Option> { - // if the test start token specified in the test file's header is on the line - if let Some(index) = line.find(testtoken_start) { - let (before_token_start, token_and_rest_of_line) = line.split_at(index); - - if let Some(captures) = SYNTAX_TEST_ASSERTION_PATTERN.captures(&token_and_rest_of_line[testtoken_start.len()..]) { - let mut sst = captures.get(3).unwrap().as_str(); // get the scope selector text - let mut only_whitespace_after_token_end = true; - - if let Some(token) = testtoken_end { // if there is an end token defined in the test file header - if let Some(end_token_pos) = sst.find(token) { // and there is an end token in the line - let (ss, after_token_end) = sst.split_at(end_token_pos); // the scope selector text ends at the end token - sst = &ss; - only_whitespace_after_token_end = after_token_end.trim_right().is_empty(); - } - } - return Some(AssertionRange { - begin_char: index + if captures.get(2).is_some() { testtoken_start.len() + captures.get(2).unwrap().start() } else { 0 }, - end_char: index + if captures.get(2).is_some() { testtoken_start.len() + captures.get(2).unwrap().end() } else { 1 }, - scope_selector_text: sst, - is_pure_assertion_line: before_token_start.trim_left().is_empty() && only_whitespace_after_token_end, // if only whitespace surrounds the test tokens on the line, then it is a pure assertion line - }); - } - } - None -} - -fn process_assertions(assertion: &AssertionRange, test_against_line_scopes: &Vec) -> Vec { - // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" - // and they are sometimes in the syntax test as ^^^-comment, for example - let selector = ScopeSelectors::from_str(&format!(" {}", &assertion.scope_selector_text)).unwrap(); - // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached - let mut results = Vec::new(); - for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) { - let match_value = selector.does_match(scoped_text.scope.as_slice()); - let result = RangeTestResult { - column_begin: max(scoped_text.char_start, assertion.begin_char), - column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char), - success: match_value.is_some() - }; - results.push(result); - } - // don't ignore assertions after the newline, they should be treated as though they are asserting against the newline - let last = test_against_line_scopes.last().unwrap(); - if last.char_start + last.text_len < assertion.end_char { - let match_value = selector.does_match(last.scope.as_slice()); - let result = RangeTestResult { - column_begin: max(last.char_start + last.text_len, assertion.begin_char), - column_end: assertion.end_char, - success: match_value.is_some() - }; - results.push(result); - } - results } -/// If `parse_test_lines` is `false` then lines that only contain assertions are not parsed -fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: OutputOptions) -> Result { - use syntect::util::debug_print_ops; +fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> Result { let f = File::open(path).unwrap(); let mut reader = BufReader::new(f); - let mut line = String::new(); + let mut header_line = String::new(); // read the first line from the file - if we have reached EOF already, it's an invalid file - if reader.read_line(&mut line).unwrap() == 0 { + if reader.read_line(&mut header_line).unwrap() == 0 { return Err(SyntaxTestHeaderError::MalformedHeader); } - line = line.replace("\r", &""); - // parse the syntax test header in the first line of the file - let header_line = line.clone(); let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line); let captures = search_result.ok_or(SyntaxTestHeaderError::MalformedHeader)?; @@ -165,101 +63,19 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: Outp // find the relevant syntax definition to parse the file with - case is important! if !out_opts.summary { - println!("The test file references syntax definition file: {}", syntax_file); + println!("The test file references syntax definition file: {}", syntax_file); //" and the start test token is {} and the end token is {:?}", testtoken_start, testtoken_end); } let syntax = ss.find_syntax_by_path(syntax_file).ok_or(SyntaxTestHeaderError::SyntaxDefinitionNotFound)?; - // iterate over the lines of the file, testing them - let mut state = ParseState::new(syntax); - let mut stack = ScopeStack::new(); + let mut contents = String::new(); + contents.push_str(&header_line); + reader.read_to_string(&mut contents).expect("Unable to read file"); + contents = contents.replace("\r", &""); - let mut current_line_number = 1; - let mut test_against_line_number = 1; - let mut scopes_on_line_being_tested = Vec::new(); - let mut previous_non_assertion_line = line.to_string(); - - let mut assertion_failures: usize = 0; - let mut total_assertions: usize = 0; - - loop { // over lines of file, starting with the header line - let mut line_only_has_assertion = false; - let mut line_has_assertion = false; - if let Some(assertion) = get_line_assertion_details(testtoken_start, testtoken_end, &line) { - let result = process_assertions(&assertion, &scopes_on_line_being_tested); - total_assertions += &assertion.end_char - &assertion.begin_char; - for failure in result.iter().filter(|r|!r.success) { - let length = failure.column_end - failure.column_begin; - let text: String = previous_non_assertion_line.chars().skip(failure.column_begin).take(length).collect(); - if !out_opts.summary { - println!(" Assertion selector {:?} \ - from line {:?} failed against line {:?}, column range {:?}-{:?} \ - (with text {:?}) \ - has scope {:?}", - assertion.scope_selector_text.trim(), - current_line_number, test_against_line_number, failure.column_begin, failure.column_end, - text, - scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope - ); - } - assertion_failures += failure.column_end - failure.column_begin; - } - line_only_has_assertion = assertion.is_pure_assertion_line; - line_has_assertion = true; - } - if !line_only_has_assertion || parse_test_lines { - if !line_has_assertion { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against - scopes_on_line_being_tested.clear(); - test_against_line_number = current_line_number; - previous_non_assertion_line = line.to_string(); - } - if out_opts.debug && !line_only_has_assertion { - println!("-- debugging line {} -- scope stack: {:?}", current_line_number, stack); - } - let ops = state.parse_line(&line); - if out_opts.debug && !line_only_has_assertion { - if ops.is_empty() && !line.is_empty() { - println!("no operations for this line..."); - } else { - debug_print_ops(&line, &ops); - } - } - let mut col: usize = 0; - for (s, op) in ScopeRegionIterator::new(&ops, &line) { - stack.apply(op); - if s.is_empty() { // in this case we don't care about blank tokens - continue; - } - if !line_has_assertion { - // if the line has no assertions on it, remember the scopes on the line so we can test against them later - let len = s.chars().count(); - scopes_on_line_being_tested.push( - ScopedText { - char_start: col, - text_len: len, - scope: stack.as_slice().to_vec() - } - ); - // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired - col += len; - } - } - } - - line.clear(); - current_line_number += 1; - if reader.read_line(&mut line).unwrap() == 0 { - break; - } - line = line.replace("\r", &""); - } - let res = if assertion_failures > 0 { - Ok(SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions)) - } else { - Ok(SyntaxTestFileResult::Success(total_assertions)) - }; + let res = process_syntax_test_assertions(&syntax, &contents, testtoken_start, testtoken_end, &out_opts); if out_opts.summary { - if let Ok(SyntaxTestFileResult::FailedAssertions(failures, _)) = res { + if let SyntaxTestFileResult::FailedAssertions(failures, _) = res { // Don't print total assertion count so that diffs don't pick up new succeeding tests println!("FAILED {}: {}", path.display(), failures); } @@ -267,7 +83,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: Outp println!("{:?}", res); } - res + Ok(res) } fn main() { @@ -309,7 +125,7 @@ fn main() { ss.link_syntaxes(); } - let out_opts = OutputOptions { + let out_opts = SyntaxTestOutputOptions { debug: matches.opt_present("debug"), time: matches.opt_present("time"), summary: matches.opt_present("summary"), @@ -322,7 +138,7 @@ fn main() { } -fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 { +fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions) -> i32 { let mut exit_code: i32 = 0; // exit with code 0 by default, if all tests pass let walker = WalkDir::new(path).into_iter(); @@ -341,7 +157,7 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 { println!("Testing file {}", path.display()); } let start = Instant::now(); - let result = test_file(&ss, path, true, out_opts); + let result = test_file(&ss, path, out_opts); let elapsed = start.elapsed(); if out_opts.time { let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64; diff --git a/src/easy.rs b/src/easy.rs index 027f5a9d..58acc8fb 100644 --- a/src/easy.rs +++ b/src/easy.rs @@ -2,7 +2,7 @@ //! files without caring about intermediate semantic representation //! and caching. -use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp}; +use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp, Scope}; use highlighting::{Highlighter, HighlightState, HighlightIterator, Theme, Style}; use std::io::{self, BufReader}; use std::fs::File; @@ -174,6 +174,236 @@ impl<'a> Iterator for ScopeRegionIterator<'a> { } } +#[derive(Clone, Copy)] +pub struct SyntaxTestOutputOptions { + pub time: bool, + pub debug: bool, + pub summary: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SyntaxTestFileResult { + FailedAssertions(usize, usize), + Success(usize), +} + +pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult { + use std::collections::VecDeque; + use highlighting::ScopeSelectors; + + #[derive(Debug)] + struct SyntaxTestAssertionRange { + test_line_offset: usize, + line_number: usize, + begin_char: usize, + end_char: usize, + scope_selector: ScopeSelectors, + scope_selector_text: String, + } + + fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque { + use std::str::FromStr; + + let mut assertions = VecDeque::new(); + let mut test_line_offset = 0; + //let mut test_line_len = 0; + let mut line_number = 0; + let mut offset = 0; + //let mut remainder = None; + for line in text.lines() { + line_number += 1; + let mut line_has_assertions = false; + + // if the test start token specified is on the line + if let Some(index) = line.find(token_start) { + let token_and_rest_of_line = line.split_at(index).1; + + let rest_of_line = &token_and_rest_of_line[token_start.len()..]; + if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) { + let mut assertion_range = 0; + while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') { + assertion_range += 1; + } + let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range }; + + let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text + + if let Some(token) = token_end { // if there is an end token defined in the test file header + if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line + selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token + } + } + + let assertion = SyntaxTestAssertionRange { + test_line_offset: test_line_offset, + line_number: line_number, + begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 }, + end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 }, + + // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" + // and they are sometimes in the syntax test as ^^^-comment, for example + scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)), + scope_selector_text: selector_text, + }; + /*if assertion.end_char > test_line_len { + remainder = Some(SyntaxTestAssertionRange { + test_line_offset: test_line_offset + test_line_len, + line_number: line_number, + begin_char: assertion.begin_char - test_line_len, + end_char: assertion.end_char - test_line_len, + scope_selector: assertion.scope_selector.clone(), + scope_selector_text: assertion.scope_selector_text.clone(), + }); + }*/ + assertions.push_back(assertion); + + line_has_assertions = true; + } + } + if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text + test_line_offset = offset; + //test_line_len = line.len() + 1; + } + offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405 + } + assertions + } + + #[derive(Debug)] + struct ScopedText { + scope: Vec, + char_start: usize, + text_len: usize, + } + + #[derive(Debug)] + struct RangeTestResult { + column_begin: usize, + column_end: usize, + success: bool, + actual_scope: String, + } + + fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec) -> Vec { + use std::cmp::{min, max}; + // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached + let mut results = Vec::new(); + for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) { + let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice()); + let result = RangeTestResult { + column_begin: max(scoped_text.char_start, assertion.begin_char), + column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char), + success: match_value.is_some(), + actual_scope: format!("{:?}", scoped_text.scope.as_slice()), + }; + results.push(result); + } + results + } + + let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); + //println!("{:?}", assertions); + use util::debug_print_ops; + + // iterate over the lines of the file, testing them + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + + let mut offset = 0; + let mut scopes_on_line_being_tested = Vec::new(); + let mut line_number = 0; + let mut relevant_assertions = Vec::new(); + + let mut assertion_failures: usize = 0; + let mut total_assertions: usize = 0; + + for line_without_char in text.lines() { + let line = &(line_without_char.to_owned() + "\n"); + line_number += 1; + + let eol_offset = offset + line.len(); + + // parse the line + let ops = state.parse_line(&line); + // find assertions that relate to the current line + relevant_assertions.clear(); + while let Some(assertion) = assertions.pop_front() { + let pos = assertion.test_line_offset + assertion.begin_char; + if pos >= offset && pos < eol_offset { + relevant_assertions.push(assertion); + } else { + assertions.push_front(assertion); + break; + } + } + if !relevant_assertions.is_empty() { + scopes_on_line_being_tested.clear(); + if out_opts.debug { + println!("-- debugging line {} -- scope stack: {:?}", line_number, stack); + if ops.is_empty() && !line.is_empty() { + println!("no operations for this line..."); + } else { + debug_print_ops(&line, &ops); + } + } + } + + { + let mut col: usize = 0; + for (s, op) in ScopeRegionIterator::new(&ops, &line) { + stack.apply(op); + if s.is_empty() { // in this case we don't care about blank tokens + continue; + } + if !relevant_assertions.is_empty() { + let len = s.chars().count(); + scopes_on_line_being_tested.push( + ScopedText { + char_start: col, + text_len: len, + scope: stack.as_slice().to_vec() + } + ); + col += len; + } + } + } + + for assertion in &relevant_assertions { + let results = process_assertions(&assertion, &scopes_on_line_being_tested); + + for result in results { + let length = result.column_end - result.column_begin; + total_assertions += length; + if !result.success { + assertion_failures += length; + let text: String = line.chars().skip(result.column_begin).take(length).collect(); + if !out_opts.summary { + println!(" Assertion selector {:?} \ + from line {:?} failed against line {:?}, column range {:?}-{:?} \ + (with text {:?}) \ + has scope {:?}", + &assertion.scope_selector_text.trim(), + &assertion.line_number, line_number, result.column_begin, result.column_end, + text, + result.actual_scope, + ); + } + } + } + } + + offset = eol_offset; + } + + let res = if assertion_failures > 0 { + SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions) + } else { + SyntaxTestFileResult::Success(total_assertions) + }; + res +} + #[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] #[cfg(test)] mod tests { From fdaf8bd9e384deadf3b471d04e747301cb486dc4 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Mon, 2 Jul 2018 19:39:41 +0300 Subject: [PATCH 02/12] put syntax test functionality in it's own module --- examples/syntest.rs | 2 +- src/easy.rs | 232 +------------------------------------- src/lib.rs | 2 + src/syntax_tests.rs | 263 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 267 insertions(+), 232 deletions(-) create mode 100644 src/syntax_tests.rs diff --git a/examples/syntest.rs b/examples/syntest.rs index 6c56ecce..a5b9b84b 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -16,7 +16,7 @@ extern crate getopts; //extern crate onig; use syntect::parsing::{SyntaxSet}; -use syntect::easy::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions}; +use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions}; use std::path::Path; use std::io::prelude::*; diff --git a/src/easy.rs b/src/easy.rs index 58acc8fb..027f5a9d 100644 --- a/src/easy.rs +++ b/src/easy.rs @@ -2,7 +2,7 @@ //! files without caring about intermediate semantic representation //! and caching. -use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp, Scope}; +use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp}; use highlighting::{Highlighter, HighlightState, HighlightIterator, Theme, Style}; use std::io::{self, BufReader}; use std::fs::File; @@ -174,236 +174,6 @@ impl<'a> Iterator for ScopeRegionIterator<'a> { } } -#[derive(Clone, Copy)] -pub struct SyntaxTestOutputOptions { - pub time: bool, - pub debug: bool, - pub summary: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SyntaxTestFileResult { - FailedAssertions(usize, usize), - Success(usize), -} - -pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult { - use std::collections::VecDeque; - use highlighting::ScopeSelectors; - - #[derive(Debug)] - struct SyntaxTestAssertionRange { - test_line_offset: usize, - line_number: usize, - begin_char: usize, - end_char: usize, - scope_selector: ScopeSelectors, - scope_selector_text: String, - } - - fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque { - use std::str::FromStr; - - let mut assertions = VecDeque::new(); - let mut test_line_offset = 0; - //let mut test_line_len = 0; - let mut line_number = 0; - let mut offset = 0; - //let mut remainder = None; - for line in text.lines() { - line_number += 1; - let mut line_has_assertions = false; - - // if the test start token specified is on the line - if let Some(index) = line.find(token_start) { - let token_and_rest_of_line = line.split_at(index).1; - - let rest_of_line = &token_and_rest_of_line[token_start.len()..]; - if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) { - let mut assertion_range = 0; - while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') { - assertion_range += 1; - } - let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range }; - - let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text - - if let Some(token) = token_end { // if there is an end token defined in the test file header - if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line - selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token - } - } - - let assertion = SyntaxTestAssertionRange { - test_line_offset: test_line_offset, - line_number: line_number, - begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 }, - end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 }, - - // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" - // and they are sometimes in the syntax test as ^^^-comment, for example - scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)), - scope_selector_text: selector_text, - }; - /*if assertion.end_char > test_line_len { - remainder = Some(SyntaxTestAssertionRange { - test_line_offset: test_line_offset + test_line_len, - line_number: line_number, - begin_char: assertion.begin_char - test_line_len, - end_char: assertion.end_char - test_line_len, - scope_selector: assertion.scope_selector.clone(), - scope_selector_text: assertion.scope_selector_text.clone(), - }); - }*/ - assertions.push_back(assertion); - - line_has_assertions = true; - } - } - if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text - test_line_offset = offset; - //test_line_len = line.len() + 1; - } - offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405 - } - assertions - } - - #[derive(Debug)] - struct ScopedText { - scope: Vec, - char_start: usize, - text_len: usize, - } - - #[derive(Debug)] - struct RangeTestResult { - column_begin: usize, - column_end: usize, - success: bool, - actual_scope: String, - } - - fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec) -> Vec { - use std::cmp::{min, max}; - // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached - let mut results = Vec::new(); - for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) { - let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice()); - let result = RangeTestResult { - column_begin: max(scoped_text.char_start, assertion.begin_char), - column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char), - success: match_value.is_some(), - actual_scope: format!("{:?}", scoped_text.scope.as_slice()), - }; - results.push(result); - } - results - } - - let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); - //println!("{:?}", assertions); - use util::debug_print_ops; - - // iterate over the lines of the file, testing them - let mut state = ParseState::new(syntax); - let mut stack = ScopeStack::new(); - - let mut offset = 0; - let mut scopes_on_line_being_tested = Vec::new(); - let mut line_number = 0; - let mut relevant_assertions = Vec::new(); - - let mut assertion_failures: usize = 0; - let mut total_assertions: usize = 0; - - for line_without_char in text.lines() { - let line = &(line_without_char.to_owned() + "\n"); - line_number += 1; - - let eol_offset = offset + line.len(); - - // parse the line - let ops = state.parse_line(&line); - // find assertions that relate to the current line - relevant_assertions.clear(); - while let Some(assertion) = assertions.pop_front() { - let pos = assertion.test_line_offset + assertion.begin_char; - if pos >= offset && pos < eol_offset { - relevant_assertions.push(assertion); - } else { - assertions.push_front(assertion); - break; - } - } - if !relevant_assertions.is_empty() { - scopes_on_line_being_tested.clear(); - if out_opts.debug { - println!("-- debugging line {} -- scope stack: {:?}", line_number, stack); - if ops.is_empty() && !line.is_empty() { - println!("no operations for this line..."); - } else { - debug_print_ops(&line, &ops); - } - } - } - - { - let mut col: usize = 0; - for (s, op) in ScopeRegionIterator::new(&ops, &line) { - stack.apply(op); - if s.is_empty() { // in this case we don't care about blank tokens - continue; - } - if !relevant_assertions.is_empty() { - let len = s.chars().count(); - scopes_on_line_being_tested.push( - ScopedText { - char_start: col, - text_len: len, - scope: stack.as_slice().to_vec() - } - ); - col += len; - } - } - } - - for assertion in &relevant_assertions { - let results = process_assertions(&assertion, &scopes_on_line_being_tested); - - for result in results { - let length = result.column_end - result.column_begin; - total_assertions += length; - if !result.success { - assertion_failures += length; - let text: String = line.chars().skip(result.column_begin).take(length).collect(); - if !out_opts.summary { - println!(" Assertion selector {:?} \ - from line {:?} failed against line {:?}, column range {:?}-{:?} \ - (with text {:?}) \ - has scope {:?}", - &assertion.scope_selector_text.trim(), - &assertion.line_number, line_number, result.column_begin, result.column_end, - text, - result.actual_scope, - ); - } - } - } - } - - offset = eol_offset; - } - - let res = if assertion_failures > 0 { - SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions) - } else { - SyntaxTestFileResult::Success(total_assertions) - }; - res -} - #[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] #[cfg(test)] mod tests { diff --git a/src/lib.rs b/src/lib.rs index c2e9860d..3b8c53a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,8 @@ pub mod util; pub mod dumps; #[cfg(feature = "parsing")] pub mod easy; +#[cfg(feature = "parsing")] +pub mod syntax_tests; #[cfg(feature = "html")] pub mod html; #[cfg(feature = "html")] diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs new file mode 100644 index 00000000..f8122387 --- /dev/null +++ b/src/syntax_tests.rs @@ -0,0 +1,263 @@ +//! API for running syntax tests. +//! See http://www.sublimetext.com/docs/3/syntax.html#testing + +use parsing::{ScopeStack, ParseState, SyntaxDefinition, Scope}; +//use std::io::Write; +use std::str::FromStr; +use util::debug_print_ops; +use easy::{ScopeRegionIterator}; + +#[derive(Clone, Copy)] +pub struct SyntaxTestOutputOptions { + pub time: bool, + pub debug: bool, + pub summary: bool, + //pub output: &'a Write, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SyntaxTestFileResult { + FailedAssertions(usize, usize), + Success(usize), +} + +use std::collections::VecDeque; +use highlighting::ScopeSelectors; + +#[derive(Debug)] +struct SyntaxTestAssertionRange { + test_line_offset: usize, + line_number: usize, + begin_char: usize, + end_char: usize, + scope_selector: ScopeSelectors, + scope_selector_text: String, +} + +fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque { + let mut assertions = VecDeque::new(); + let mut test_line_offset = 0; + //let mut test_line_len = 0; + let mut line_number = 0; + let mut offset = 0; + //let mut remainder = None; + for line in text.lines() { + line_number += 1; + let mut line_has_assertions = false; + + // if the test start token specified is on the line + if let Some(index) = line.find(token_start) { + let token_and_rest_of_line = line.split_at(index).1; + + let rest_of_line = &token_and_rest_of_line[token_start.len()..]; + if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) { + let mut assertion_range = 0; + while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') { + assertion_range += 1; + } + let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range }; + + let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text + + if let Some(token) = token_end { // if there is an end token defined in the test file header + if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line + selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token + } + } + + let assertion = SyntaxTestAssertionRange { + test_line_offset: test_line_offset, + line_number: line_number, + begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 }, + end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 }, + + // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" + // and they are sometimes in the syntax test as ^^^-comment, for example + scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)), + scope_selector_text: selector_text, + }; + /*if assertion.end_char > test_line_len { + remainder = Some(SyntaxTestAssertionRange { + test_line_offset: test_line_offset + test_line_len, + line_number: line_number, + begin_char: assertion.begin_char - test_line_len, + end_char: assertion.end_char - test_line_len, + scope_selector: assertion.scope_selector.clone(), + scope_selector_text: assertion.scope_selector_text.clone(), + }); + }*/ + assertions.push_back(assertion); + + line_has_assertions = true; + } + } + if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text + test_line_offset = offset; + //test_line_len = line.len() + 1; + } + offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405 + } + assertions +} + +/// Process the syntax test assertions in the given text, for the given syntax definition, using the test token(s) specified. +/// `text` is the code containing syntax test assertions to be parsed and checked. +/// `testtoken_start` is the token (normally a comment in the given syntax) that represents that assertions could be on the line. +/// `testtoken_end` is an optional token that will be stripped from the line when retrieving the scope selector. Useful for syntaxes when the start token represents a block comment, to make the tests easier to construct. +pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult { + #[derive(Debug)] + struct ScopedText { + scope: Vec, + char_start: usize, + text_len: usize, + } + + #[derive(Debug)] + struct RangeTestResult { + column_begin: usize, + column_end: usize, + success: bool, + actual_scope: String, + } + + fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec) -> Vec { + use std::cmp::{min, max}; + // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached + let mut results = Vec::new(); + for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) { + let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice()); + let result = RangeTestResult { + column_begin: max(scoped_text.char_start, assertion.begin_char), + column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char), + success: match_value.is_some(), + actual_scope: format!("{:?}", scoped_text.scope.as_slice()), + }; + results.push(result); + } + results + } + + let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); + //println!("{:?}", assertions); + + // iterate over the lines of the file, testing them + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + + let mut offset = 0; + let mut scopes_on_line_being_tested = Vec::new(); + let mut line_number = 0; + let mut relevant_assertions = Vec::new(); + + let mut assertion_failures: usize = 0; + let mut total_assertions: usize = 0; + + for line_without_char in text.lines() { + let line = &(line_without_char.to_owned() + "\n"); + line_number += 1; + + let eol_offset = offset + line.len(); + + // parse the line + let ops = state.parse_line(&line); + // find assertions that relate to the current line + relevant_assertions.clear(); + while let Some(assertion) = assertions.pop_front() { + let pos = assertion.test_line_offset + assertion.begin_char; + if pos >= offset && pos < eol_offset { + relevant_assertions.push(assertion); + } else { + assertions.push_front(assertion); + break; + } + } + if !relevant_assertions.is_empty() { + scopes_on_line_being_tested.clear(); + if out_opts.debug { + println!("-- debugging line {} -- scope stack: {:?}", line_number, stack); + if ops.is_empty() && !line.is_empty() { + println!("no operations for this line..."); + } else { + debug_print_ops(&line, &ops); + } + } + } + + { + let mut col: usize = 0; + for (s, op) in ScopeRegionIterator::new(&ops, &line) { + stack.apply(op); + if s.is_empty() { // in this case we don't care about blank tokens + continue; + } + if !relevant_assertions.is_empty() { + let len = s.chars().count(); + scopes_on_line_being_tested.push( + ScopedText { + char_start: col, + text_len: len, + scope: stack.as_slice().to_vec() + } + ); + col += len; + } + } + } + + for assertion in &relevant_assertions { + let results = process_assertions(&assertion, &scopes_on_line_being_tested); + + for result in results { + let length = result.column_end - result.column_begin; + total_assertions += length; + if !result.success { + assertion_failures += length; + let text: String = line.chars().skip(result.column_begin).take(length).collect(); + if !out_opts.summary { + println!(" Assertion selector {:?} \ + from line {:?} failed against line {:?}, column range {:?}-{:?} \ + (with text {:?}) \ + has scope {:?}", + &assertion.scope_selector_text.trim(), + &assertion.line_number, line_number, result.column_begin, result.column_end, + text, + result.actual_scope, + ); + } + } + } + } + + offset = eol_offset; + } + + let res = if assertion_failures > 0 { + SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions) + } else { + SyntaxTestFileResult::Success(total_assertions) + }; + res +} + +// #[cfg(test)] +// mod tests { + #[test] + fn can_find_test_assertions() { + let result = get_syntax_test_assertions(&"//", None, + " + hello world + // <- assertion1 + // ^^ assertion2 + + foobar + // ^ - assertion3 + "); + + assert_eq!(result.len(), 3); + assert_eq!(result[0].line_number, 3); + assert_eq!(result[1].line_number, 4); + assert_eq!(result[2].line_number, 7); + assert_eq!(result[0].test_line_offset, result[1].test_line_offset); + assert!(result[2].test_line_offset > result[0].test_line_offset); + } +// } From 2dee5fcd5177861eb7b6f02479e6cf0553333607 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Tue, 3 Jul 2018 09:30:17 +0300 Subject: [PATCH 03/12] add more tests for finding syntax test assertions --- src/syntax_tests.rs | 82 +++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index f8122387..5f4c2a82 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -44,7 +44,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: for line in text.lines() { line_number += 1; let mut line_has_assertions = false; - + // if the test start token specified is on the line if let Some(index) = line.find(token_start) { let token_and_rest_of_line = line.split_at(index).1; @@ -87,7 +87,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: }); }*/ assertions.push_back(assertion); - + line_has_assertions = true; } } @@ -111,7 +111,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes char_start: usize, text_len: usize, } - + #[derive(Debug)] struct RangeTestResult { column_begin: usize, @@ -119,7 +119,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes success: bool, actual_scope: String, } - + fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec) -> Vec { use std::cmp::{min, max}; // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached @@ -136,10 +136,10 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } results } - + let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); //println!("{:?}", assertions); - + // iterate over the lines of the file, testing them let mut state = ParseState::new(syntax); let mut stack = ScopeStack::new(); @@ -148,16 +148,16 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes let mut scopes_on_line_being_tested = Vec::new(); let mut line_number = 0; let mut relevant_assertions = Vec::new(); - + let mut assertion_failures: usize = 0; let mut total_assertions: usize = 0; for line_without_char in text.lines() { let line = &(line_without_char.to_owned() + "\n"); line_number += 1; - + let eol_offset = offset + line.len(); - + // parse the line let ops = state.parse_line(&line); // find assertions that relate to the current line @@ -182,7 +182,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } } } - + { let mut col: usize = 0; for (s, op) in ScopeRegionIterator::new(&ops, &line) { @@ -206,7 +206,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes for assertion in &relevant_assertions { let results = process_assertions(&assertion, &scopes_on_line_being_tested); - + for result in results { let length = result.column_end - result.column_begin; total_assertions += length; @@ -227,7 +227,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } } } - + offset = eol_offset; } @@ -243,21 +243,59 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes // mod tests { #[test] fn can_find_test_assertions() { - let result = get_syntax_test_assertions(&"//", None, - " - hello world - // <- assertion1 - // ^^ assertion2 - - foobar - // ^ - assertion3 - "); + let text = "\ + hello world\n\ + // <- assertion1\n\ + // ^^ assertion2\n\ + \n\ + foobar\n\ + // ^ - assertion3\n\ + "; + let result = get_syntax_test_assertions(&"//", None, &text); + + assert_eq!(result.len(), 3); + assert_eq!(result[0].line_number, 2); + assert_eq!(result[1].line_number, 3); + assert_eq!(result[2].line_number, 6); + assert_eq!(result[0].test_line_offset, result[1].test_line_offset); + assert_eq!(result[2].test_line_offset, text.find("foobar").unwrap()); + assert_eq!(result[0].scope_selector_text, " assertion1"); + assert_eq!(result[1].scope_selector_text, " assertion2"); + assert_eq!(result[2].scope_selector_text, " - assertion3"); + assert_eq!(result[0].begin_char, 0); + assert_eq!(result[0].end_char, 1); + assert_eq!(result[1].begin_char, 3); + assert_eq!(result[1].end_char, 5); + assert_eq!(result[2].begin_char, 6); + assert_eq!(result[2].end_char, 7); + } + + #[test] + fn can_find_test_assertions_with_end_tokens() { + let text = " +hello world + + +"; + let result = get_syntax_test_assertions(&""), &text); assert_eq!(result.len(), 3); assert_eq!(result[0].line_number, 3); assert_eq!(result[1].line_number, 4); assert_eq!(result[2].line_number, 7); assert_eq!(result[0].test_line_offset, result[1].test_line_offset); - assert!(result[2].test_line_offset > result[0].test_line_offset); + assert_eq!(result[2].test_line_offset, text.find("foobar").unwrap()); + assert_eq!(result[0].scope_selector_text, " assertion1 "); + assert_eq!(result[1].scope_selector_text, "assertion2"); + assert_eq!(result[2].scope_selector_text, " - assertion3 "); + assert_eq!(result[0].begin_char, 1); + assert_eq!(result[0].end_char, 2); + assert_eq!(result[1].begin_char, 6); + assert_eq!(result[1].end_char, 8); + assert_eq!(result[2].begin_char, 5); + assert_eq!(result[2].end_char, 6); } // } From 91af5e9aa2422f3763b455aa992f3050b46e5b9f Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Tue, 3 Jul 2018 15:04:20 +0300 Subject: [PATCH 04/12] add tests to syntest to prove it works with assertions than span lines --- src/syntax_tests.rs | 77 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 5f4c2a82..9b47bdb2 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -37,10 +37,10 @@ struct SyntaxTestAssertionRange { fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque { let mut assertions = VecDeque::new(); let mut test_line_offset = 0; - //let mut test_line_len = 0; + let mut test_line_len = 0; let mut line_number = 0; let mut offset = 0; - //let mut remainder = None; + for line in text.lines() { line_number += 1; let mut line_has_assertions = false; @@ -65,7 +65,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: } } - let assertion = SyntaxTestAssertionRange { + let mut assertion = SyntaxTestAssertionRange { test_line_offset: test_line_offset, line_number: line_number, begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 }, @@ -76,24 +76,30 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)), scope_selector_text: selector_text, }; - /*if assertion.end_char > test_line_len { - remainder = Some(SyntaxTestAssertionRange { + // if the assertion spans over the line being tested + if assertion.end_char > test_line_len { + // calculate where on the next line the assertions will occur + let remainder = SyntaxTestAssertionRange { test_line_offset: test_line_offset + test_line_len, line_number: line_number, - begin_char: assertion.begin_char - test_line_len, + begin_char: 0, end_char: assertion.end_char - test_line_len, scope_selector: assertion.scope_selector.clone(), scope_selector_text: assertion.scope_selector_text.clone(), - }); - }*/ - assertions.push_back(assertion); + }; + assertion.end_char = test_line_len; + assertions.push_back(assertion); + assertions.push_back(remainder); + } else { + assertions.push_back(assertion); + } line_has_assertions = true; } } if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text test_line_offset = offset; - //test_line_len = line.len() + 1; + test_line_len = line.len() + 1; } offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405 } @@ -203,7 +209,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } } } - + for assertion in &relevant_assertions { let results = process_assertions(&assertion, &scopes_on_line_being_tested); @@ -230,7 +236,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes offset = eol_offset; } - + let res = if assertion_failures > 0 { SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions) } else { @@ -281,7 +287,7 @@ foobar "; let result = get_syntax_test_assertions(&""), &text); - + assert_eq!(result.len(), 3); assert_eq!(result[0].line_number, 3); assert_eq!(result[1].line_number, 4); @@ -298,4 +304,49 @@ foobar assert_eq!(result[2].begin_char, 5); assert_eq!(result[2].end_char, 6); } + + #[test] + fn can_find_test_assertions_that_spans_lines() { + let text = " +hello world + +foobar + +"; + let result = get_syntax_test_assertions(&""), &text); + println!("{:?}", result); + + assert_eq!(result.len(), 6); + assert_eq!(result[0].line_number, 3); + assert_eq!(result[1].line_number, 3); + assert_eq!(result[2].line_number, 4); + assert_eq!(result[3].line_number, 4); + assert_eq!(result[4].line_number, 6); + assert_eq!(result[5].line_number, 6); + assert_eq!(result[0].scope_selector_text, " assertion1"); + assert_eq!(result[1].scope_selector_text, " assertion1"); + assert_eq!(result[2].scope_selector_text, " assertion2 "); + assert_eq!(result[3].scope_selector_text, " assertion2 "); + assert_eq!(result[4].scope_selector_text, " -assertion3"); + assert_eq!(result[5].scope_selector_text, " -assertion3"); + assert_eq!(result[0].begin_char, 6); + assert_eq!(result[0].end_char, 12); + assert_eq!(result[0].test_line_offset, 1); + assert_eq!(result[1].begin_char, 0); + assert_eq!(result[1].end_char, 3); + assert_eq!(result[1].test_line_offset, "\nhello world\n".len()); + + assert_eq!(result[2].begin_char, 8); + assert_eq!(result[2].end_char, 12); + assert_eq!(result[2].test_line_offset, 1); + assert_eq!(result[3].begin_char, 0); + assert_eq!(result[3].end_char, 3); + assert_eq!(result[3].test_line_offset, "\nhello world\n".len()); + + assert_eq!(result[4].begin_char, 5); + assert_eq!(result[4].end_char, 7); + assert_eq!(result[5].begin_char, 0); + assert_eq!(result[5].end_char, 1); + } // } From 67c0e05367f261663d3ebfa0ddfceb5213d5a205 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Tue, 3 Jul 2018 15:10:41 +0300 Subject: [PATCH 05/12] fix syntest seemingly skipping some files with crlf line endings --- examples/syntest.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/syntest.rs b/examples/syntest.rs index a5b9b84b..97224f43 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -52,6 +52,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> if reader.read_line(&mut header_line).unwrap() == 0 { return Err(SyntaxTestHeaderError::MalformedHeader); } + header_line = header_line.replace("\r", &""); // parse the syntax test header in the first line of the file let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line); @@ -165,6 +166,7 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions) } if exit_code != 2 { // leave exit code 2 if there was an error if let Err(_) = result { // set exit code 2 if there was an error + println!("{:?}", result); exit_code = 2; } else if let Ok(ok) = result { if let SyntaxTestFileResult::FailedAssertions(_, _) = ok { From f485533b23ad407db857bfabc99c8eab31c18b20 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Wed, 4 Jul 2018 14:12:20 +0300 Subject: [PATCH 06/12] add failfast mode to syntest, add more comments --- examples/syntest.rs | 8 ++++++++ src/syntax_tests.rs | 21 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/examples/syntest.rs b/examples/syntest.rs index 97224f43..4b9121d2 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -93,6 +93,7 @@ fn main() { opts.optflag("d", "debug", "Show parsing results for each test line"); opts.optflag("t", "time", "Time execution as a more broad-ranging benchmark"); opts.optflag("s", "summary", "Print only summary of test failures"); + opts.optflag("f", "failfast", "Stop at first failure"); let matches = match opts.parse(&args[1..]) { Ok(m) => { m } @@ -130,6 +131,7 @@ fn main() { debug: matches.opt_present("debug"), time: matches.opt_present("time"), summary: matches.opt_present("summary"), + failfast: matches.opt_present("failfast"), }; let exit_code = recursive_walk(&ss, &tests_path, out_opts); @@ -168,9 +170,15 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions) if let Err(_) = result { // set exit code 2 if there was an error println!("{:?}", result); exit_code = 2; + if out_opts.failfast { + break; + } } else if let Ok(ok) = result { if let SyntaxTestFileResult::FailedAssertions(_, _) = ok { exit_code = 1; // otherwise, if there were failures, exit with code 1 + if out_opts.failfast { + break; + } } } } diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 9b47bdb2..c29cb1f6 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -12,6 +12,7 @@ pub struct SyntaxTestOutputOptions { pub time: bool, pub debug: bool, pub summary: bool, + pub failfast: bool, //pub output: &'a Write, } @@ -107,10 +108,13 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: } /// Process the syntax test assertions in the given text, for the given syntax definition, using the test token(s) specified. +/// It works by finding all the syntax test assertions, then parsing the text line by line. If the line has some assertions against it, those are checked. +/// Assertions are counted according to their status - succeeded or failed. Failures are also logged to stdout, depending on the output options. +/// When there are no assertions left to check, it returns those counts. /// `text` is the code containing syntax test assertions to be parsed and checked. /// `testtoken_start` is the token (normally a comment in the given syntax) that represents that assertions could be on the line. /// `testtoken_end` is an optional token that will be stripped from the line when retrieving the scope selector. Useful for syntaxes when the start token represents a block comment, to make the tests easier to construct. -pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult { +pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult { #[derive(Debug)] struct ScopedText { scope: Vec, @@ -166,7 +170,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes // parse the line let ops = state.parse_line(&line); - // find assertions that relate to the current line + // find all the assertions that relate to the current line relevant_assertions.clear(); while let Some(assertion) = assertions.pop_front() { let pos = assertion.test_line_offset + assertion.begin_char; @@ -178,6 +182,8 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } } if !relevant_assertions.is_empty() { + // if there are assertions for the line, show the operations for debugging purposes if specified in the output options + // (if there are no assertions, or the line contains assertions, debugging it would probably just add noise) scopes_on_line_being_tested.clear(); if out_opts.debug { println!("-- debugging line {} -- scope stack: {:?}", line_number, stack); @@ -196,6 +202,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes if s.is_empty() { // in this case we don't care about blank tokens continue; } + // if there are assertions against this line, store the scopes for comparison with the assertions if !relevant_assertions.is_empty() { let len = s.chars().count(); scopes_on_line_being_tested.push( @@ -218,8 +225,8 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes total_assertions += length; if !result.success { assertion_failures += length; - let text: String = line.chars().skip(result.column_begin).take(length).collect(); if !out_opts.summary { + let text: String = line.chars().skip(result.column_begin).take(length).collect(); println!(" Assertion selector {:?} \ from line {:?} failed against line {:?}, column range {:?}-{:?} \ (with text {:?}) \ @@ -235,6 +242,14 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes } offset = eol_offset; + + // no point continuing to parse the file if there are no syntax test assertions left + // (unless we want to prove that no panics etc. occur while parsing the rest of the file ofc...) + if assertions.is_empty() || (assertion_failures > 0 && out_opts.failfast) { + // NOTE: the total counts only really show how many assertions were checked when failing fast + // - they are not accurate total counts + break; + } } let res = if assertion_failures > 0 { From 6cba0699e5813ffbbea1f7f56f43833f5db50cc9 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Thu, 5 Jul 2018 09:14:40 +0300 Subject: [PATCH 07/12] switch syntax tests back to a normal Vec instead of a VecDeque --- src/syntax_tests.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index c29cb1f6..528c1feb 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -22,7 +22,6 @@ pub enum SyntaxTestFileResult { Success(usize), } -use std::collections::VecDeque; use highlighting::ScopeSelectors; #[derive(Debug)] @@ -35,8 +34,8 @@ struct SyntaxTestAssertionRange { scope_selector_text: String, } -fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque { - let mut assertions = VecDeque::new(); +fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec { + let mut assertions = Vec::new(); let mut test_line_offset = 0; let mut test_line_len = 0; let mut line_number = 0; @@ -89,10 +88,10 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: scope_selector_text: assertion.scope_selector_text.clone(), }; assertion.end_char = test_line_len; - assertions.push_back(assertion); - assertions.push_back(remainder); + assertions.push(assertion); + assertions.push(remainder); } else { - assertions.push_back(assertion); + assertions.push(assertion); } line_has_assertions = true; @@ -147,7 +146,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text results } - let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); + let assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text); //println!("{:?}", assertions); // iterate over the lines of the file, testing them @@ -158,6 +157,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text let mut scopes_on_line_being_tested = Vec::new(); let mut line_number = 0; let mut relevant_assertions = Vec::new(); + let mut assertion_index = 0; let mut assertion_failures: usize = 0; let mut total_assertions: usize = 0; @@ -172,12 +172,13 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text let ops = state.parse_line(&line); // find all the assertions that relate to the current line relevant_assertions.clear(); - while let Some(assertion) = assertions.pop_front() { + while assertion_index < assertions.len() { + let assertion = &assertions[assertion_index]; let pos = assertion.test_line_offset + assertion.begin_char; if pos >= offset && pos < eol_offset { relevant_assertions.push(assertion); + assertion_index += 1; } else { - assertions.push_front(assertion); break; } } @@ -245,7 +246,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text // no point continuing to parse the file if there are no syntax test assertions left // (unless we want to prove that no panics etc. occur while parsing the rest of the file ofc...) - if assertions.is_empty() || (assertion_failures > 0 && out_opts.failfast) { + if assertion_index == assertions.len() || (assertion_failures > 0 && out_opts.failfast) { // NOTE: the total counts only really show how many assertions were checked when failing fast // - they are not accurate total counts break; From 1d754cf9a0620ea7673e7257a16170f430286f03 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Mon, 15 Oct 2018 14:46:00 +0300 Subject: [PATCH 08/12] make the get_syntax_test_assertions method public this will allow other functionality to be developed that can use the same syntax test implementation and show the test results in a more friendly manner etc. --- src/syntax_tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 8a5a14d1..a47e0460 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -5,7 +5,8 @@ use parsing::{ScopeStack, ParseState, SyntaxReference, SyntaxSet, Scope}; //use std::io::Write; use std::str::FromStr; use util::debug_print_ops; -use easy::{ScopeRegionIterator}; +use easy::ScopeRegionIterator; +use highlighting::ScopeSelectors; #[derive(Clone, Copy)] pub struct SyntaxTestOutputOptions { @@ -22,10 +23,8 @@ pub enum SyntaxTestFileResult { Success(usize), } -use highlighting::ScopeSelectors; - #[derive(Debug)] -struct SyntaxTestAssertionRange { +pub struct SyntaxTestAssertionRange { test_line_offset: usize, line_number: usize, begin_char: usize, @@ -34,7 +33,7 @@ struct SyntaxTestAssertionRange { scope_selector_text: String, } -fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec { +pub fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec { let mut assertions = Vec::new(); let mut test_line_offset = 0; let mut test_line_len = 0; From b39cd4e6be1ba08f985ac9a40966312c7dff189d Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Sun, 21 Oct 2018 12:06:25 +0300 Subject: [PATCH 09/12] add doc comment for get_syntax_test_assertions --- src/syntax_tests.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index a47e0460..28771436 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -25,14 +25,18 @@ pub enum SyntaxTestFileResult { #[derive(Debug)] pub struct SyntaxTestAssertionRange { - test_line_offset: usize, - line_number: usize, - begin_char: usize, - end_char: usize, - scope_selector: ScopeSelectors, - scope_selector_text: String, + pub test_line_offset: usize, + pub line_number: usize, + pub begin_char: usize, + pub end_char: usize, + pub scope_selector: ScopeSelectors, + pub scope_selector_text: String, } +/// Given a start token, option end token and text, parse the syntax tests in the text +/// that follow the format described at http://www.sublimetext.com/docs/3/syntax.html#testing +/// and return the scope selector assertions found, so that when the text is parsed, +/// the assertions can be checked pub fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec { let mut assertions = Vec::new(); let mut test_line_offset = 0; From 6edd291e13729fd1f33281aec02fb7d94049b9e5 Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Sun, 21 Oct 2018 13:52:42 +0300 Subject: [PATCH 10/12] make syntax test header parsing public also --- Cargo.toml | 4 +-- examples/syntest.rs | 21 ++------------- src/syntax_tests.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d998980d..fd481033 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ yaml-rust = { version = "0.4", optional = true } onig = { version = "4.1", optional = true } walkdir = "2.0" regex-syntax = { version = "0.6", optional = true } +regex = { version = "1.0", optional = true } lazy_static = "1.0" lazycell = "1.0" bitflags = "1.0" @@ -32,7 +33,6 @@ serde_json = "1.0" [dev-dependencies] criterion = "0.2" rayon = "1.0.0" -regex = "1.0" getopts = "0.2" pretty_assertions = "0.5.0" @@ -51,7 +51,7 @@ dump-create = ["flate2/default", "bincode"] # Pure Rust dump creation, worse compressor so produces larger dumps than dump-create dump-create-rs = ["flate2/rust_backend", "bincode"] -parsing = ["onig", "regex-syntax", "fnv"] +parsing = ["onig", "regex-syntax", "fnv", "regex"] # The `assets` feature enables inclusion of the default theme and syntax packages. # For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set. assets = [] diff --git a/examples/syntest.rs b/examples/syntest.rs index 5d51f466..468164c3 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -9,13 +9,10 @@ // cargo run --example syntest testdata/Packages/JavaScript/syntax_test_json.json testdata/Packages/JavaScript/ extern crate syntect; extern crate walkdir; -#[macro_use] -extern crate lazy_static; -extern crate regex; extern crate getopts; use syntect::parsing::{SyntaxSet, SyntaxSetBuilder}; -use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions}; +use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions, parse_syntax_test_header_line, SyntaxTestHeader}; use std::path::Path; use std::io::prelude::*; @@ -24,7 +21,6 @@ use std::fs::File; use std::time::Instant; use getopts::Options; -use regex::Regex; use walkdir::{DirEntry, WalkDir}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -33,14 +29,6 @@ pub enum SyntaxTestHeaderError { SyntaxDefinitionNotFound, } -lazy_static! { - pub static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm) - ^(?P\s*\S+) - \s+SYNTAX\sTEST\s+ - "(?P[^"]+)" - \s*(?P\S+)?$ - "#).unwrap(); -} fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> Result { let f = File::open(path).unwrap(); @@ -54,12 +42,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> header_line = header_line.replace("\r", &""); // parse the syntax test header in the first line of the file - let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line); - let captures = search_result.ok_or(SyntaxTestHeaderError::MalformedHeader)?; - - let testtoken_start = captures.name("testtoken_start").unwrap().as_str(); - let testtoken_end = captures.name("testtoken_end").map_or(None, |c|Some(c.as_str())); - let syntax_file = captures.name("syntax_file").unwrap().as_str(); + let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?; // find the relevant syntax definition to parse the file with - case is important! if !out_opts.summary { diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 28771436..8e87ac81 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -8,6 +8,12 @@ use util::debug_print_ops; use easy::ScopeRegionIterator; use highlighting::ScopeSelectors; +// #[macro_use] +// extern crate lazy_static; +extern crate regex; + +use self::regex::Regex; + #[derive(Clone, Copy)] pub struct SyntaxTestOutputOptions { pub time: bool, @@ -33,6 +39,32 @@ pub struct SyntaxTestAssertionRange { pub scope_selector_text: String, } +lazy_static! { + static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm) + ^(?P\s*\S+) + \s+SYNTAX\sTEST\s+ + "(?P[^"]+)" + \s*(?P\S+)?\r?$ + "#).unwrap(); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyntaxTestHeader<'a> { + pub testtoken_start: &'a str, + pub testtoken_end: Option<&'a str>, + pub syntax_file: &'a str, +} + +pub fn parse_syntax_test_header_line(header_line: &str) -> Option { + let captures = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line/*.replace("\r", &"")*/)?; + + Some(SyntaxTestHeader { + testtoken_start: captures.name("testtoken_start").unwrap().as_str(), + testtoken_end: captures.name("testtoken_end").map_or(None, |c|Some(c.as_str())), + syntax_file: captures.name("syntax_file").unwrap().as_str(), + }) +} + /// Given a start token, option end token and text, parse the syntax tests in the text /// that follow the format described at http://www.sublimetext.com/docs/3/syntax.html#testing /// and return the scope selector assertions found, so that when the text is parsed, @@ -368,4 +400,36 @@ foobar assert_eq!(result[5].begin_char, 0); assert_eq!(result[5].end_char, 1); } + + #[test] + fn can_parse_syntax_test_header_with_end_token() { + let header = parse_syntax_test_header_line(&"").unwrap(); + assert_eq!(&header.testtoken_start, &""); + assert_eq!(&header.syntax_file, &"XML.sublime-syntax"); + } + + #[test] + fn can_parse_syntax_test_header_with_end_token_and_carriage_return() { + let header = parse_syntax_test_header_line(&"\r\n").unwrap(); + assert_eq!(&header.testtoken_start, &""); + assert_eq!(&header.syntax_file, &"XML.sublime-syntax"); + } + + #[test] + fn can_parse_syntax_test_header_with_no_end_token() { + let header = parse_syntax_test_header_line(&"// SYNTAX TEST \"Packages/Example/Example.sublime-syntax\"\n").unwrap(); + assert_eq!(&header.testtoken_start, &"//"); + assert!(!header.testtoken_end.is_some()); + assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax"); + } + + #[test] + fn can_parse_syntax_test_header_with_no_end_token_and_carriage_return() { + let header = parse_syntax_test_header_line(&"// SYNTAX TEST \"Packages/Example/Example.sublime-syntax\"\r").unwrap(); + assert_eq!(&header.testtoken_start, &"//"); + assert!(header.testtoken_end.is_none()); + assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax"); + } // } From 77f0de12130a20b01d471d1380932b59324b2c6d Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Mon, 22 Oct 2018 09:31:29 +0300 Subject: [PATCH 11/12] remove regex dependency --- Cargo.toml | 3 +-- src/syntax_tests.rs | 39 ++++++++++++++++----------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fd481033..0e1c20f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ yaml-rust = { version = "0.4", optional = true } onig = { version = "4.1", optional = true } walkdir = "2.0" regex-syntax = { version = "0.6", optional = true } -regex = { version = "1.0", optional = true } lazy_static = "1.0" lazycell = "1.0" bitflags = "1.0" @@ -51,7 +50,7 @@ dump-create = ["flate2/default", "bincode"] # Pure Rust dump creation, worse compressor so produces larger dumps than dump-create dump-create-rs = ["flate2/rust_backend", "bincode"] -parsing = ["onig", "regex-syntax", "fnv", "regex"] +parsing = ["onig", "regex-syntax", "fnv"] # The `assets` feature enables inclusion of the default theme and syntax packages. # For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set. assets = [] diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 8e87ac81..81671d51 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -8,12 +8,6 @@ use util::debug_print_ops; use easy::ScopeRegionIterator; use highlighting::ScopeSelectors; -// #[macro_use] -// extern crate lazy_static; -extern crate regex; - -use self::regex::Regex; - #[derive(Clone, Copy)] pub struct SyntaxTestOutputOptions { pub time: bool, @@ -39,15 +33,6 @@ pub struct SyntaxTestAssertionRange { pub scope_selector_text: String, } -lazy_static! { - static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm) - ^(?P\s*\S+) - \s+SYNTAX\sTEST\s+ - "(?P[^"]+)" - \s*(?P\S+)?\r?$ - "#).unwrap(); -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct SyntaxTestHeader<'a> { pub testtoken_start: &'a str, @@ -55,14 +40,22 @@ pub struct SyntaxTestHeader<'a> { pub syntax_file: &'a str, } -pub fn parse_syntax_test_header_line(header_line: &str) -> Option { - let captures = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line/*.replace("\r", &"")*/)?; - - Some(SyntaxTestHeader { - testtoken_start: captures.name("testtoken_start").unwrap().as_str(), - testtoken_end: captures.name("testtoken_end").map_or(None, |c|Some(c.as_str())), - syntax_file: captures.name("syntax_file").unwrap().as_str(), - }) +pub fn parse_syntax_test_header_line(header_line: &str) -> Option { // TODO: use a "impl<'a> From<&'a str> for SyntaxTestHeader<'a>" instead? + if let Some(pos) = &header_line.find(&" SYNTAX TEST \"") { + let filename_part = &header_line[*pos + " SYNTAX TEST \"".len()..]; + if let Some(close_pos) = filename_part.find(&"\"") { + let end_token = filename_part[close_pos + 1..].trim(); + Some(SyntaxTestHeader { + testtoken_start: &header_line[0..*pos], + testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) }, + syntax_file: &filename_part[0..close_pos], + }) + } else { + None + } + } else { + None + } } /// Given a start token, option end token and text, parse the syntax tests in the text From 59ddb378947c1b07e25a5915b04c45cfa16f653d Mon Sep 17 00:00:00 2001 From: Keith Hall Date: Tue, 14 Apr 2020 12:17:16 +0300 Subject: [PATCH 12/12] handle new syntax test header format which caters for reindentation --- examples/syntest.rs | 2 +- src/syntax_tests.rs | 42 +++++++++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/examples/syntest.rs b/examples/syntest.rs index 468164c3..c120ea99 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -42,7 +42,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> header_line = header_line.replace("\r", &""); // parse the syntax test header in the first line of the file - let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?; + let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file, reindent_text: _ } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?; // find the relevant syntax definition to parse the file with - case is important! if !out_opts.summary { diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs index 87789c68..1a32ee65 100644 --- a/src/syntax_tests.rs +++ b/src/syntax_tests.rs @@ -38,24 +38,27 @@ pub struct SyntaxTestHeader<'a> { pub testtoken_start: &'a str, pub testtoken_end: Option<&'a str>, pub syntax_file: &'a str, + pub reindent_text: Option<&'a str>, } pub fn parse_syntax_test_header_line(header_line: &str) -> Option { // TODO: use a "impl<'a> From<&'a str> for SyntaxTestHeader<'a>" instead? - if let Some(pos) = &header_line.find(&" SYNTAX TEST \"") { - let filename_part = &header_line[*pos + " SYNTAX TEST \"".len()..]; - if let Some(close_pos) = filename_part.find(&"\"") { - let end_token = filename_part[close_pos + 1..].trim(); - Some(SyntaxTestHeader { - testtoken_start: &header_line[0..*pos], - testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) }, - syntax_file: &filename_part[0..close_pos], - }) - } else { - None + if let Some(pos) = &header_line.find(&" SYNTAX TEST ") { + let after_text_pos = *pos + &" SYNTAX TEST ".len(); + if let Some(quote_start_pos) = &header_line[after_text_pos..].find("\"") { + let reindent_text = if *quote_start_pos == 0 { None } else { Some(header_line[after_text_pos..after_text_pos + *quote_start_pos].trim()) }; + let filename_part = &header_line[after_text_pos + *quote_start_pos + 1..]; + if let Some(close_pos) = filename_part.find(&"\"") { + let end_token = filename_part[close_pos + 1..].trim(); + return Some(SyntaxTestHeader { + testtoken_start: &header_line[0..*pos], + testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) }, + syntax_file: &filename_part[0..close_pos], + reindent_text: reindent_text, + }); + } } - } else { - None } + None } /// Given a start token, option end token and text, parse the syntax tests in the text @@ -400,6 +403,7 @@ foobar assert_eq!(&header.testtoken_start, &""); assert_eq!(&header.syntax_file, &"XML.sublime-syntax"); + assert!(&header.reindent_text.is_none()); } #[test] @@ -408,6 +412,7 @@ foobar assert_eq!(&header.testtoken_start, &""); assert_eq!(&header.syntax_file, &"XML.sublime-syntax"); + assert!(&header.reindent_text.is_none()); } #[test] @@ -416,6 +421,7 @@ foobar assert_eq!(&header.testtoken_start, &"//"); assert!(!header.testtoken_end.is_some()); assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax"); + assert!(&header.reindent_text.is_none()); } #[test] @@ -424,5 +430,15 @@ foobar assert_eq!(&header.testtoken_start, &"//"); assert!(header.testtoken_end.is_none()); assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax"); + assert!(&header.reindent_text.is_none()); + } + + #[test] + fn can_parse_syntax_test_reindentation_header() { + let header = parse_syntax_test_header_line(&"// SYNTAX TEST reindent-unchanged reindent-unindented \"Packages/PHP/PHP.sublime-syntax\"").unwrap(); + assert_eq!(&header.testtoken_start, &"//"); + assert!(header.testtoken_end.is_none()); + assert_eq!(&header.syntax_file, &"Packages/PHP/PHP.sublime-syntax"); + assert_eq!(&header.reindent_text.unwrap(), &"reindent-unchanged reindent-unindented"); } // }