From b3e8c8bbe27e21a2e67039d9fb9ea41cb83b1499 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 30 Jul 2019 13:45:08 +0300 Subject: [PATCH] adapt rustdoc to infailable lexer --- src/librustdoc/html/highlight.rs | 27 +-- .../passes/check_code_block_syntax.rs | 32 +--- src/test/rustdoc-ui/invalid-syntax.stderr | 168 ++++++++++++++++-- 3 files changed, 177 insertions(+), 50 deletions(-) diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 92d85ef9cacc5..5d86ee9721b75 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -44,7 +44,7 @@ pub fn render_with_highlighting( let mut highlighted_source = vec![]; if classifier.write_source(&mut highlighted_source).is_err() { - Err(classifier.lexer.buffer_fatal_errors()) + Err(()) } else { Ok(String::from_utf8_lossy(&highlighted_source).into_owned()) } @@ -59,14 +59,9 @@ pub fn render_with_highlighting( } write_footer(&mut out).unwrap(); } - Err(errors) => { - // If errors are encountered while trying to highlight, cancel the errors and just emit - // the unhighlighted source. The errors will have already been reported in the - // `check-code-block-syntax` pass. - for mut error in errors { - error.cancel(); - } - + Err(()) => { + // If errors are encountered while trying to highlight, just emit + // the unhighlighted source. write!(out, "
{}
", src).unwrap(); } } @@ -192,14 +187,20 @@ impl<'a> Classifier<'a> { if let Some(token) = self.peek_token.take() { return Ok(token); } - self.lexer.try_next_token().map_err(|()| HighlightError::LexError) + let token = self.lexer.next_token(); + if let token::Unknown(..) = &token.kind { + return Err(HighlightError::LexError); + } + Ok(token) } fn peek(&mut self) -> Result<&Token, HighlightError> { if self.peek_token.is_none() { - self.peek_token = Some( - self.lexer.try_next_token().map_err(|()| HighlightError::LexError)? - ); + let token = self.lexer.next_token(); + if let token::Unknown(..) = &token.kind { + return Err(HighlightError::LexError); + } + self.peek_token = Some(token); } Ok(self.peek_token.as_ref().unwrap()) } diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs index 0488153e7cb73..357e17d2d1bc4 100644 --- a/src/librustdoc/passes/check_code_block_syntax.rs +++ b/src/librustdoc/passes/check_code_block_syntax.rs @@ -32,24 +32,20 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { dox[code_block.code].to_owned(), ); - let errors = { + let has_errors = { + let mut has_errors = false; let mut lexer = Lexer::new(&sess, source_file, None); - while let Ok(token::Token { kind, .. }) = lexer.try_next_token() { - if kind == token::Eof { - break; + loop { + match lexer.next_token().kind { + token::Eof => break, + token::Unknown(..) => has_errors = true, + _ => (), } } - - let errors = lexer.buffer_fatal_errors(); - - if !errors.is_empty() { - Err(errors) - } else { - Ok(()) - } + has_errors }; - if let Err(errors) = errors { + if has_errors { let mut diag = if let Some(sp) = super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs) { @@ -58,11 +54,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { .sess() .struct_span_warn(sp, "could not parse code block as Rust code"); - for mut err in errors { - diag.note(&format!("error from rustc: {}", err.message())); - err.cancel(); - } - if code_block.syntax.is_none() && code_block.is_fenced { let sp = sp.from_inner(InnerSpan::new(0, 3)); diag.span_suggestion( @@ -82,11 +73,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { "doc comment contains an invalid Rust code block", ); - for mut err in errors { - // Don't bother reporting the error, because we can't show where it happened. - err.cancel(); - } - if code_block.syntax.is_none() && code_block.is_fenced { diag.help("mark blocks that do not contain Rust code as text: ```text"); } diff --git a/src/test/rustdoc-ui/invalid-syntax.stderr b/src/test/rustdoc-ui/invalid-syntax.stderr index b4ed747b44c81..3bebbecb9dfcf 100644 --- a/src/test/rustdoc-ui/invalid-syntax.stderr +++ b/src/test/rustdoc-ui/invalid-syntax.stderr @@ -1,3 +1,21 @@ +error: unknown start of token: \ + --> :1:1 + | +1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/ + | ^ + +error: unknown start of token: \ + --> :1:43 + | +1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/ + | ^ + +error: unknown start of token: \ + --> :1:60 + | +1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/ + | ^ + warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:3:5 | @@ -6,13 +24,31 @@ LL | /// ``` LL | | /// \__________pkt->size___________/ \_result->size_/ \__pkt->size__/ LL | | /// ``` | |_______^ - | - = note: error from rustc: unknown start of token: \ help: mark blocks that do not contain Rust code as text | LL | /// ```text | ^^^^^^^ +error: unknown start of token: ` + --> :3:30 + | +3 | | ^^^^^^ did you mean `baz::foobar`? + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +3 | | ^^^^^^ did you mean 'baz::foobar`? + | ^ + +error: unknown start of token: ` + --> :3:42 + | +3 | | ^^^^^^ did you mean `baz::foobar`? + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +3 | | ^^^^^^ did you mean `baz::foobar'? + | ^ + warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:8:5 | @@ -23,13 +59,17 @@ LL | | /// LL | use foobar::Baz; LL | | /// | ^^^^^^ did you mean `baz::foobar`? LL | | /// ``` | |_______^ - | - = note: error from rustc: unknown start of token: ` help: mark blocks that do not contain Rust code as text | LL | /// ```text | ^^^^^^^ +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:19:5 | @@ -38,13 +78,17 @@ LL | /// ``` LL | | /// \_ LL | | /// ``` | |_______^ - | - = note: error from rustc: unknown start of token: \ help: mark blocks that do not contain Rust code as text | LL | /// ```text | ^^^^^^^ +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:32:5 | @@ -53,8 +97,12 @@ LL | /// ```rust LL | | /// \_ LL | | /// ``` | |_______^ - | - = note: error from rustc: unknown start of token: \ + +error: unknown start of token: \ + --> :2:5 + | +2 | \_ + | ^ warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:41:9 @@ -63,16 +111,48 @@ LL | /// code with bad syntax | _________^ LL | | /// \_ | |__________^ - | - = note: error from rustc: unknown start of token: \ + +error: unknown start of token: ` + --> :1:1 + | +1 | ``` + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +1 | '`` + | ^ + +error: unknown start of token: ` + --> :1:2 + | +1 | ``` + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +1 | `'` + | ^ + +error: unknown start of token: ` + --> :1:3 + | +1 | ``` + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +1 | ``' + | ^ warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:55:9 | LL | /// ``` | ^^^ - | - = note: error from rustc: unknown start of token: ` + +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ warning: could not parse code block as Rust code --> $DIR/invalid-syntax.rs:58:5 @@ -82,8 +162,12 @@ LL | /// ```edition2018 LL | | /// \_ LL | | /// ``` | |_______^ - | - = note: error from rustc: unknown start of token: \ + +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ warning: doc comment contains an invalid Rust code block --> $DIR/invalid-syntax.rs:63:1 @@ -95,3 +179,59 @@ LL | | #[doc = "```"] | = help: mark blocks that do not contain Rust code as text: ```text +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + +error: unknown start of token: ` + --> :1:1 + | +1 | ``` + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +1 | '`` + | ^ + +error: unknown start of token: \ + --> :2:1 + | +2 | \_ + | ^ + +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + +error: unknown start of token: \ + --> :1:1 + | +1 | \_ + | ^ + +error: unknown start of token: ` + --> :3:30 + | +3 | | ^^^^^^ did you mean `baz::foobar`? + | ^ +help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not + | +3 | | ^^^^^^ did you mean 'baz::foobar`? + | ^ + +error: unknown start of token: \ + --> :1:1 + | +1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/ + | ^ +