From 4c0bd28a61d911a179be86e8e82a1501853406a4 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sun, 8 Oct 2023 14:28:55 -0700 Subject: [PATCH 1/2] Add regression test for issue 410 --- tests/test.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test.rs b/tests/test.rs index 4a6ca75..fa3fa34 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -328,10 +328,17 @@ fn literal_span() { #[cfg(span_locations)] #[test] fn source_text() { - let input = " 𓀕 "; - let tokens = input.parse::().unwrap(); - let ident = tokens.into_iter().next().unwrap(); + let input = " 𓀕 c "; + let mut tokens = input + .parse::() + .unwrap() + .into_iter(); + + let ident = tokens.next().unwrap(); assert_eq!("𓀕", ident.span().source_text().unwrap()); + + let ident = tokens.next().unwrap(); + assert_eq!("c", ident.span().source_text().unwrap()); // FIXME } #[test] From 137ae0a341be5cb04be12f251f48911437b566cd Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sun, 8 Oct 2023 14:31:09 -0700 Subject: [PATCH 2/2] Fix source_text treating span.lo as byte offset not char index --- src/fallback.rs | 5 ++++- tests/test.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fallback.rs b/src/fallback.rs index 281b7b2..70deb4b 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -364,7 +364,10 @@ impl FileInfo { fn source_text(&self, span: Span) -> String { let lo = (span.lo - self.span.lo) as usize; - let trunc_lo = &self.source_text[lo..]; + let trunc_lo = match self.source_text.char_indices().nth(lo) { + Some((offset, _ch)) => &self.source_text[offset..], + None => return String::new(), + }; let char_len = (span.hi - span.lo) as usize; let source_text = match trunc_lo.char_indices().nth(char_len) { Some((offset, _ch)) => &trunc_lo[..offset], diff --git a/tests/test.rs b/tests/test.rs index fa3fa34..1916a85 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -338,7 +338,7 @@ fn source_text() { assert_eq!("𓀕", ident.span().source_text().unwrap()); let ident = tokens.next().unwrap(); - assert_eq!("c", ident.span().source_text().unwrap()); // FIXME + assert_eq!("c", ident.span().source_text().unwrap()); } #[test]