Merge pull request #191 from dandavison/188-non-utf-8-handling

Handle non utf-8 with lossy decoding of raw bytes

Cargo.toml

@@ -20,6 +20,7 @@ ansi_colours = "1.0.1"
 ansi_term = "0.12.1"
 atty = "0.2.14"
 box_drawing = "0.1.2"
+bytelines = "2.2.2"
 console = "0.11.2"
 dirs = "2.0"
 lazy_static = "1.4"

src/delta.rs

@@ -1,6 +1,8 @@
 use std::io::Write;
 
+use bytelines::ByteLines;
 use console::strip_ansi_codes;
+use std::io::BufRead;
 use unicode_segmentation::UnicodeSegmentation;
 
 use crate::bat::assets::HighlightingAssets;
@@ -51,21 +53,22 @@ impl State {
 // | HunkPlus    | flush, emit | flush, emit | flush, emit | flush, emit | flush, push | push     |
 
 pub fn delta<I>(
-    lines: I,
+    mut lines: ByteLines<I>,
     config: &Config,
     assets: &HighlightingAssets,
     writer: &mut dyn Write,
 ) -> std::io::Result<()>
 where
-    I: Iterator<Item = String>,
+    I: BufRead,
 {
     let mut painter = Painter::new(writer, config, assets);
     let mut minus_file = "".to_string();
     let mut plus_file;
     let mut state = State::Unknown;
     let mut source = Source::Unknown;
 
-    for raw_line in lines {
+    while let Some(Ok(raw_line_bytes)) = lines.next() {
+        let raw_line = String::from_utf8_lossy(&raw_line_bytes);
         let line = strip_ansi_codes(&raw_line).to_string();
         if source == Source::Unknown {
             source = detect_source(&line);
@@ -402,6 +405,7 @@ mod tests {
     use super::*;
     use console::strip_ansi_codes;
     use std::env;
+    use std::io::BufReader;
     use syntect::highlighting::StyleModifier;
 
     use crate::paint;
@@ -633,7 +637,7 @@ mod tests {
         let config = cli::process_command_line_arguments(&assets, &options);
 
         delta(
-            input.split("\n").map(String::from),
+            ByteLines::new(BufReader::new(input.as_bytes())),
             &config,
             &assets,
             &mut writer,
@@ -734,11 +738,8 @@ mod tests {
             let mut options = get_command_line_options();
             options.color_only = true;
             let output = run_delta(input, &options);
-            assert_eq!(
-                strip_ansi_codes(&output).to_string(),
-                input.to_owned() + "\n"
-            );
-            assert_ne!(output, input.to_owned() + "\n");
+            assert_eq!(strip_ansi_codes(&output).to_string(), input.to_owned());
+            assert_ne!(output, input.to_owned());
         }
     }
 
@@ -748,7 +749,7 @@ mod tests {
         let output = run_delta(DIFF_WITH_MERGE_CONFLICT, &options);
         // TODO: The + in the first column is being removed.
         assert!(strip_ansi_codes(&output).contains("+>>>>>>> Stashed changes"));
-        assert_eq!(output.split('\n').count(), 47);
+        assert_eq!(output.split('\n').count(), 46);
     }
 
     #[test]
@@ -758,7 +759,7 @@ mod tests {
         let output = run_delta(DIFF_WITH_MERGE_CONFLICT, &options);
         assert_eq!(
             strip_ansi_codes(&output).to_string(),
-            DIFF_WITH_MERGE_CONFLICT.to_owned() + "\n"
+            DIFF_WITH_MERGE_CONFLICT.to_owned()
         );
     }
 

src/main.rs

@@ -13,11 +13,12 @@ mod paint;
 mod parse;
 mod style;
 
-use std::io::{self, BufRead, ErrorKind, Read, Write};
+use std::io::{self, ErrorKind, Read, Write};
 use std::process;
 
 use ansi_term;
 use atty;
+use bytelines::ByteLinesReader;
 use structopt::StructOpt;
 use syntect::highlighting::{Color, FontStyle, Style};
 
@@ -62,10 +63,7 @@ fn main() -> std::io::Result<()> {
     let mut writer = output_type.handle().unwrap();
 
     if let Err(error) = delta(
-        io::stdin()
-            .lock()
-            .lines()
-            .map(|l| l.unwrap_or_else(|_| "<delta: invalid utf-8 data>".to_string())),
+        io::stdin().lock().byte_lines(),
         &config,
         &assets,
         &mut writer,
@@ -122,10 +120,15 @@ fn get_painted_rgb_string(color: Color, true_color: bool) -> String {
 }
 
 fn list_themes(assets: &HighlightingAssets) -> std::io::Result<()> {
+    use bytelines::ByteLines;
+    use std::io::BufReader;
     let opt = cli::Opt::from_args();
-    let mut input = String::new();
-    if atty::is(atty::Stream::Stdin) {
-        input = "\
+    let input = if !atty::is(atty::Stream::Stdin) {
+        let mut buf = Vec::new();
+        io::stdin().lock().read_to_end(&mut buf)?;
+        buf
+    } else {
+        b"\
 diff --git a/example.rs b/example.rs
 index f38589a..0f1bb83 100644
 --- a/example.rs
@@ -140,10 +143,8 @@ index f38589a..0f1bb83 100644
 +    let result = f64::powf(num, 3.0);
 +    println!(\"The cube of {:.2} is {:.2}.\", num, result);
  }"
-        .to_string()
-    } else {
-        io::stdin().read_to_string(&mut input)?;
-    }
+        .to_vec()
+    };
 
     let stdout = io::stdout();
     let mut stdout = stdout.lock();
@@ -166,7 +167,7 @@ index f38589a..0f1bb83 100644
         let mut writer = output_type.handle().unwrap();
 
         if let Err(error) = delta(
-            input.split('\n').map(String::from),
+            ByteLines::new(BufReader::new(&input[0..])),
             &config,
             &assets,
             &mut writer,