Improve UTF-8 support

CHANGELOG.md

@@ -1,13 +1,18 @@
 # Changelog
 
 ## Unreleased
+- Improve UTF-8 support (#267)
+- Bump acpi from 4.0.0 to 4.1.0 (#265)
 - Add shell redirections (#262)
+- Bump linked_list_allocator from 0.9.0 to 0.9.1 (#256)
 - Add calc command (#263)
 - Add website (#261)
 - Fix VGA issues with real hardware (#258)
 - Add rust binaries support (#255)
 - Add dynamical disk information (#252)
+- Remove array-macro dependency (#253)
 - Add spawn syscall (#251)
+- Bump x86_64 from 0.14.5 to 0.14.6 (#247)
 - Add ELF loader (#248)
 - Add basic userspace (#228)
 - Bump acpi from 3.1.0 to 4.0.0 (#243)

src/api/console.rs

@@ -81,7 +81,9 @@ fn color_to_bg(name: &str) -> Option<usize> {
 
 pub fn is_printable(c: char) -> bool {
     if cfg!(feature = "video") {
-        c.is_ascii() && sys::vga::is_printable(c as u8)
+        // Check if the char can be converted to ASCII or Extended ASCII before
+        // asking the VGA driver if it's printable.
+        ((c as u32) < 0xFF) && sys::vga::is_printable(c as u8)
     } else {
         true // TODO
     }

src/api/io.rs

@@ -13,10 +13,11 @@ impl Stdin {
     }
 
     pub fn read_char(&self) -> Option<char> {
-        let mut buf = vec![0; 1];
+        let mut buf = vec![0; 4];
         if let Some(bytes) = syscall::read(0, &mut buf) {
             if bytes > 0 {
-                return Some(buf[0] as char);
+                buf.resize(bytes, 0);
+                return Some(String::from_utf8_lossy(&buf).to_string().remove(0));
             }
         }
         None

src/api/prompt.rs

@@ -10,7 +10,7 @@ pub struct Prompt {
     pub history: History,
     offset: usize, // Offset line by the length of the prompt string
     cursor: usize,
-    line: String,
+    line: Vec<char>, // UTF-32
 }
 
 impl Prompt {
@@ -20,15 +20,15 @@ impl Prompt {
             history: History::new(),
             offset: 0,
             cursor: 0,
-            line: String::new(),
+            line: Vec::with_capacity(80),
         }
     }
 
     pub fn input(&mut self, prompt: &str) -> Option<String> {
         print!("{}", prompt);
         self.offset = offset_from_prompt(prompt);
         self.cursor = self.offset;
-        self.line = String::new();
+        self.line = Vec::with_capacity(80);
         let mut parser = Parser::new();
         while let Some(c) = io::stdin().read_char() {
             match c {
@@ -44,11 +44,11 @@ impl Prompt {
                     self.update_completion();
                     self.update_history();
                     println!();
-                    return Some(self.line.clone());
+                    return Some(self.line.iter().collect());
                 },
                 c => {
-                    if c.is_ascii() {
-                        parser.advance(self, c as u8);
+                   for b in c.to_string().as_bytes() {
+                        parser.advance(self, *b);
                     }
                 }
             }
@@ -59,15 +59,15 @@ impl Prompt {
 
     fn update_history(&mut self) {
         if let Some(i) = self.history.pos {
-            self.line = self.history.entries[i].clone();
+            self.line = self.history.entries[i].chars().collect();
             self.history.pos = None;
         }
     }
 
     fn update_completion(&mut self) {
         if let Some(i) = self.completion.pos {
-            let complete = self.completion.entries[i].clone();
-            self.line.push_str(&complete);
+            let mut complete: Vec<char> = self.completion.entries[i].chars().collect();
+            self.line.append(&mut complete);
             self.cursor += complete.len();
             self.completion.pos = None;
             self.completion.entries = Vec::new();
@@ -83,15 +83,16 @@ impl Prompt {
                     self.update_completion();
                     return;
                 }
-                let bs = self.completion.entries[pos].len();
+                let bs = self.completion.entries[pos].chars().count();
                 if pos + 1 < n {
                     (bs, pos + 1)
                 } else {
                     (bs, 0)
                 }
             },
             None => {
-                self.completion.entries = (self.completion.completer)(&self.line);
+                let line: String = self.line.iter().collect();
+                self.completion.entries = (self.completion.completer)(&line);
                 if !self.completion.entries.is_empty() {
                     (0, 0)
                 } else {
@@ -112,14 +113,14 @@ impl Prompt {
             return;
         }
         let (bs, i) = match self.history.pos {
-            Some(i) => (self.history.entries[i].len(), cmp::max(i, 1) - 1),
+            Some(i) => (self.history.entries[i].chars().count(), cmp::max(i, 1) - 1),
             None => (self.line.len(), n - 1),
         };
         let line = &self.history.entries[i];
         let blank = ' '.to_string().repeat((self.offset + bs) - self.cursor);
         let erase = '\x08'.to_string().repeat(bs);
         print!("{}{}{}", blank, erase, line);
-        self.cursor = self.offset + line.len();
+        self.cursor = self.offset + line.chars().count();
         self.history.pos = Some(i);
     }
 
@@ -130,17 +131,17 @@ impl Prompt {
             return;
         }
         let (bs, i) = match self.history.pos {
-            Some(i) => (self.history.entries[i].len(), i + 1),
+            Some(i) => (self.history.entries[i].chars().count(), i + 1),
             None => return,
         };
         let (pos, line) = if i < n {
-            (Some(i), &self.history.entries[i])
+            (Some(i), self.history.entries[i].clone())
         } else {
-            (None, &self.line)
+            (None, self.line.iter().collect())
         };
         let erase = '\x08'.to_string().repeat(bs);
         print!("{}{}", erase, line);
-        self.cursor = self.offset + line.len();
+        self.cursor = self.offset + line.chars().count();
         self.history.pos = pos;
     }
 
@@ -168,8 +169,10 @@ impl Prompt {
         if self.cursor < self.offset + self.line.len() {
             let i = self.cursor - self.offset;
             self.line.remove(i);
-            let s = &self.line[i..];
-            print!("{} \x1b[{}D", s, s.len() + 1);
+            let s = &self.line[i..]; // UTF-32
+            let n = s.len() + 1;
+            let s: String = s.into_iter().collect(); // UTF-8
+            print!("{} \x1b[{}D", s, n);
         }
     }
 
@@ -179,8 +182,10 @@ impl Prompt {
         if self.cursor > self.offset {
             let i = self.cursor - self.offset - 1;
             self.line.remove(i);
-            let s = &self.line[i..];
-            print!("{}{} \x1b[{}D", '\x08', s, s.len() + 1);
+            let s = &self.line[i..]; // UTF-32
+            let n = s.len() + 1;
+            let s: String = s.into_iter().collect(); // UTF-8
+            print!("{}{} \x1b[{}D", '\x08', s, n);
             self.cursor -= 1;
         }
     }
@@ -189,11 +194,15 @@ impl Prompt {
         self.update_completion();
         self.update_history();
         if console::is_printable(c) {
+            let c = (c as u8) as char;
             let i = self.cursor - self.offset;
             self.line.insert(i, c);
-            let s = &self.line[i..];
-            print!("{} \x1b[{}D", s, s.len());
+            let s = &self.line[i..]; // UTF-32
+            let n = s.len();
+            let s: String = s.into_iter().collect(); // UTF-8
+            print!("{} \x1b[{}D", s, n);
             self.cursor += 1;
+        } else {
         }
     }
 }

src/api/regex.rs

@@ -79,11 +79,11 @@ impl Regex {
         self.find(text).is_some()
     }
     pub fn find(&self, text: &str) -> Option<(usize, usize)> {
-        let vec_re: Vec<char> = self.0.chars().collect();
-        let vec_text: Vec<char> = text.chars().collect();
+        let text: Vec<char> = text.chars().collect(); // UTF-32
+        let re: Vec<char> = self.0.chars().collect(); // UTF-32
         let mut start = 0;
         let mut end = 0;
-        if is_match(&vec_re[..], &vec_text[..], &mut start, &mut end) {
+        if is_match(&re[..], &text[..], &mut start, &mut end) {
             Some((start, end))
         } else {
             None

src/sys/console.rs

@@ -24,7 +24,7 @@ impl Console {
 
 impl FileIO for Console {
     fn read(&mut self, buf: &mut [u8]) -> Result<usize, ()> {
-        let mut s = if buf.len() == 1 {
+        let mut s = if buf.len() == 4 {
             read_char().to_string()
         } else {
             read_line()
@@ -105,12 +105,13 @@ pub fn key_handle(key: char) {
             if is_echo_enabled() {
                 let n = match c {
                     ETX_KEY | EOT_KEY | ESC_KEY => 2,
-                    _ => c.len_utf8(),
+                    _ => if (c as u32) < 0xFF { 1 } else { c.len_utf8() },
                 };
                 print_fmt(format_args!("{}", BS_KEY.to_string().repeat(n)));
             }
         }
     } else {
+        let key = if (key as u32) < 0xFF { (key as u8) as char } else { key };
         stdin.push(key);
         if is_echo_enabled() {
             match key {