Escape fewer Unicode codepoints in Debug impl of str

Use the same procedure as Python to determine whether a character is
printable, described in [PEP 3138]. In particular, this means that the
following character classes are escaped:

- Cc (Other, Control)
- Cf (Other, Format)
- Cs (Other, Surrogate), even though they can't appear in Rust strings
- Co (Other, Private Use)
- Cn (Other, Not Assigned)
- Zl (Separator, Line)
- Zp (Separator, Paragraph)
- Zs (Separator, Space), except for the ASCII space `' '` (`0x20`)

This allows for user-friendly inspection of strings that are not
English (e.g. compare `"\u{e9}\u{e8}\u{ea}"` to `"éèê"`).

Fixes rust-lang#34318.

[PEP 3138]: https://www.python.org/dev/peps/pep-3138/

src/etc/char_private.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+#
+# Copyright 2011-2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This script uses the following Unicode tables:
+# - Categories.txt
+
+import os
+import subprocess
+
+def to_ranges(iter):
+    current = None
+    for i in iter:
+        if current is None or i != current[1] or i in (0x10000, 0x20000):
+            if current is not None:
+                yield tuple(current)
+            current = [i, i + 1]
+        else:
+            current[1] += 1
+    if current is not None:
+        yield tuple(current)
+
+def get_escaped(dictionary):
+    for i in range(0x110000):
+        if dictionary.get(i, "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and i != ord(' '):
+            yield i
+
+def get_file(f):
+    try:
+        return open(os.path.basename(f))
+    except FileNotFoundError:
+        subprocess.run(["curl", "-O", f], check=True)
+        return open(os.path.basename(f))
+
+def main():
+    file = get_file("http://www.unicode.org/notes/tn36/Categories.txt")
+
+    dictionary = {int(line.split()[0], 16): line.split()[1] for line in file}
+
+    CUTOFF=0x10000
+    singletons0 = []
+    singletons1 = []
+    normal0 = []
+    normal1 = []
+    extra = []
+
+    for a, b in to_ranges(get_escaped(dictionary)):
+        if a > 2 * CUTOFF:
+            extra.append((a, b - a))
+        elif a == b - 1:
+            if a & CUTOFF:
+                singletons1.append(a & ~CUTOFF)
+            else:
+                singletons0.append(a)
+        elif a == b - 2:
+            if a & CUTOFF:
+                singletons1.append(a & ~CUTOFF)
+                singletons1.append((a + 1) & ~CUTOFF)
+            else:
+                singletons0.append(a)
+                singletons0.append(a + 1)
+        else:
+            if a >= 2 * CUTOFF:
+                extra.append((a, b - a))
+            elif a & CUTOFF:
+                normal1.append((a & ~CUTOFF, b - a))
+            else:
+                normal0.append((a, b - a))
+
+    print("""\
+// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// NOTE: The following code was generated by "src/etc/char_private.py",
+//       do not edit directly!
+
+use slice::SliceExt;
+
+fn check(x: u16, singletons: &[u16], normal: &[u16]) -> bool {
+    for &s in singletons {
+        if x == s {
+            return false;
+        } else if x < s {
+            break;
+        }
+    }
+    for w in normal.chunks(2) {
+        let start = w[0];
+        let len = w[1];
+        let difference = (x as i32) - (start as i32);
+        if 0 <= difference {
+            if difference < len as i32 {
+                return false;
+            }
+        } else {
+            break;
+        }
+    }
+    true
+}
+
+pub fn is_printable(x: char) -> bool {
+    let x = x as u32;
+    let lower = x as u16;
+    if x < 0x10000 {
+        check(lower, SINGLETONS0, NORMAL0)
+    } else if x < 0x20000 {
+        check(lower, SINGLETONS1, NORMAL1)
+    } else {\
+""")
+    for a, b in extra:
+        print("        if 0x{:x} <= x && x < 0x{:x} {{".format(a, a + b))
+        print("            return false;")
+        print("        }")
+    print("""\
+        true
+    }
+}\
+""")
+    print()
+    print("const SINGLETONS0: &'static [u16] = &[")
+    for s in singletons0:
+        print("    0x{:x},".format(s))
+    print("];")
+    print("const SINGLETONS1: &'static [u16] = &[")
+    for s in singletons1:
+        print("    0x{:x},".format(s))
+    print("];")
+    print("const NORMAL0: &'static [u16] = &[")
+    for a, b in normal0:
+        print("    0x{:x}, 0x{:x},".format(a, b))
+    print("];")
+    print("const NORMAL1: &'static [u16] = &[")
+    for a, b in normal1:
+        print("    0x{:x}, 0x{:x},".format(a, b))
+    print("];")
+
+if __name__ == '__main__':
+    main()

src/libcollectionstest/str.rs

@@ -707,12 +707,14 @@ fn test_escape_unicode() {
 fn test_escape_default() {
     assert_eq!("abc".escape_default(), "abc");
     assert_eq!("a c".escape_default(), "a c");
+    assert_eq!("éèê".escape_default(), "éèê");
     assert_eq!("\r\n\t".escape_default(), "\\r\\n\\t");
     assert_eq!("'\"\\".escape_default(), "\\'\\\"\\\\");
-    assert_eq!("\u{100}\u{ffff}".escape_default(), "\\u{100}\\u{ffff}");
-    assert_eq!("\u{10000}\u{10ffff}".escape_default(), "\\u{10000}\\u{10ffff}");
-    assert_eq!("ab\u{fb00}".escape_default(), "ab\\u{fb00}");
-    assert_eq!("\u{1d4ea}\r".escape_default(), "\\u{1d4ea}\\r");
+    assert_eq!("\u{7f}\u{ff}".escape_default(), "\\u{7f}\u{ff}");
+    assert_eq!("\u{100}\u{ffff}".escape_default(), "\u{100}\\u{ffff}");
+    assert_eq!("\u{10000}\u{10ffff}".escape_default(), "\u{10000}\\u{10ffff}");
+    assert_eq!("ab\u{200b}".escape_default(), "ab\\u{200b}");
+    assert_eq!("\u{10d4ea}\r".escape_default(), "\\u{10d4ea}\\r");
 }
 
 #[test]

src/libcore/char.rs

@@ -17,6 +17,7 @@
 
 use prelude::v1::*;
 
+use char_private::is_printable;
 use mem::transmute;
 
 // UTF-8 ranges and tags for encoding characters
@@ -320,8 +321,8 @@ impl CharExt for char {
             '\r' => EscapeDefaultState::Backslash('r'),
             '\n' => EscapeDefaultState::Backslash('n'),
             '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
-            '\x20' ... '\x7e' => EscapeDefaultState::Char(self),
-            _ => EscapeDefaultState::Unicode(self.escape_unicode())
+            c if is_printable(c) => EscapeDefaultState::Char(c),
+            c => EscapeDefaultState::Unicode(c.escape_unicode()),
         };
         EscapeDefault { state: init_state }
     }