idna maintenance and performance improvements

idna/Cargo.toml

@@ -19,6 +19,7 @@ harness = false
 name = "unit"
 
 [dev-dependencies]
+assert_matches = "1.3"
 bencher = "0.1"
 rustc-test = "0.3"
 serde_json = "1.0"

idna/src/lib.rs

@@ -38,7 +38,7 @@ extern crate matches;
 pub mod punycode;
 mod uts46;
 
-pub use crate::uts46::{Config, Errors};
+pub use crate::uts46::{Codec, Config, Errors};
 
 /// The [domain to ASCII](https://url.spec.whatwg.org/#concept-domain-to-ascii) algorithm.
 ///

idna/src/punycode.rs

@@ -52,81 +52,157 @@ pub fn decode_to_string(input: &str) -> Option<String> {
 /// Overflow can only happen on inputs that take more than
 /// 63 encoded bytes, the DNS limit on domain name labels.
 pub fn decode(input: &str) -> Option<Vec<char>> {
-    // Handle "basic" (ASCII) code points.
-    // They are encoded as-is before the last delimiter, if any.
-    let (mut output, input) = match input.rfind(DELIMITER) {
-        None => (Vec::new(), input),
-        Some(position) => (
-            input[..position].chars().collect(),
-            if position > 0 {
-                &input[position + 1..]
-            } else {
-                input
-            },
-        ),
-    };
-    let mut code_point = INITIAL_N;
-    let mut bias = INITIAL_BIAS;
-    let mut i = 0;
-    let mut iter = input.bytes();
-    loop {
-        let previous_i = i;
-        let mut weight = 1;
-        let mut k = BASE;
-        let mut byte = match iter.next() {
-            None => break,
-            Some(byte) => byte,
+    Some(Decoder::default().decode(input).ok()?.collect())
+}
+
+#[derive(Default)]
+pub(crate) struct Decoder {
+    insertions: Vec<(usize, char)>,
+}
+
+impl Decoder {
+    /// Split the input iterator and return a Vec with insertions of encoded characters
+    pub(crate) fn decode<'a>(&'a mut self, input: &'a str) -> Result<Decode<'a>, ()> {
+        self.insertions.clear();
+        // Handle "basic" (ASCII) code points.
+        // They are encoded as-is before the last delimiter, if any.
+        let (base, input) = match input.rfind(DELIMITER) {
+            None => ("", input),
+            Some(position) => (
+                &input[..position],
+                if position > 0 {
+                    &input[position + 1..]
+                } else {
+                    input
+                },
+            ),
         };
-        // Decode a generalized variable-length integer into delta,
-        // which gets added to i.
+
+        let base_len = base.len();
+        let mut length = base_len as u32;
+        let mut code_point = INITIAL_N;
+        let mut bias = INITIAL_BIAS;
+        let mut i = 0;
+        let mut iter = input.bytes();
         loop {
-            let digit = match byte {
-                byte @ b'0'..=b'9' => byte - b'0' + 26,
-                byte @ b'A'..=b'Z' => byte - b'A',
-                byte @ b'a'..=b'z' => byte - b'a',
-                _ => return None,
-            } as u32;
-            if digit > (u32::MAX - i) / weight {
-                return None; // Overflow
-            }
-            i += digit * weight;
-            let t = if k <= bias {
-                T_MIN
-            } else if k >= bias + T_MAX {
-                T_MAX
-            } else {
-                k - bias
+            let previous_i = i;
+            let mut weight = 1;
+            let mut k = BASE;
+            let mut byte = match iter.next() {
+                None => break,
+                Some(byte) => byte,
             };
-            if digit < t {
-                break;
+
+            // Decode a generalized variable-length integer into delta,
+            // which gets added to i.
+            loop {
+                let digit = match byte {
+                    byte @ b'0'..=b'9' => byte - b'0' + 26,
+                    byte @ b'A'..=b'Z' => byte - b'A',
+                    byte @ b'a'..=b'z' => byte - b'a',
+                    _ => return Err(()),
+                } as u32;
+                if digit > (u32::MAX - i) / weight {
+                    return Err(()); // Overflow
+                }
+                i += digit * weight;
+                let t = if k <= bias {
+                    T_MIN
+                } else if k >= bias + T_MAX {
+                    T_MAX
+                } else {
+                    k - bias
+                };
+                if digit < t {
+                    break;
+                }
+                if weight > u32::MAX / (BASE - t) {
+                    return Err(()); // Overflow
+                }
+                weight *= BASE - t;
+                k += BASE;
+                byte = match iter.next() {
+                    None => return Err(()), // End of input before the end of this delta
+                    Some(byte) => byte,
+                };
             }
-            if weight > u32::MAX / (BASE - t) {
-                return None; // Overflow
+
+            bias = adapt(i - previous_i, length + 1, previous_i == 0);
+            if i / (length + 1) > u32::MAX - code_point {
+                return Err(()); // Overflow
             }
-            weight *= BASE - t;
-            k += BASE;
-            byte = match iter.next() {
-                None => return None, // End of input before the end of this delta
-                Some(byte) => byte,
+
+            // i was supposed to wrap around from length+1 to 0,
+            // incrementing code_point each time.
+            code_point += i / (length + 1);
+            i %= length + 1;
+            let c = match char::from_u32(code_point) {
+                Some(c) => c,
+                None => return Err(()),
             };
+
+            // Move earlier insertions farther out in the string
+            for (idx, _) in &mut self.insertions {
+                if *idx >= i as usize {
+                    *idx += 1;
+                }
+            }
+            self.insertions.push((i as usize, c));
+            length += 1;
+            i += 1;
         }
-        let length = output.len() as u32;
-        bias = adapt(i - previous_i, length + 1, previous_i == 0);
-        if i / (length + 1) > u32::MAX - code_point {
-            return None; // Overflow
+
+        self.insertions.sort_by_key(|(i, _)| *i);
+        Ok(Decode {
+            base: base.chars(),
+            insertions: &self.insertions,
+            inserted: 0,
+            position: 0,
+            len: base_len + self.insertions.len(),
+        })
+    }
+}
+
+pub(crate) struct Decode<'a> {
+    base: std::str::Chars<'a>,
+    pub(crate) insertions: &'a [(usize, char)],
+    inserted: usize,
+    position: usize,
+    len: usize,
+}
+
+impl<'a> Iterator for Decode<'a> {
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.insertions.get(self.inserted) {
+                Some((pos, c)) if *pos == self.position => {
+                    self.inserted += 1;
+                    self.position += 1;
+                    return Some(*c);
+                }
+                _ => {}
+            }
+            if let Some(c) = self.base.next() {
+                self.position += 1;
+                return Some(c);
+            } else if self.inserted >= self.insertions.len() {
+                return None;
+            }
         }
-        // i was supposed to wrap around from length+1 to 0,
-        // incrementing code_point each time.
-        code_point += i / (length + 1);
-        i %= length + 1;
-        let c = match char::from_u32(code_point) {
-            Some(c) => c,
-            None => return None,
-        };
-        output.insert(i as usize, c);
-        i += 1;
     }
-    Some(output)
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let len = self.len - self.position;
+        (len, Some(len))
+    }
+}
+
+impl<'a> ExactSizeIterator for Decode<'a> {
+    fn len(&self) -> usize {
+        self.len - self.position
+    }
 }
 
 /// Convert an Unicode `str` to Punycode.