Auto merge of servo#248 - Manishearth:percent-fragment, r=SimonSapin

Percent-encode fragments while parsing

Based on whatwg/url#169

Fixes servo#246

cc @valenting

r? @SimonSapin

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/rust-url/248)
<!-- Reviewable:end -->

src/parser.rs

@@ -1083,12 +1083,13 @@ impl<'a> Parser<'a> {
     }
 
     pub fn parse_fragment(&mut self, mut input: Input) {
-        while let Some(c) = input.next() {
+        while let Some((c, utf8_c)) = input.next_utf8() {
             if c ==  '\0' {
                 self.syntax_violation("NULL characters are ignored in URL fragment identifiers")
             } else {
                 self.check_url_code_point(c, &input);
-                self.serialization.push(c);  // No percent-encoding here.
+                self.serialization.extend(utf8_percent_encode(utf8_c,
+                                                              SIMPLE_ENCODE_SET));
             }
         }
     }

tests/setters_tests.json

@@ -1127,21 +1127,21 @@
             }
         },
         {
-            "comment": "No percent-encoding at all (!); nuls, tabs, and newlines are removed. Leading or training C0 controls and space are removed.",
+            "comment": "Simple percent-encoding; nuls, tabs, and newlines are removed",
             "href": "a:/",
             "new_value": "\u0000\u0001\t\n\r\u001f !\u0000\u0001\t\n\r\u001f !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~\u007f\u0080\u0081Éé",
             "expected": {
-                "href": "a:/#!\u0001\u001f !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~\u007f\u0080\u0081Éé",
-                "hash": "#!\u0001\u001f !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~\u007f\u0080\u0081Éé"
+                "href": "a:/#!%01%1F !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~%7F%C2%80%C2%81%C3%89%C3%A9",
+                "hash": "#!%01%1F !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~%7F%C2%80%C2%81%C3%89%C3%A9"
             }
         },
         {
             "comment": "Bytes already percent-encoded are left as-is",
             "href": "http://example.net",
             "new_value": "%c3%89té",
             "expected": {
-                "href": "http://example.net/#%c3%89té",
-                "hash": "#%c3%89té"
+                "href": "http://example.net/#%c3%89t%C3%A9",
+                "hash": "#%c3%89t%C3%A9"
             }
         }
     ]

tests/urltestdata.json

@@ -1292,7 +1292,7 @@
   {
     "input": "#β",
     "base": "http://example.org/foo/bar",
-    "href": "http://example.org/foo/bar#β",
+    "href": "http://example.org/foo/bar#%CE%B2",
     "origin": "http://example.org",
     "protocol": "http:",
     "username": "",
@@ -1302,7 +1302,7 @@
     "port": "",
     "pathname": "/foo/bar",
     "search": "",
-    "hash": "#β"
+    "hash": "#%CE%B2"
   },
   {
     "input": "data:text/html,test#test",
@@ -2161,7 +2161,7 @@
   {
     "input": "http://www.google.com/foo?bar=baz# »",
     "base": "about:blank",
-    "href": "http://www.google.com/foo?bar=baz# »",
+    "href": "http://www.google.com/foo?bar=baz# %C2%BB",
     "origin": "http://www.google.com",
     "protocol": "http:",
     "username": "",
@@ -2171,12 +2171,12 @@
     "port": "",
     "pathname": "/foo",
     "search": "?bar=baz",
-    "hash": "# »"
+    "hash": "# %C2%BB"
   },
   {
     "input": "data:test# »",
     "base": "about:blank",
-    "href": "data:test# »",
+    "href": "data:test# %C2%BB",
     "origin": "null",
     "protocol": "data:",
     "username": "",
@@ -2186,7 +2186,7 @@
     "port": "",
     "pathname": "test",
     "search": "",
-    "hash": "# »"
+    "hash": "# %C2%BB"
   },
   {
     "input": "http://[www.google.com]/",
@@ -4356,5 +4356,22 @@
     "search": "",
     "searchParams": "",
     "hash": ""
+  },
+  "# Percent encoding of fragments",
+  {
+    "input": "http://foo.bar/baz?qux#foo\bbar",
+    "base": "about:blank",
+    "href": "http://foo.bar/baz?qux#foo%08bar",
+    "origin": "http://foo.bar",
+    "protocol": "http:",
+    "username": "",
+    "password": "",
+    "host": "foo.bar",
+    "hostname": "foo.bar",
+    "port": "",
+    "pathname": "/baz",
+    "search": "?qux",
+    "searchParams": "",
+    "hash": "#foo%08bar"
   }
 ]