Merge pull request #1445 from Martiusweb/client_idna

Fix support of IDNA in Client

CHANGES.rst

@@ -32,7 +32,7 @@ CHANGES
 
 - Fix bugs with client proxy target path and proxy host with port #1413
 
--
+- Fix bugs related to the use of unicode hostnames #1444
 
 -
 

aiohttp/client_reqrep.py

@@ -165,7 +165,7 @@ def update_auto_headers(self, skip_auto_headers):
 
         # add host
         if hdrs.HOST not in used_headers:
-            netloc = self.url.host
+            netloc = self.url.raw_host
             if not self.url.is_default_port():
                 netloc += ':' + str(self.url.port)
             self.headers[hdrs.HOST] = netloc
@@ -437,7 +437,7 @@ def send(self, writer, reader):
         # - not CONNECT proxy must send absolute form URI
         # - most common is origin form URI
         if self.method == hdrs.METH_CONNECT:
-            path = '{}:{}'.format(self.url.host, self.url.port)
+            path = '{}:{}'.format(self.url.raw_host, self.url.port)
         elif self.proxy and not self.ssl:
             path = str(self.url)
         else:

aiohttp/cookiejar.py

@@ -91,7 +91,7 @@ def _expire_cookie(self, when, domain, name):
 
     def update_cookies(self, cookies, response_url=URL()):
         """Update cookies."""
-        hostname = response_url.host
+        hostname = response_url.raw_host
 
         if not self._unsafe and is_ip_address(hostname):
             # Don't accept cookies from IPs
@@ -168,7 +168,7 @@ def filter_cookies(self, request_url=URL()):
         """Returns this jar's cookies filtered by their attributes."""
         self._do_expiration()
         filtered = SimpleCookie()
-        hostname = request_url.host or ""
+        hostname = request_url.raw_host or ""
         is_not_secure = request_url.scheme not in ("https", "wss")
 
         for cookie in self:

docs/spelling_wordlist.txt

@@ -203,6 +203,7 @@ tp
 tuples
 UI
 un
+unicode
 unittest
 Unittest
 unix

tests/test_client_request.py

@@ -143,6 +143,16 @@ def test_host_header_host_with_nondefault_port(make_request):
     assert req.headers['HOST'] == 'python.org:99'
 
 
+def test_host_header_host_idna_encode(make_request):
+    req = make_request('get', 'http://xn--9caa.com')
+    assert req.headers['HOST'] == 'xn--9caa.com'
+
+
+def test_host_header_host_unicode(make_request):
+    req = make_request('get', 'http://éé.com')
+    assert req.headers['HOST'] == 'xn--9caa.com'
+
+
 def test_host_header_explicit_host(make_request):
     req = make_request('get', 'http://python.org/',
                        headers={'host': 'example.com'})

tests/test_cookiejar.py

@@ -162,6 +162,32 @@ def test_save_load(loop, cookies_to_send, cookies_to_receive):
     assert jar_test == cookies_to_receive
 
 
+def test_update_cookie_with_unicode_domain(loop):
+    cookies = (
+        "idna-domain-first=first; Domain=xn--9caa.com; Path=/;",
+        "idna-domain-second=second; Domain=xn--9caa.com; Path=/;",
+    )
+
+    jar = CookieJar(loop=loop)
+    jar.update_cookies(SimpleCookie(cookies[0]), URL("http://éé.com/"))
+    jar.update_cookies(SimpleCookie(cookies[1]), URL("http://xn--9caa.com/"))
+
+    jar_test = SimpleCookie()
+    for cookie in jar:
+        jar_test[cookie.key] = cookie
+
+    assert jar_test == SimpleCookie(" ".join(cookies))
+
+
+def test_filter_cookie_with_unicode_domain(loop):
+    jar = CookieJar(loop=loop)
+    jar.update_cookies(SimpleCookie(
+        "idna-domain-first=first; Domain=xn--9caa.com; Path=/; "
+    ))
+    assert len(jar.filter_cookies(URL("http://éé.com"))) == 1
+    assert len(jar.filter_cookies(URL("http://xn--9caa.com"))) == 1
+
+
 def test_ctor_ith_default_loop(loop):
     asyncio.set_event_loop(loop)
     jar = CookieJar()

tests/test_proxy.py

@@ -62,6 +62,21 @@ def test_proxy_auth(self):
             "proxy_auth must be None or BasicAuth() tuple",
         )
 
+    @mock.patch('aiohttp.Request')
+    def test_connect_request_with_unicode_host(self, Request_mock):
+        loop = mock.Mock()
+        request = ClientRequest("CONNECT", URL("http://éé.com/"),
+                                loop=loop)
+
+        request.response_class = mock.Mock()
+        request.write_bytes = mock.Mock()
+        request.write_bytes.return_value = asyncio.Future(loop=loop)
+        request.write_bytes.return_value.set_result(None)
+        request.send(mock.Mock(), mock.Mock())
+
+        Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80",
+                                        mock.ANY)
+
     def test_proxy_connection_error(self):
         connector = aiohttp.TCPConnector(loop=self.loop)
         connector._resolve_host = make_mocked_coro(