[3.6] Compat python 3.8 (#4056). (#4073)

(cherry picked from commit 6dedbca)

Co-authored-by: 秋葉 <[email protected]>

CHANGES/4056.feature

@@ -0,0 +1 @@
+Compat Python 3.8.

aiohttp/base_protocol.py

@@ -6,7 +6,7 @@
 
 class BaseProtocol(asyncio.Protocol):
     __slots__ = ('_loop', '_paused', '_drain_waiter',
-                 '_connection_lost', 'transport')
+                 '_connection_lost', '_reading_paused', 'transport')
 
     def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
         self._loop = loop  # type: asyncio.AbstractEventLoop

aiohttp/helpers.py

@@ -52,6 +52,7 @@
 
 PY_36 = sys.version_info >= (3, 6)
 PY_37 = sys.version_info >= (3, 7)
+PY_38 = sys.version_info >= (3, 8)
 
 if not PY_37:
     import idna_ssl

aiohttp/streams.py

@@ -13,7 +13,6 @@
 except ImportError:
     from typing_extensions import Deque  # noqa
 
-
 __all__ = (
     'EMPTY_PAYLOAD', 'EofStream', 'StreamReader', 'DataQueue',
     'FlowControlDataQueue')
@@ -421,7 +420,7 @@ async def readexactly(self, n: int) -> bytes:
             block = await self.read(n)
             if not block:
                 partial = b''.join(blocks)
-                raise asyncio.streams.IncompleteReadError(
+                raise asyncio.IncompleteReadError(
                     partial, len(partial) + n)
             blocks.append(block)
             n -= len(block)
@@ -526,7 +525,7 @@ async def readchunk(self) -> Tuple[bytes, bool]:
         return (b'', True)
 
     async def readexactly(self, n: int) -> bytes:
-        raise asyncio.streams.IncompleteReadError(b'', n)
+        raise asyncio.IncompleteReadError(b'', n)
 
     def read_nowait(self) -> bytes:
         return b''

aiohttp/test_utils.py

@@ -4,6 +4,7 @@
 import contextlib
 import functools
 import gc
+import inspect
 import socket
 import sys
 import unittest
@@ -659,10 +660,11 @@ def make_mocked_request(method: str, path: str,
 def make_mocked_coro(return_value: Any=sentinel,
                      raise_exception: Any=sentinel) -> Any:
     """Creates a coroutine mock."""
-    @asyncio.coroutine
-    def mock_coro(*args: Any, **kwargs: Any) -> Any:
+    async def mock_coro(*args: Any, **kwargs: Any) -> Any:
         if raise_exception is not sentinel:
             raise raise_exception
-        return return_value
+        if not inspect.isawaitable(return_value):
+            return return_value
+        await return_value
 
     return mock.Mock(wraps=mock_coro)

tests/test_client_functional.py

@@ -2420,30 +2420,28 @@ async def handler(request):
         await aiohttp.request('GET', server.make_url('/'))
 
 
-@asyncio.coroutine
-def test_yield_from_in_session_request(aiohttp_client) -> None:
+async def test_yield_from_in_session_request(aiohttp_client) -> None:
     # a test for backward compatibility with yield from syntax
     async def handler(request):
         return web.Response()
 
     app = web.Application()
     app.router.add_get('/', handler)
 
-    client = yield from aiohttp_client(app)
-    resp = yield from client.get('/')
+    client = await aiohttp_client(app)
+    resp = await client.get('/')
     assert resp.status == 200
 
 
-@asyncio.coroutine
-def test_close_context_manager(aiohttp_client) -> None:
+async def test_close_context_manager(aiohttp_client) -> None:
     # a test for backward compatibility with yield from syntax
     async def handler(request):
         return web.Response()
 
     app = web.Application()
     app.router.add_get('/', handler)
 
-    client = yield from aiohttp_client(app)
+    client = await aiohttp_client(app)
     ctx = client.get('/')
     ctx.close()
     assert not ctx._coro.cr_running

tests/test_client_request.py

@@ -952,7 +952,7 @@ async def gen():
     assert req.headers['TRANSFER-ENCODING'] == 'chunked'
 
     async def throw_exc():
-        await asyncio.sleep(0.01, loop=loop)
+        await asyncio.sleep(0.01)
         fut.set_exception(ValueError)
 
     loop.create_task(throw_exc())
@@ -1004,7 +1004,7 @@ async def gen():
     inner_exc = ValueError()
 
     async def throw_exc():
-        await asyncio.sleep(0.01, loop=loop)
+        await asyncio.sleep(0.01)
         fut.set_exception(inner_exc)
 
     loop.create_task(throw_exc())
@@ -1062,7 +1062,7 @@ async def gen():
     assert req.chunked
 
     async def coro():
-        await asyncio.sleep(0.0001, loop=loop)
+        await asyncio.sleep(0.0001)
         req._continue.set_result(1)
 
     loop.create_task(coro())
@@ -1108,7 +1108,7 @@ async def test_data_continue(loop, buf, conn) -> None:
         expect100=True, loop=loop)
 
     async def coro():
-        await asyncio.sleep(0.0001, loop=loop)
+        await asyncio.sleep(0.0001)
         req._continue.set_result(1)
 
     loop.create_task(coro())

tests/test_client_session.py

@@ -17,6 +17,7 @@
 from aiohttp.client_reqrep import ClientRequest
 from aiohttp.connector import BaseConnector, TCPConnector
 from aiohttp.helpers import DEBUG, PY_36
+from aiohttp.test_utils import make_mocked_coro
 
 
 @pytest.fixture
@@ -164,7 +165,11 @@ async def test_merge_headers_with_list_of_tuples_duplicated_names(
 
 
 def test_http_GET(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    # Python 3.8 will auto use mock.AsyncMock, it has different behavior
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.get("http://test.example.com",
                     params={"x": 1},
                     **params)
@@ -177,7 +182,10 @@ def test_http_GET(session, params) -> None:
 
 
 def test_http_OPTIONS(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.options("http://opt.example.com",
                         params={"x": 2},
                         **params)
@@ -190,7 +198,10 @@ def test_http_OPTIONS(session, params) -> None:
 
 
 def test_http_HEAD(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.head("http://head.example.com",
                      params={"x": 2},
                      **params)
@@ -203,7 +214,10 @@ def test_http_HEAD(session, params) -> None:
 
 
 def test_http_POST(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.post("http://post.example.com",
                      params={"x": 2},
                      data="Some_data",
@@ -217,7 +231,10 @@ def test_http_POST(session, params) -> None:
 
 
 def test_http_PUT(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.put("http://put.example.com",
                     params={"x": 2},
                     data="Some_data",
@@ -231,7 +248,10 @@ def test_http_PUT(session, params) -> None:
 
 
 def test_http_PATCH(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.patch("http://patch.example.com",
                       params={"x": 2},
                       data="Some_data",
@@ -245,7 +265,10 @@ def test_http_PATCH(session, params) -> None:
 
 
 def test_http_DELETE(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.delete("http://delete.example.com",
                        params={"x": 2},
                        **params)
@@ -506,7 +529,10 @@ async def test_session_loop(loop) -> None:
 
 
 def test_proxy_str(session, params) -> None:
-    with mock.patch("aiohttp.client.ClientSession._request") as patched:
+    with mock.patch(
+        "aiohttp.client.ClientSession._request",
+        new_callable=mock.MagicMock
+    ) as patched:
         session.get("http://test.example.com",
                     proxy='http://proxy.com',
                     **params)
@@ -530,9 +556,9 @@ async def handler(request):
     body = 'This is request body'
     gathered_req_body = BytesIO()
     gathered_res_body = BytesIO()
-    on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
-    on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
-    on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
+    on_request_start = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
+    on_request_redirect = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
+    on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
 
     async def on_request_chunk_sent(session, context, params):
         gathered_req_body.write(params.chunk)
@@ -583,9 +609,9 @@ async def on_response_chunk_received(session, context, params):
 
 
 async def test_request_tracing_exception(loop) -> None:
-    on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
+    on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
     on_request_exception = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig()

tests/test_connector.py

@@ -751,24 +751,24 @@ async def test_tcp_connector_dns_throttle_requests_cancelled_when_close(
         await asyncio.sleep(0, loop=loop)
         conn.close()
 
-        with pytest.raises(asyncio.futures.CancelledError):
+        with pytest.raises(asyncio.CancelledError):
             await f
 
 
 async def test_tcp_connector_dns_tracing(loop, dns_response) -> None:
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_dns_resolvehost_start = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_dns_resolvehost_end = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_dns_cache_hit = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_dns_cache_miss = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(
@@ -835,10 +835,10 @@ async def test_tcp_connector_dns_tracing_cache_disabled(loop,
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_dns_resolvehost_start = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_dns_resolvehost_end = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(
@@ -909,10 +909,10 @@ async def test_tcp_connector_dns_tracing_throttle_requests(
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_dns_cache_hit = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_dns_cache_miss = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(
@@ -1046,10 +1046,10 @@ async def test_connect_tracing(loop) -> None:
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_connection_create_start = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_connection_create_end = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(
@@ -1436,10 +1436,10 @@ async def test_connect_queued_operation_tracing(loop, key) -> None:
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_connection_queued_start = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
     on_connection_queued_end = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(
@@ -1496,7 +1496,7 @@ async def test_connect_reuseconn_tracing(loop, key) -> None:
     session = mock.Mock()
     trace_config_ctx = mock.Mock()
     on_connection_reuseconn = mock.Mock(
-        side_effect=asyncio.coroutine(mock.Mock())
+        side_effect=make_mocked_coro(mock.Mock())
     )
 
     trace_config = aiohttp.TraceConfig(

tests/test_helpers.py

@@ -334,7 +334,7 @@ def test_timer_context_no_task(loop) -> None:
 async def test_weakref_handle(loop) -> None:
     cb = mock.Mock()
     helpers.weakref_handle(cb, 'test', 0.01, loop, False)
-    await asyncio.sleep(0.1, loop=loop)
+    await asyncio.sleep(0.1)
     assert cb.test.called
 
 
@@ -343,7 +343,7 @@ async def test_weakref_handle_weak(loop) -> None:
     helpers.weakref_handle(cb, 'test', 0.01, loop, False)
     del cb
     gc.collect()
-    await asyncio.sleep(0.1, loop=loop)
+    await asyncio.sleep(0.1)
 
 
 def test_ceil_call_later() -> None:

tests/test_proxy.py

@@ -224,8 +224,7 @@ async def make_conn():
 
         seq = 0
 
-        @asyncio.coroutine
-        def create_connection(*args, **kwargs):
+        async def create_connection(*args, **kwargs):
             nonlocal seq
             seq += 1
 
@@ -275,8 +274,7 @@ async def make_conn():
 
         seq = 0
 
-        @asyncio.coroutine
-        def create_connection(*args, **kwargs):
+        async def create_connection(*args, **kwargs):
             nonlocal seq
             seq += 1