Skip to content

Commit

Permalink
pythonGH-107803: double linked list implementation for asyncio tasks (p…
Browse files Browse the repository at this point in the history
…ythonGH-107804)

* linked list

* add tail optmiization to linked list

* wip

* wip

* wip

* more fixes

* finally it works

* add tests

* remove weakreflist

* add some comments

* reduce code duplication in _asynciomodule.c

* address some review comments

* add invariants about the state of the linked list

* add better explanation

* clinic regen

* reorder branches for better branch prediction

* Update Modules/_asynciomodule.c

* Apply suggestions from code review

Co-authored-by: Itamar Oren <[email protected]>

* fix capturing of eager tasks

* add comment to task finalization

* fix tests and couple c implmentation to c task

improved linked-list logic and more comments

* fix test

---------

Co-authored-by: Itamar Oren <[email protected]>
  • Loading branch information
2 people authored and estyxx committed Jul 17, 2024
1 parent 653f7c4 commit c5a0305
Show file tree
Hide file tree
Showing 8 changed files with 375 additions and 97 deletions.
1 change: 1 addition & 0 deletions Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(displayhook)
STRUCT_FOR_ID(dklen)
STRUCT_FOR_ID(doc)
STRUCT_FOR_ID(done)
STRUCT_FOR_ID(dont_inherit)
STRUCT_FOR_ID(dst)
STRUCT_FOR_ID(dst_dir_fd)
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions Lib/asyncio/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1097,14 +1097,14 @@ def _unregister_eager_task(task):
_py_enter_task = _enter_task
_py_leave_task = _leave_task
_py_swap_current_task = _swap_current_task

_py_all_tasks = all_tasks

try:
from _asyncio import (_register_task, _register_eager_task,
_unregister_task, _unregister_eager_task,
_enter_task, _leave_task, _swap_current_task,
_scheduled_tasks, _eager_tasks, _current_tasks,
current_task)
current_task, all_tasks)
except ImportError:
pass
else:
Expand All @@ -1116,3 +1116,4 @@ def _unregister_eager_task(task):
_c_enter_task = _enter_task
_c_leave_task = _leave_task
_c_swap_current_task = _swap_current_task
_c_all_tasks = all_tasks
34 changes: 23 additions & 11 deletions Lib/test/test_asyncio/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ class BaseTaskTests:

Task = None
Future = None
all_tasks = None

def new_task(self, loop, coro, name='TestTask', context=None):
return self.__class__.Task(coro, loop=loop, name=name, context=context)
Expand Down Expand Up @@ -2267,7 +2268,7 @@ async def kill_me(loop):
coro = kill_me(self.loop)
task = asyncio.ensure_future(coro, loop=self.loop)

self.assertEqual(asyncio.all_tasks(loop=self.loop), {task})
self.assertEqual(self.all_tasks(loop=self.loop), {task})

asyncio.set_event_loop(None)

Expand All @@ -2282,7 +2283,7 @@ async def kill_me(loop):
# no more reference to kill_me() task: the task is destroyed by the GC
support.gc_collect()

self.assertEqual(asyncio.all_tasks(loop=self.loop), set())
self.assertEqual(self.all_tasks(loop=self.loop), set())

mock_handler.assert_called_with(self.loop, {
'message': 'Task was destroyed but it is pending!',
Expand Down Expand Up @@ -2431,7 +2432,7 @@ async def coro():
message = m_log.error.call_args[0][0]
self.assertIn('Task was destroyed but it is pending', message)

self.assertEqual(asyncio.all_tasks(self.loop), set())
self.assertEqual(self.all_tasks(self.loop), set())

def test_create_task_with_noncoroutine(self):
with self.assertRaisesRegex(TypeError,
Expand Down Expand Up @@ -2731,6 +2732,7 @@ async def func():
# Add patched Task & Future back to the test case
cls.Task = Task
cls.Future = Future
cls.all_tasks = tasks.all_tasks

# Add an extra unit-test
cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture
Expand Down Expand Up @@ -2804,6 +2806,7 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest,

Task = getattr(tasks, '_CTask', None)
Future = getattr(futures, '_CFuture', None)
all_tasks = getattr(tasks, '_c_all_tasks', None)

@support.refcount_test
def test_refleaks_in_task___init__(self):
Expand Down Expand Up @@ -2835,6 +2838,7 @@ class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase):

Task = getattr(tasks, '_CTask', None)
Future = getattr(futures, '_CFuture', None)
all_tasks = getattr(tasks, '_c_all_tasks', None)


@unittest.skipUnless(hasattr(tasks, '_CTask'),
Expand All @@ -2844,6 +2848,7 @@ class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase):

Task = getattr(tasks, '_CTask', None)
Future = futures._PyFuture
all_tasks = getattr(tasks, '_c_all_tasks', None)


@unittest.skipUnless(hasattr(futures, '_CFuture'),
Expand All @@ -2853,6 +2858,7 @@ class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase):

Future = getattr(futures, '_CFuture', None)
Task = tasks._PyTask
all_tasks = tasks._py_all_tasks


@unittest.skipUnless(hasattr(tasks, '_CTask'),
Expand All @@ -2861,6 +2867,7 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase):

Task = getattr(tasks, '_CTask', None)
Future = futures._PyFuture
all_tasks = getattr(tasks, '_c_all_tasks', None)


@unittest.skipUnless(hasattr(futures, '_CFuture'),
Expand All @@ -2869,13 +2876,15 @@ class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase):

Task = tasks._PyTask
Future = getattr(futures, '_CFuture', None)
all_tasks = staticmethod(tasks._py_all_tasks)


class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest,
test_utils.TestCase):

Task = tasks._PyTask
Future = futures._PyFuture
all_tasks = staticmethod(tasks._py_all_tasks)


@add_subclass_tests
Expand Down Expand Up @@ -2915,6 +2924,7 @@ class BaseTaskIntrospectionTests:
_unregister_task = None
_enter_task = None
_leave_task = None
all_tasks = None

def test__register_task_1(self):
class TaskLike:
Expand All @@ -2928,9 +2938,9 @@ def done(self):
task = TaskLike()
loop = mock.Mock()

self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())
self._register_task(task)
self.assertEqual(asyncio.all_tasks(loop), {task})
self.assertEqual(self.all_tasks(loop), {task})
self._unregister_task(task)

def test__register_task_2(self):
Expand All @@ -2944,9 +2954,9 @@ def done(self):
task = TaskLike()
loop = mock.Mock()

self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())
self._register_task(task)
self.assertEqual(asyncio.all_tasks(loop), {task})
self.assertEqual(self.all_tasks(loop), {task})
self._unregister_task(task)

def test__register_task_3(self):
Expand All @@ -2960,9 +2970,9 @@ def done(self):
task = TaskLike()
loop = mock.Mock()

self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())
self._register_task(task)
self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())
self._unregister_task(task)

def test__enter_task(self):
Expand Down Expand Up @@ -3013,20 +3023,21 @@ def test__unregister_task(self):
task.get_loop = lambda: loop
self._register_task(task)
self._unregister_task(task)
self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())

def test__unregister_task_not_registered(self):
task = mock.Mock()
loop = mock.Mock()
self._unregister_task(task)
self.assertEqual(asyncio.all_tasks(loop), set())
self.assertEqual(self.all_tasks(loop), set())


class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests):
_register_task = staticmethod(tasks._py_register_task)
_unregister_task = staticmethod(tasks._py_unregister_task)
_enter_task = staticmethod(tasks._py_enter_task)
_leave_task = staticmethod(tasks._py_leave_task)
all_tasks = staticmethod(tasks._py_all_tasks)


@unittest.skipUnless(hasattr(tasks, '_c_register_task'),
Expand All @@ -3037,6 +3048,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests):
_unregister_task = staticmethod(tasks._c_unregister_task)
_enter_task = staticmethod(tasks._c_enter_task)
_leave_task = staticmethod(tasks._c_leave_task)
all_tasks = staticmethod(tasks._c_all_tasks)
else:
_register_task = _unregister_task = _enter_task = _leave_task = None

Expand Down
Loading

0 comments on commit c5a0305

Please sign in to comment.