From 1cecbdb8824f552ddc8399967aff667f0150beea Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Fri, 30 Dec 2022 01:45:11 -0800 Subject: [PATCH 01/22] GH-84559: Deprecate fork being the multiprocessing default. This starts the process. Users who don't specify their own start method and use the default on platforms where it is 'fork' will see a DeprecationWarning upon multiprocessing.Pool() construction or upon multiprocessing.Process.start(). --- Lib/multiprocessing/context.py | 34 +++++++++++- Lib/multiprocessing/pool.py | 13 +++++ Lib/test/test_multiprocessing_defaults.py | 63 +++++++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_multiprocessing_defaults.py diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index b1960ea296fe20..5f8da431dd187e 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -280,6 +280,32 @@ def _Popen(process_obj): from .popen_fork import Popen return Popen(process_obj) + class _DeprecatedForkProcess(ForkProcess): + @classmethod + def _warn(cls, stacklevel): + import warnings + warnings.warn( + "Use of the multiprocessing 'fork' start method by default is " + "deprecated. " + "The default will change in Python >= 3.14, per GH-84559. " + "Please use an explicit multiprocessing.get_context or " + "multiprocessing.set_start_method API call to specify your " + "application's start method if you want to use 'fork'." + " The 'spawn' and 'forkserver' start methods are safer " + "depending on the platform and application.", + category=DeprecationWarning, + stacklevel=stacklevel, + ) + + # Blame the process.start() method call. + _ORIG_WARNING_STACKLEVEL = 5 + _warning_stacklevel = _ORIG_WARNING_STACKLEVEL + + @classmethod + def _Popen(cls, process_obj): + cls._warn(stacklevel=cls._warning_stacklevel) + return super()._Popen(process_obj) + class SpawnProcess(process.BaseProcess): _start_method = 'spawn' @staticmethod @@ -303,6 +329,9 @@ class ForkContext(BaseContext): _name = 'fork' Process = ForkProcess + class _DefaultForkContext(ForkContext): + Process = _DeprecatedForkProcess + class SpawnContext(BaseContext): _name = 'spawn' Process = SpawnProcess @@ -318,13 +347,16 @@ def _check_available(self): 'fork': ForkContext(), 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), + # Remove None and _DefaultForkContext() when changing the default + # in 3.14 for https://github.com/python/cpython/issues/84559. + None: _DefaultForkContext(), } if sys.platform == 'darwin': # bpo-33725: running arbitrary code after fork() is no longer reliable # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. _default_context = DefaultContext(_concrete_contexts['spawn']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts[None]) else: diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 4f5d88cb975cb7..3ea95c5ead6ed5 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -326,7 +326,20 @@ def _repopulate_pool_static(ctx, Process, processes, pool, inqueue, wrap_exception)) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True + if _warn := getattr(w, '_warn', None): # GH-84559 + # Hacky, but BaseProcess._Popen does everything via the class + # using static or class methods with no instance. The way + # multiprocessing APIs get used, a concurrency mess here + # with a Pool() being constructed at the same time as other + # threads are calling Process.start() is highly unlikely. + # If so: The oddball application gets the warning attributed + # to an unusual spot in the stack on occasion. + process_class = w.__class__ + process_class._warning_stacklevel = 8 # blame mp.Pool() w.start() + if _warn: + process_class._warning_stacklevel = ( + process_class._ORIG_WARNING_STACKLEVEL) pool.append(w) util.debug('added worker') diff --git a/Lib/test/test_multiprocessing_defaults.py b/Lib/test/test_multiprocessing_defaults.py new file mode 100644 index 00000000000000..f03091b3f6984d --- /dev/null +++ b/Lib/test/test_multiprocessing_defaults.py @@ -0,0 +1,63 @@ +"""Test default behavior of multiprocessing.""" + +from inspect import currentframe, getframeinfo +import multiprocessing +import sys +from test.support import threading_helper +import unittest +import warnings + + +def do_nothing(): + pass + + +# Process has the same API as Thread so this helper works. +join_process = threading_helper.join_thread + + +class DefaultWarningsTest(unittest.TestCase): + + @unittest.skipIf(sys.platform in ('win32', 'darwin'), + 'The default is not "fork" on Windows or macOS.') + def setUp(self): + self.assertEqual(multiprocessing.get_start_method(), 'fork') + self.assertIsInstance(multiprocessing.get_context(), + multiprocessing.context._DefaultForkContext) + + def test_default_fork_start_method_warning_process(self): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter('always') + process = multiprocessing.Process(target=do_nothing) + process.start() # warning should point here. + join_process(process) + self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + self.assertIn(__file__, ws[0].filename) + self.assertEqual(getframeinfo(currentframe()).lineno-4, ws[0].lineno) + self.assertIn("'fork'", str(ws[0].message)) + self.assertIn("start_method API", str(ws[0].message)) + + def test_default_fork_start_method_warning_pool(self): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter('always') + pool = multiprocessing.Pool(1) # warning should point here. + pool.terminate() + pool.join() + self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + self.assertIn(__file__, ws[0].filename) + self.assertEqual(getframeinfo(currentframe()).lineno-5, ws[0].lineno) + self.assertIn("'fork'", str(ws[0].message)) + self.assertIn("start_method API", str(ws[0].message)) + + def test_no_warning_when_using_explicit_fork_mp_context(self): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter('always') # Enable all warnings. + fork_mp = multiprocessing.get_context('fork') + pool = fork_mp.Pool(1) + pool.terminate() + pool.join() + self.assertEqual(len(ws), 0, msg=[str(x) for x in ws]) + + +if __name__ == '__main__': + unittest.main() From df83fba6c7c3727664dda2e644decd9e423f3b50 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 04:10:04 +0000 Subject: [PATCH 02/22] address CI test failures? --- Lib/multiprocessing/context.py | 5 ++++- Lib/test/_test_multiprocessing.py | 21 +++++++++++++-------- Lib/test/test_multiprocessing_defaults.py | 18 ++++++++++++------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 5f8da431dd187e..5349ec166e9800 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -23,6 +23,9 @@ class TimeoutError(ProcessError): class AuthenticationError(ProcessError): pass +class DefaultsDeprecationWarning(DeprecationWarning): + pass + # # Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py # @@ -293,7 +296,7 @@ def _warn(cls, stacklevel): "application's start method if you want to use 'fork'." " The 'spawn' and 'forkserver' start methods are safer " "depending on the platform and application.", - category=DeprecationWarning, + category=DefaultsDeprecationWarning, stacklevel=stacklevel, ) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 2fa75eb4d11311..e4a60a4d674607 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4098,9 +4098,10 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. - cmd = '''if 1: + cmd = f'''if 1: from multiprocessing.managers import SharedMemoryManager - + from multiprocessing import set_start_method + set_start_method({multiprocessing.get_start_method()!r}) smm = SharedMemoryManager() smm.start() @@ -4967,11 +4968,13 @@ def run_in_grandchild(cls, conn): conn.send(tuple(sys.flags)) @classmethod - def run_in_child(cls): + def run_in_child(cls, start_method): import json - r, w = multiprocessing.Pipe(duplex=False) - p = multiprocessing.Process(target=cls.run_in_grandchild, args=(w,)) - p.start() + mp = multiprocessing.get_context(start_method) + r, w = mp.Pipe(duplex=False) + p = mp.Process(target=cls.run_in_grandchild, args=(w,)) + with warnings.catch_warnings(category=DeprecationWarning): + p.start() grandchild_flags = r.recv() p.join() r.close() @@ -4982,8 +4985,10 @@ def run_in_child(cls): def test_flags(self): import json # start child process using unusual flags - prog = ('from test._test_multiprocessing import TestFlags; ' + - 'TestFlags.run_in_child()') + prog = ( + 'from test._test_multiprocessing import TestFlags; ' + f'TestFlags.run_in_child({multiprocessing.get_start_method()!r})' + ) data = subprocess.check_output( [sys.executable, '-E', '-S', '-O', '-c', prog]) child_flags, grandchild_flags = json.loads(data.decode('ascii')) diff --git a/Lib/test/test_multiprocessing_defaults.py b/Lib/test/test_multiprocessing_defaults.py index f03091b3f6984d..d63578aa147784 100644 --- a/Lib/test/test_multiprocessing_defaults.py +++ b/Lib/test/test_multiprocessing_defaults.py @@ -2,6 +2,7 @@ from inspect import currentframe, getframeinfo import multiprocessing +from multiprocessing.context import DefaultsDeprecationWarning import sys from test.support import threading_helper import unittest @@ -27,31 +28,36 @@ def setUp(self): def test_default_fork_start_method_warning_process(self): with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter('always') + warnings.simplefilter('ignore') + warnings.filterwarnings('always', category=DefaultsDeprecationWarning) process = multiprocessing.Process(target=do_nothing) process.start() # warning should point here. join_process(process) - self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + self.assertIsInstance(ws[0].message, DefaultsDeprecationWarning) self.assertIn(__file__, ws[0].filename) self.assertEqual(getframeinfo(currentframe()).lineno-4, ws[0].lineno) self.assertIn("'fork'", str(ws[0].message)) self.assertIn("start_method API", str(ws[0].message)) + self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) def test_default_fork_start_method_warning_pool(self): with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter('always') + warnings.simplefilter('ignore') + warnings.filterwarnings('always', category=DefaultsDeprecationWarning) pool = multiprocessing.Pool(1) # warning should point here. pool.terminate() pool.join() - self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + self.assertIsInstance(ws[0].message, DefaultsDeprecationWarning) self.assertIn(__file__, ws[0].filename) self.assertEqual(getframeinfo(currentframe()).lineno-5, ws[0].lineno) self.assertIn("'fork'", str(ws[0].message)) self.assertIn("start_method API", str(ws[0].message)) + self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) - def test_no_warning_when_using_explicit_fork_mp_context(self): + def test_no_mp_warning_when_using_explicit_fork_context(self): with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter('always') # Enable all warnings. + warnings.simplefilter('ignore') + warnings.filterwarnings('always', category=DefaultsDeprecationWarning) fork_mp = multiprocessing.get_context('fork') pool = fork_mp.Pool(1) pool.terminate() From ec1e868554635f381c830a780c3d2a06de52c17a Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 04:45:57 +0000 Subject: [PATCH 03/22] the new warning needs no pickle name mapping --- Lib/test/test_pickle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 44fdca7a6b1688..80e7a4d23a4ba8 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -533,6 +533,8 @@ def test_exceptions(self): def test_multiprocessing_exceptions(self): module = import_helper.import_module('multiprocessing.context') for name, exc in get_exceptions(module): + if issubclass(exc, Warning): + continue with self.subTest(name): self.assertEqual(reverse_mapping('multiprocessing.context', name), ('multiprocessing', name)) From a1064dee146334e77641a39fa6cb3b868badd15f Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 05:20:11 +0000 Subject: [PATCH 04/22] avoid DeprecationWarnings vs -Werror --- Lib/test/_test_venv_multiprocessing.py | 1 + Lib/test/test_asyncio/test_events.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py index af72e915ba52bb..044a0c6cd3f5ca 100644 --- a/Lib/test/_test_venv_multiprocessing.py +++ b/Lib/test/_test_venv_multiprocessing.py @@ -30,6 +30,7 @@ def test_func(): def main(): + multiprocessing.set_start_method('spawn') test_pool = multiprocessing.Process(target=test_func) test_pool.start() test_pool.join() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index e7771edd2e4afb..1a21e80bbe674a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -4,6 +4,7 @@ import concurrent.futures import functools import io +import multiprocessing import os import platform import re @@ -2696,7 +2697,13 @@ def test_get_event_loop_new_process(self): support.skip_if_broken_multiprocessing_synchronize() async def main(): - pool = concurrent.futures.ProcessPoolExecutor() + if multiprocessing.get_start_method() == 'fork': + # Avoid 'fork' DeprecationWarning. + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + pool = concurrent.futures.ProcessPoolExecutor( + mp_context=mp_context) result = await self.loop.run_in_executor( pool, _test_get_event_loop_new_process__sub_proc) pool.shutdown() From 223b748ac3792e2ef0781ebe23ab0a6189f37b23 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 05:42:49 +0000 Subject: [PATCH 05/22] prefer forkserver over fork for concurrent.futures --- Lib/compileall.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index a388931fb5a99d..d394156cedc4e7 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -97,9 +97,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True if workers != 1 and ProcessPoolExecutor is not None: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + mp_context=mp_context) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, From fc64626d033dee7b24b5484f678fd0d6a9161a9f Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 05:43:17 +0000 Subject: [PATCH 06/22] explicitly use spawn for simple process launchers. --- Lib/test/test_fcntl.py | 8 +++++--- Lib/test/test_logging.py | 5 +++-- Lib/test/test_re.py | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index fc8c39365f12b7..113c7802821dd4 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -1,11 +1,11 @@ """Test program for the fcntl C module. """ +import multiprocessing import platform import os import struct import sys import unittest -from multiprocessing import Process from test.support import verbose, cpython_only from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -160,7 +160,8 @@ def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_EX | fcntl.LOCK_NB fcntl.lockf(self.f, cmd) - p = Process(target=try_lockf_on_other_process_fail, args=(TESTFN, cmd)) + mp = multiprocessing.get_context('spawn') + p = mp.Process(target=try_lockf_on_other_process_fail, args=(TESTFN, cmd)) p.start() p.join() fcntl.lockf(self.f, fcntl.LOCK_UN) @@ -171,7 +172,8 @@ def test_lockf_share(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_SH | fcntl.LOCK_NB fcntl.lockf(self.f, cmd) - p = Process(target=try_lockf_on_other_process, args=(TESTFN, cmd)) + mp = multiprocessing.get_context('spawn') + p = mp.Process(target=try_lockf_on_other_process, args=(TESTFN, cmd)) p.start() p.join() fcntl.lockf(self.f, fcntl.LOCK_UN) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 072056d3722106..8a12d570f26f13 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4759,8 +4759,9 @@ def test_multiprocessing(self): # In other processes, processName is correct when multiprocessing in imported, # but it is (incorrectly) defaulted to 'MainProcess' otherwise (bpo-38762). import multiprocessing - parent_conn, child_conn = multiprocessing.Pipe() - p = multiprocessing.Process( + mp = multiprocessing.get_context('spawn') + parent_conn, child_conn = mp.Pipe() + p = mp.Process( target=self._extract_logrecord_process_name, args=(2, LOG_MULTI_PROCESSING, child_conn,) ) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 11628a236ade9a..eacb1a7c82a54d 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2431,7 +2431,8 @@ def test_regression_gh94675(self): input_js = '''a(function() { /////////////////////////////////////////////////////////////////// });''' - p = multiprocessing.Process(target=pattern.sub, args=('', input_js)) + mp = multiprocessing.get_context('spawn') + p = mp.Process(target=pattern.sub, args=('', input_js)) p.start() p.join(SHORT_TIMEOUT) try: From e45fab669e5fcf8ac797c640224eb3768381a88c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 07:19:27 +0000 Subject: [PATCH 07/22] warn from concurrent.futures, reword things. --- Lib/concurrent/futures/process.py | 19 ++++++++++++++++--- Lib/multiprocessing/context.py | 22 ++++++++++------------ Lib/test/test_multiprocessing_defaults.py | 16 ++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 7e2f5fa30e8264..e125d91ea0e851 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -616,9 +616,9 @@ def __init__(self, max_workers=None, mp_context=None, max_workers: The maximum number of processes that can be used to execute the given calls. If None or not given then as many worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers. This - object should provide SimpleQueue, Queue and Process. Useful - to allow specific multiprocessing start methods. + mp_context: A multiprocessing context to launch the workers created + using the multiprocessing.get_context('start method') API. This + object should provide SimpleQueue, Queue and Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. max_tasks_per_child: The maximum number of tasks a worker process @@ -650,6 +650,19 @@ def __init__(self, max_workers=None, mp_context=None, mp_context = mp.get_context("spawn") else: mp_context = mp.get_context() + if (getattr(mp_context, "get_start_method", lambda: None)() == "fork" + and mp_context == mp.context._default_context._default_context): + import warnings + warnings.warn( + "The multiprocessing 'fork' start method will change " + "away from 'fork' in Python >= 3.14, per GH-84559. " + "concurrent.futures.process is built upon multiprocessing. " + "If your application requires continued use of 'fork', " + "pass a mp_context= parameter created with the start method " + "explicitly specified.", + category=mp.context.DefaultForkDeprecationWarning, + stacklevel=2, + ) self._mp_context = mp_context # https://github.com/python/cpython/issues/90622 diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 5349ec166e9800..534456c5454888 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -23,7 +23,7 @@ class TimeoutError(ProcessError): class AuthenticationError(ProcessError): pass -class DefaultsDeprecationWarning(DeprecationWarning): +class DefaultForkDeprecationWarning(DeprecationWarning): pass # @@ -261,6 +261,7 @@ def get_start_method(self, allow_none=False): return self._actual_context._name def get_all_start_methods(self): + """Returns a list of the supported start methods, default first.""" if sys.platform == 'win32': return ['spawn'] else: @@ -284,19 +285,16 @@ def _Popen(process_obj): return Popen(process_obj) class _DeprecatedForkProcess(ForkProcess): - @classmethod - def _warn(cls, stacklevel): + @staticmethod + def _warn(stacklevel): import warnings warnings.warn( - "Use of the multiprocessing 'fork' start method by default is " - "deprecated. " - "The default will change in Python >= 3.14, per GH-84559. " - "Please use an explicit multiprocessing.get_context or " - "multiprocessing.set_start_method API call to specify your " - "application's start method if you want to use 'fork'." - " The 'spawn' and 'forkserver' start methods are safer " - "depending on the platform and application.", - category=DefaultsDeprecationWarning, + "The multiprocessing 'fork' start method will change " + "change away from 'fork' in Python >= 3.14, per GH-84559. " + "Use a multiprocessing.get_context(X) context or " + "call multiprocessing.set_start_method(X) to explicitly " + "specify your application's need if you really want 'fork'.", + category=DefaultForkDeprecationWarning, stacklevel=stacklevel, ) diff --git a/Lib/test/test_multiprocessing_defaults.py b/Lib/test/test_multiprocessing_defaults.py index d63578aa147784..b075826d3bc780 100644 --- a/Lib/test/test_multiprocessing_defaults.py +++ b/Lib/test/test_multiprocessing_defaults.py @@ -2,7 +2,7 @@ from inspect import currentframe, getframeinfo import multiprocessing -from multiprocessing.context import DefaultsDeprecationWarning +from multiprocessing.context import DefaultForkDeprecationWarning import sys from test.support import threading_helper import unittest @@ -29,35 +29,35 @@ def setUp(self): def test_default_fork_start_method_warning_process(self): with warnings.catch_warnings(record=True) as ws: warnings.simplefilter('ignore') - warnings.filterwarnings('always', category=DefaultsDeprecationWarning) + warnings.filterwarnings('always', category=DefaultForkDeprecationWarning) process = multiprocessing.Process(target=do_nothing) process.start() # warning should point here. join_process(process) - self.assertIsInstance(ws[0].message, DefaultsDeprecationWarning) + self.assertIsInstance(ws[0].message, DefaultForkDeprecationWarning) self.assertIn(__file__, ws[0].filename) self.assertEqual(getframeinfo(currentframe()).lineno-4, ws[0].lineno) self.assertIn("'fork'", str(ws[0].message)) - self.assertIn("start_method API", str(ws[0].message)) + self.assertIn("get_context", str(ws[0].message)) self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) def test_default_fork_start_method_warning_pool(self): with warnings.catch_warnings(record=True) as ws: warnings.simplefilter('ignore') - warnings.filterwarnings('always', category=DefaultsDeprecationWarning) + warnings.filterwarnings('always', category=DefaultForkDeprecationWarning) pool = multiprocessing.Pool(1) # warning should point here. pool.terminate() pool.join() - self.assertIsInstance(ws[0].message, DefaultsDeprecationWarning) + self.assertIsInstance(ws[0].message, DefaultForkDeprecationWarning) self.assertIn(__file__, ws[0].filename) self.assertEqual(getframeinfo(currentframe()).lineno-5, ws[0].lineno) self.assertIn("'fork'", str(ws[0].message)) - self.assertIn("start_method API", str(ws[0].message)) + self.assertIn("get_context", str(ws[0].message)) self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) def test_no_mp_warning_when_using_explicit_fork_context(self): with warnings.catch_warnings(record=True) as ws: warnings.simplefilter('ignore') - warnings.filterwarnings('always', category=DefaultsDeprecationWarning) + warnings.filterwarnings('always', category=DefaultForkDeprecationWarning) fork_mp = multiprocessing.get_context('fork') pool = fork_mp.Pool(1) pool.terminate() From 29764a2feb908a843291c1e13128fcefa4c3beac Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 07:25:07 +0000 Subject: [PATCH 08/22] avoid double warning in futures --- Lib/concurrent/futures/process.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index e125d91ea0e851..a44184bd9b051d 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -650,6 +650,9 @@ def __init__(self, max_workers=None, mp_context=None, mp_context = mp.get_context("spawn") else: mp_context = mp.get_context() + # getattr(...)() instead of .get_start_method() as the docs imply that + # this method is not required. Hokey! But if someone passed us an odd + # duck, we just skip our introspection and warning. if (getattr(mp_context, "get_start_method", lambda: None)() == "fork" and mp_context == mp.context._default_context._default_context): import warnings @@ -663,6 +666,9 @@ def __init__(self, max_workers=None, mp_context=None, category=mp.context.DefaultForkDeprecationWarning, stacklevel=2, ) + # Avoid the equivalent warning from multiprocessing itself via + # a non-default fork context. + mp_context = mp.get_context("fork") self._mp_context = mp_context # https://github.com/python/cpython/issues/90622 From c52b9dba4c06e7bc51f62fe44bbe0de243a30b8f Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 20:17:48 +0000 Subject: [PATCH 09/22] Cleanup messages, test futures warning. --- Lib/concurrent/futures/process.py | 19 ++++++++----------- Lib/multiprocessing/context.py | 10 +++++----- Lib/test/test_concurrent_futures.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a44184bd9b051d..601988bdb1e82c 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -650,19 +650,16 @@ def __init__(self, max_workers=None, mp_context=None, mp_context = mp.get_context("spawn") else: mp_context = mp.get_context() - # getattr(...)() instead of .get_start_method() as the docs imply that - # this method is not required. Hokey! But if someone passed us an odd - # duck, we just skip our introspection and warning. - if (getattr(mp_context, "get_start_method", lambda: None)() == "fork" - and mp_context == mp.context._default_context._default_context): + if (mp_context.get_start_method() == "fork" and + mp_context == mp.context._default_context._default_context): import warnings warnings.warn( - "The multiprocessing 'fork' start method will change " - "away from 'fork' in Python >= 3.14, per GH-84559. " - "concurrent.futures.process is built upon multiprocessing. " - "If your application requires continued use of 'fork', " - "pass a mp_context= parameter created with the start method " - "explicitly specified.", + "The default multiprocessing start method will change " + "away from 'fork' in Python >= 3.14, per GH-84559. " + "ProcessPoolExecutor uses multiprocessing. " + "If your application requires 'fork', explicitly specify " + "that by passing a mp_context= parameter. " + "The safest start method is 'spawn'.", category=mp.context.DefaultForkDeprecationWarning, stacklevel=2, ) diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 534456c5454888..49e651d48240c5 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -289,11 +289,11 @@ class _DeprecatedForkProcess(ForkProcess): def _warn(stacklevel): import warnings warnings.warn( - "The multiprocessing 'fork' start method will change " - "change away from 'fork' in Python >= 3.14, per GH-84559. " - "Use a multiprocessing.get_context(X) context or " - "call multiprocessing.set_start_method(X) to explicitly " - "specify your application's need if you really want 'fork'.", + "The default multiprocessing start method will change " + "away from 'fork' in Python >= 3.14, per GH-84559. " + "Use multiprocessing.get_context(X) or .set_start_method(X) to " + "explicitly specify it when your application requires 'fork'. " + "The safest start method is 'spawn'.", category=DefaultForkDeprecationWarning, stacklevel=stacklevel, ) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index fe9fdc4f44d37b..379f0be242b998 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -18,6 +18,7 @@ import threading import time import unittest +import warnings import weakref from pickle import PicklingError @@ -571,6 +572,24 @@ def test_shutdown_no_wait(self): assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) +@unittest.skipIf(mp.get_all_start_methods()[0] != "fork", "non-fork default.") +class ProcessPoolExecutorDefaultForkWarning(unittest.TestCase): + def test_fork_default_warns(self): + with self.assertWarns(mp.context.DefaultForkDeprecationWarning): + with futures.ProcessPoolExecutor(2): + pass + + def test_explicit_fork_does_not_warn(self): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("ignore") + warnings.filterwarnings( + 'always', category=mp.context.DefaultForkDeprecationWarning) + ctx = mp.get_context("fork") # Non-default fork context. + with futures.ProcessPoolExecutor(2, mp_context=ctx): + pass + self.assertEqual(len(ws), 0, msg=[str(x) for x in ws]) + + create_executor_tests(ProcessPoolShutdownTest, executor_mixins=(ProcessPoolForkMixin, ProcessPoolForkserverMixin, From 4c8cc342ffc789c46180f0bdb29722085ee7edf1 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 20:33:00 +0000 Subject: [PATCH 10/22] cleanup warn stacklevel logic --- Lib/multiprocessing/pool.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 3ea95c5ead6ed5..089087e82b3404 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -326,20 +326,21 @@ def _repopulate_pool_static(ctx, Process, processes, pool, inqueue, wrap_exception)) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True - if _warn := getattr(w, '_warn', None): # GH-84559 - # Hacky, but BaseProcess._Popen does everything via the class - # using static or class methods with no instance. The way - # multiprocessing APIs get used, a concurrency mess here - # with a Pool() being constructed at the same time as other - # threads are calling Process.start() is highly unlikely. - # If so: The oddball application gets the warning attributed + proc_class = w.__class__ + orig_level = getattr(proc_class, '_ORIG_WARNING_STACKLEVEL', None) + if orig_level: + # Direct the GH-84559 warning to a useful place. + # Hacky, but BaseProcess._Popen does everything via class + # and static methods with no instance. The way multiprocessing + # APIs get used, a concurrency mess here with a Pool() being + # constructed at the same time other threads are calling + # Process.start() is highly unlikely. + # If so: That oddball application gets the warning attributed # to an unusual spot in the stack on occasion. - process_class = w.__class__ - process_class._warning_stacklevel = 8 # blame mp.Pool() + proc_class._warning_stacklevel = 8 # blame mp.Pool() w.start() - if _warn: - process_class._warning_stacklevel = ( - process_class._ORIG_WARNING_STACKLEVEL) + if orig_level: + proc_class._warning_stacklevel = orig_level pool.append(w) util.debug('added worker') From 717968499df2872d2e897fd348780621045f462b Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sat, 31 Dec 2022 23:16:34 +0000 Subject: [PATCH 11/22] Document all the things! --- Doc/library/concurrent.futures.rst | 13 +++-- Doc/library/multiprocessing.rst | 84 +++++++++++++++++++----------- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 8106cc235e5a3c..211791f08e5458 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -250,9 +250,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then the default chosen will be at most ``61``, even if more processors are available. - *mp_context* can be a multiprocessing context or None. It will be used to - launch the workers. If *mp_context* is ``None`` or not given, the default - multiprocessing context is used. + *mp_context* can be a :mod:`multiprocessing` context or ``None``. It will be + used to launch the workers. If *mp_context* is ``None`` or not given, the + default :mod:`multiprocessing` context is used. *initializer* is an optional callable that is called at the start of each worker process; *initargs* is a tuple of arguments passed to the @@ -284,6 +284,13 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. + .. versionchanged:: 3.12 + The implcit use of the :mod:`multiprocessing` *fork* start method as a + platform default (see :ref:`multiprocessing-start-methods`) now raises a + :exc:`DeprecationWarning` as the default will be changing in Python >= + 3.14. Code that requires *fork* it should explicitly specify that when + creating their ProcessPoolExecutor by passing a + ``mp_context=multiprocessing.get_context('fork')`` parameter. .. _processpoolexecutor-example: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b5ceeb796f8f2f..ef3a7ba39fc4ae 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -19,7 +19,7 @@ offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due to this, the :mod:`multiprocessing` module allows the programmer to fully -leverage multiple processors on a given machine. It runs on both Unix and +leverage multiple processors on a given machine. It runs on both POSIX and Windows. The :mod:`multiprocessing` module also introduces APIs which do not have @@ -99,11 +99,11 @@ necessary, see :ref:`multiprocessing-programming`. +.. _multiprocessing-start-methods: + Contexts and start methods ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _multiprocessing-start-methods: - Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are @@ -115,7 +115,7 @@ to start a process. These *start methods* are will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. - Available on Unix and Windows. The default on Windows and macOS. + Available on POSIX and Windows platforms. The default on Windows and macOS. *fork* The parent process uses :func:`os.fork` to fork the Python @@ -124,32 +124,39 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on Unix only. The default on Unix. + Available on POSIX systems. The default on POSIX other than macOS. + + .. versionchanged:: 3.12 + The implcit use of the *fork* start method as the default now raises a + :exc:`DeprecationWarning`. Code that requires it should explicitly + specify *fork* via :func:`get_context` or :func:`set_start_method`. + The default will change in 3.14. *forkserver* When the program starts and selects the *forkserver* start method, - a server process is started. From then on, whenever a new process + a server process is spawned. From then on, whenever a new process is needed, the parent process connects to the server and requests - that it fork a new process. The fork server process is single - threaded so it is safe for it to use :func:`os.fork`. No + that it fork a new process. The fork server process is single threaded + unless system libraries or preloaded imports spawned threads as a + side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on Unix platforms which support passing file descriptors - over Unix pipes. + Available on POSIX platforms which support passing file descriptors + over Unix pipes such as Linux. .. versionchanged:: 3.8 On macOS, the *spawn* start method is now the default. The *fork* start method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. + subprocess as macOS system libraries may start threads. See :issue:`33725`. .. versionchanged:: 3.4 - *spawn* added on all Unix platforms, and *forkserver* added for - some Unix platforms. + *spawn* added on all POSIX platforms, and *forkserver* added for + some POSIX platforms. Child processes no longer inherit all of the parents inheritable handles on Windows. -On Unix using the *spawn* or *forkserver* start methods will also +On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or :class:`~multiprocessing.shared_memory.SharedMemory` objects) created @@ -211,9 +218,9 @@ library user. .. warning:: - The ``'spawn'`` and ``'forkserver'`` start methods cannot currently + The ``'spawn'`` and ``'forkserver'`` start methods generally cannot be used with "frozen" executables (i.e., binaries produced by - packages like **PyInstaller** and **cx_Freeze**) on Unix. + packages like **PyInstaller** and **cx_Freeze**) on POSIX systems. The ``'fork'`` start method does work. @@ -629,14 +636,14 @@ The :mod:`multiprocessing` package mostly replicates the API of the calling :meth:`join()` is simpler. On Windows, this is an OS handle usable with the ``WaitForSingleObject`` - and ``WaitForMultipleObjects`` family of API calls. On Unix, this is + and ``WaitForMultipleObjects`` family of API calls. On POSIX, this is a file descriptor usable with primitives from the :mod:`select` module. .. versionadded:: 3.3 .. method:: terminate() - Terminate the process. On Unix this is done using the ``SIGTERM`` signal; + Terminate the process. On POSIX this is done using the ``SIGTERM`` signal; on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and finally clauses, etc., will not be executed. @@ -653,7 +660,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: kill() - Same as :meth:`terminate()` but using the ``SIGKILL`` signal on Unix. + Same as :meth:`terminate()` but using the ``SIGKILL`` signal on POSIX. .. versionadded:: 3.7 @@ -815,7 +822,7 @@ For an example of the usage of queues for interprocess communication see Return the approximate size of the queue. Because of multithreading/multiprocessing semantics, this number is not reliable. - Note that this may raise :exc:`NotImplementedError` on Unix platforms like + Note that this may raise :exc:`NotImplementedError` on platforms like macOS where ``sem_getvalue()`` is not implemented. .. method:: empty() @@ -1034,9 +1041,8 @@ Miscellaneous Returns a list of the supported start methods, the first of which is the default. The possible start methods are ``'fork'``, - ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is - available. On Unix ``'fork'`` and ``'spawn'`` are always - supported, with ``'fork'`` being the default. + ``'spawn'`` and ``'forkserver'``. Not all platforms support all + methods. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 @@ -1048,7 +1054,7 @@ Miscellaneous If *method* is ``None`` then the default context is returned. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified - start method is not available. + start method is not available. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 @@ -1062,8 +1068,7 @@ Miscellaneous is true then ``None`` is returned. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` - or ``None``. ``'fork'`` is the default on Unix, while ``'spawn'`` is - the default on Windows and macOS. + or ``None``. See :ref:`multiprocessing-start-methods`. .. versionchanged:: 3.8 @@ -1084,11 +1089,26 @@ Miscellaneous before they can create child processes. .. versionchanged:: 3.4 - Now supported on Unix when the ``'spawn'`` start method is used. + Now supported on POSIX when the ``'spawn'`` start method is used. .. versionchanged:: 3.11 Accepts a :term:`path-like object`. +.. function:: set_forkserver_preload(module_names) + + Set a list of module names for the forkserver main process to attempt to + import so that their already imported state is inherited by forked + processes. Any :exc:`ImportError` when doing so is silently ignored. + This can be used as a performance enhancement to avoid repeated work + in every process. + + For this to work, it must be called before the forkserver process has been + launched (before creating a :class:`Pool` or starting a :class:`Process`). + + Only meaningful when using the ``'forkserver'`` start method. + + .. versionadded:: 3.4 + .. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. @@ -1102,6 +1122,8 @@ Miscellaneous protected inside the ``if __name__ == '__main__'`` clause of the main module. + See :ref:`multiprocessing-start-methods`. + .. versionadded:: 3.4 .. note:: @@ -2520,7 +2542,7 @@ multiple connections at the same time. *timeout* is ``None`` then it will block for an unlimited period. A negative timeout is equivalent to a zero timeout. - For both Unix and Windows, an object can appear in *object_list* if + For both POSIX and Windows, an object can appear in *object_list* if it is * a readable :class:`~multiprocessing.connection.Connection` object; @@ -2531,7 +2553,7 @@ multiple connections at the same time. A connection or socket object is ready when there is data available to be read from it, or the other end has been closed. - **Unix**: ``wait(object_list, timeout)`` almost equivalent + **POSIX**: ``wait(object_list, timeout)`` almost equivalent ``select.select(object_list, [], [], timeout)``. The difference is that, if :func:`select.select` is interrupted by a signal, it can raise :exc:`OSError` with an error number of ``EINTR``, whereas @@ -2803,7 +2825,7 @@ Thread safety of proxies Joining zombie processes - On Unix when a process finishes but has not been joined it becomes a zombie. + On POSIX when a process finishes but has not been joined it becomes a zombie. There should never be very many because each time a new process starts (or :func:`~multiprocessing.active_children` is called) all completed processes which have not yet been joined will be joined. Also calling a finished @@ -2866,7 +2888,7 @@ Joining processes that use queues Explicitly pass resources to child processes - On Unix using the *fork* start method, a child process can make + On POSIX using the *fork* start method, a child process can make use of a shared resource created in a parent process using a global resource. However, it is better to pass the object as an argument to the constructor for the child process. From f7e31618b7e469e3892b44af09cc97ca14619ff3 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sun, 1 Jan 2023 00:08:50 +0000 Subject: [PATCH 12/22] Have all doctests use 'spawn'. --- Doc/library/multiprocessing.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index ef3a7ba39fc4ae..77dcd9913972ef 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -683,7 +683,8 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. doctest:: >>> import multiprocessing, time, signal - >>> p = multiprocessing.Process(target=time.sleep, args=(1000,)) + >>> mp_context = multiprocessing.get_context('spawn') + >>> p = mp_context.Process(target=time.sleep, args=(1000,)) >>> print(p, p.is_alive()) False >>> p.start() @@ -1928,7 +1929,8 @@ their parent process exits. The manager classes are defined in the .. doctest:: - >>> manager = multiprocessing.Manager() + >>> mp_context = multiprocessing.get_context('spawn') + >>> manager = mp_context.Manager() >>> Global = manager.Namespace() >>> Global.x = 10 >>> Global.y = 'hello' @@ -2040,8 +2042,8 @@ the proxy). In this way, a proxy can be used just like its referent can: .. doctest:: - >>> from multiprocessing import Manager - >>> manager = Manager() + >>> mp_context = multiprocessing.get_context('spawn') + >>> manager = mp_context.Manager() >>> l = manager.list([i*i for i in range(10)]) >>> print(l) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] From 99478600da7d30c8dc30b6df6a9533602a762eba Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sun, 1 Jan 2023 00:23:36 +0000 Subject: [PATCH 13/22] Correct warning stacklevel for Manager. --- Lib/multiprocessing/context.py | 9 ++++++++- Lib/test/test_multiprocessing_defaults.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 49e651d48240c5..6851634f5c3e32 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -56,8 +56,15 @@ def Manager(self): can be used to create shared objects. ''' from .managers import SyncManager - m = SyncManager(ctx=self.get_context()) + ctx = self.get_context() + m = SyncManager(ctx=ctx) + proc_class = ctx.Process + orig_level = getattr(proc_class, '_ORIG_WARNING_STACKLEVEL', None) + if orig_level: + ctx.Process._warning_stacklevel = 6 # blame mp.Manager() m.start() + if orig_level: + ctx.Process._warning_stacklevel = orig_level return m def Pipe(self, duplex=True): diff --git a/Lib/test/test_multiprocessing_defaults.py b/Lib/test/test_multiprocessing_defaults.py index b075826d3bc780..1da4c065238384 100644 --- a/Lib/test/test_multiprocessing_defaults.py +++ b/Lib/test/test_multiprocessing_defaults.py @@ -54,6 +54,19 @@ def test_default_fork_start_method_warning_pool(self): self.assertIn("get_context", str(ws[0].message)) self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + def test_default_fork_start_method_warning_manager(self): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter('ignore') + warnings.filterwarnings('always', category=DefaultForkDeprecationWarning) + manager = multiprocessing.Manager() # warning should point here. + manager.shutdown() + self.assertIsInstance(ws[0].message, DefaultForkDeprecationWarning) + self.assertIn(__file__, ws[0].filename) + self.assertEqual(getframeinfo(currentframe()).lineno-4, ws[0].lineno) + self.assertIn("'fork'", str(ws[0].message)) + self.assertIn("get_context", str(ws[0].message)) + self.assertEqual(len(ws), 1, msg=[str(x) for x in ws]) + def test_no_mp_warning_when_using_explicit_fork_context(self): with warnings.catch_warnings(record=True) as ws: warnings.simplefilter('ignore') From 5834f2d2a86194f56dedd8e9c6a718c0d5b1a342 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sun, 1 Jan 2023 00:25:55 +0000 Subject: [PATCH 14/22] more doctest cleanup. --- Doc/library/multiprocessing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 77dcd9913972ef..62c56727cbcad9 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -686,14 +686,14 @@ The :mod:`multiprocessing` package mostly replicates the API of the >>> mp_context = multiprocessing.get_context('spawn') >>> p = mp_context.Process(target=time.sleep, args=(1000,)) >>> print(p, p.is_alive()) - False + <...Process ... initial> False >>> p.start() >>> print(p, p.is_alive()) - True + <...Process ... started> True >>> p.terminate() >>> time.sleep(0.1) >>> print(p, p.is_alive()) - False + <...Process ... stopped exitcode=-SIGTERM> False >>> p.exitcode == -signal.SIGTERM True From 7d48e864bb9b6034f08b88d8ad32aa4657829acd Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google]" Date: Sun, 1 Jan 2023 01:19:41 +0000 Subject: [PATCH 15/22] NEWS entry. --- .../2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst diff --git a/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst b/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst new file mode 100644 index 00000000000000..f4529ca091a48c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst @@ -0,0 +1,9 @@ +The :mod:`multiprocessing` module and +:class:`concurrent.futures.ProcessPoolExecutor` will emit a +:exc:`DeprecationWarning` on Linux and other non-macOS POSIX systems when +the default multiprocessing start method of ``'fork'`` is used implicitly +rather than being explicitly specified through a +:func:`multiprocessing.get_context` context. + +This is in preparation for default start method to change in Python 3.14 to +a default that is safe for multithreaded applications. From 28c0cfcf517ed63b5eb78d680dfb530f76e3fa21 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Mon, 9 Jan 2023 21:39:30 -0800 Subject: [PATCH 16/22] docs: add the implicit i --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 62c56727cbcad9..794e8b8df51d91 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -127,7 +127,7 @@ to start a process. These *start methods* are Available on POSIX systems. The default on POSIX other than macOS. .. versionchanged:: 3.12 - The implcit use of the *fork* start method as the default now raises a + The implicit use of the *fork* start method as the default now raises a :exc:`DeprecationWarning`. Code that requires it should explicitly specify *fork* via :func:`get_context` or :func:`set_start_method`. The default will change in 3.14. From 4c16205033c902ff76067f4f47f89b7ad36c92b5 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Fri, 27 Jan 2023 18:40:50 -0800 Subject: [PATCH 17/22] Use skip_file_prefixes. --- Lib/concurrent/futures/process.py | 2 +- Lib/multiprocessing/context.py | 21 +++++-------------- ...2-12-30-01-56-24.gh-issue-84559.pFFtaE.rst | 10 +++++++++ 3 files changed, 16 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 601988bdb1e82c..1022179a313447 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -57,6 +57,7 @@ import itertools import sys from traceback import format_exception +import warnings _threads_wakeups = weakref.WeakKeyDictionary() @@ -652,7 +653,6 @@ def __init__(self, max_workers=None, mp_context=None, mp_context = mp.get_context() if (mp_context.get_start_method() == "fork" and mp_context == mp.context._default_context._default_context): - import warnings warnings.warn( "The default multiprocessing start method will change " "away from 'fork' in Python >= 3.14, per GH-84559. " diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 6851634f5c3e32..003712e6096cb1 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -59,12 +59,7 @@ def Manager(self): ctx = self.get_context() m = SyncManager(ctx=ctx) proc_class = ctx.Process - orig_level = getattr(proc_class, '_ORIG_WARNING_STACKLEVEL', None) - if orig_level: - ctx.Process._warning_stacklevel = 6 # blame mp.Manager() m.start() - if orig_level: - ctx.Process._warning_stacklevel = orig_level return m def Pipe(self, duplex=True): @@ -291,9 +286,11 @@ def _Popen(process_obj): from .popen_fork import Popen return Popen(process_obj) + _warn_package_prefixes = (os.path.dirname(__file__),) + class _DeprecatedForkProcess(ForkProcess): - @staticmethod - def _warn(stacklevel): + @classmethod + def _Popen(cls, process_obj): import warnings warnings.warn( "The default multiprocessing start method will change " @@ -302,16 +299,8 @@ def _warn(stacklevel): "explicitly specify it when your application requires 'fork'. " "The safest start method is 'spawn'.", category=DefaultForkDeprecationWarning, - stacklevel=stacklevel, + skip_file_prefixes=_warn_package_prefixes, ) - - # Blame the process.start() method call. - _ORIG_WARNING_STACKLEVEL = 5 - _warning_stacklevel = _ORIG_WARNING_STACKLEVEL - - @classmethod - def _Popen(cls, process_obj): - cls._warn(stacklevel=cls._warning_stacklevel) return super()._Popen(process_obj) class SpawnProcess(process.BaseProcess): diff --git a/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst b/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst new file mode 100644 index 00000000000000..7c950e6819094d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst @@ -0,0 +1,10 @@ +The :mod:`multiprocessing` module now triggers a :exc:`DeprecationWarning` +when the (``fork``) start method is used implicitly via the default behavior +on non-macOS POSIX-like platforms. + +We want application authors to explicitly specify their start method (thus +avoiding the warning) when they do need to use ``fork``. Our default will +change to something more conservative and compatible with multithreaded +processes in the Python 3.14 timeframe. + +macOS and Windows already default to the ``spawn`` start method. From 7d84a72b063b8a90f520e0b49e299c0218e2c2f2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Fri, 27 Jan 2023 19:11:48 -0800 Subject: [PATCH 18/22] use the modern warnings API, doc updates. --- Doc/library/concurrent.futures.rst | 3 ++- Doc/library/multiprocessing.rst | 10 +++++----- Lib/concurrent/futures/process.py | 5 +++-- Lib/multiprocessing/pool.py | 14 -------------- .../2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst | 10 ---------- .../2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst | 2 ++ 6 files changed, 12 insertions(+), 32 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 211791f08e5458..1605b5de4f5f9b 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -253,6 +253,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *mp_context* can be a :mod:`multiprocessing` context or ``None``. It will be used to launch the workers. If *mp_context* is ``None`` or not given, the default :mod:`multiprocessing` context is used. + See :ref:`multiprocessing-start-methods`. *initializer* is an optional callable that is called at the start of each worker process; *initargs* is a tuple of arguments passed to the @@ -288,7 +289,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. The implcit use of the :mod:`multiprocessing` *fork* start method as a platform default (see :ref:`multiprocessing-start-methods`) now raises a :exc:`DeprecationWarning` as the default will be changing in Python >= - 3.14. Code that requires *fork* it should explicitly specify that when + 3.14. Code that requires *fork* should explicitly specify that when creating their ProcessPoolExecutor by passing a ``mp_context=multiprocessing.get_context('fork')`` parameter. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 794e8b8df51d91..6e01de3438e1e1 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,7 +124,7 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. The default on POSIX other than macOS. + Available on POSIX systems. Currently the default on POSIX (not macOS). .. versionchanged:: 3.12 The implicit use of the *fork* start method as the default now raises a @@ -137,9 +137,9 @@ to start a process. These *start methods* are a server process is spawned. From then on, whenever a new process is needed, the parent process connects to the server and requests that it fork a new process. The fork server process is single threaded - unless system libraries or preloaded imports spawned threads as a - side-effect so it is generally safe for it to use :func:`os.fork`. No - unnecessary resources are inherited. + unless system libraries or preloaded imports spawn threads as a + side-effect so it is generally safe for it to use :func:`os.fork`. + No unnecessary resources are inherited. Available on POSIX platforms which support passing file descriptors over Unix pipes such as Linux. @@ -221,7 +221,7 @@ library user. The ``'spawn'`` and ``'forkserver'`` start methods generally cannot be used with "frozen" executables (i.e., binaries produced by packages like **PyInstaller** and **cx_Freeze**) on POSIX systems. - The ``'fork'`` start method does work. + The ``'fork'`` start method may work if code does not use threads. Exchanging objects between processes diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 1022179a313447..257dd02fbc6cce 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -657,8 +657,9 @@ def __init__(self, max_workers=None, mp_context=None, "The default multiprocessing start method will change " "away from 'fork' in Python >= 3.14, per GH-84559. " "ProcessPoolExecutor uses multiprocessing. " - "If your application requires 'fork', explicitly specify " - "that by passing a mp_context= parameter. " + "If your application requires the 'fork' multiprocessing " + "start method, explicitly specify that by passing a " + "mp_context= parameter. " "The safest start method is 'spawn'.", category=mp.context.DefaultForkDeprecationWarning, stacklevel=2, diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 089087e82b3404..4f5d88cb975cb7 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -326,21 +326,7 @@ def _repopulate_pool_static(ctx, Process, processes, pool, inqueue, wrap_exception)) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True - proc_class = w.__class__ - orig_level = getattr(proc_class, '_ORIG_WARNING_STACKLEVEL', None) - if orig_level: - # Direct the GH-84559 warning to a useful place. - # Hacky, but BaseProcess._Popen does everything via class - # and static methods with no instance. The way multiprocessing - # APIs get used, a concurrency mess here with a Pool() being - # constructed at the same time other threads are calling - # Process.start() is highly unlikely. - # If so: That oddball application gets the warning attributed - # to an unusual spot in the stack on occasion. - proc_class._warning_stacklevel = 8 # blame mp.Pool() w.start() - if orig_level: - proc_class._warning_stacklevel = orig_level pool.append(w) util.debug('added worker') diff --git a/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst b/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst deleted file mode 100644 index 7c950e6819094d..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-30-01-56-24.gh-issue-84559.pFFtaE.rst +++ /dev/null @@ -1,10 +0,0 @@ -The :mod:`multiprocessing` module now triggers a :exc:`DeprecationWarning` -when the (``fork``) start method is used implicitly via the default behavior -on non-macOS POSIX-like platforms. - -We want application authors to explicitly specify their start method (thus -avoiding the warning) when they do need to use ``fork``. Our default will -change to something more conservative and compatible with multithreaded -processes in the Python 3.14 timeframe. - -macOS and Windows already default to the ``spawn`` start method. diff --git a/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst b/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst index f4529ca091a48c..3793e0f1fddb20 100644 --- a/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst +++ b/Misc/NEWS.d/next/Library/2023-01-01-01-19-33.gh-issue-84559.zEjsEJ.rst @@ -7,3 +7,5 @@ rather than being explicitly specified through a This is in preparation for default start method to change in Python 3.14 to a default that is safe for multithreaded applications. + +Windows and macOS are unaffected as their default start method is ``spawn``. From 8b2029e856c3d8ea19c56d2d10147fa99d18d3b5 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Fri, 27 Jan 2023 20:13:55 -0800 Subject: [PATCH 19/22] documentation cleanup --- Doc/library/concurrent.futures.rst | 6 +++--- Doc/library/multiprocessing.rst | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 1605b5de4f5f9b..e34931c63017b5 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -288,9 +288,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. .. versionchanged:: 3.12 The implcit use of the :mod:`multiprocessing` *fork* start method as a platform default (see :ref:`multiprocessing-start-methods`) now raises a - :exc:`DeprecationWarning` as the default will be changing in Python >= - 3.14. Code that requires *fork* should explicitly specify that when - creating their ProcessPoolExecutor by passing a + :exc:`DeprecationWarning`. The default will change in Python 3.14. + Code that requires *fork* should explicitly specify that when creating + their :class:`ProcessPoolExecutor` by passing a ``mp_context=multiprocessing.get_context('fork')`` parameter. .. _processpoolexecutor-example: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6e01de3438e1e1..c60b229ae2d07e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,13 +124,7 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX (not macOS). - - .. versionchanged:: 3.12 - The implicit use of the *fork* start method as the default now raises a - :exc:`DeprecationWarning`. Code that requires it should explicitly - specify *fork* via :func:`get_context` or :func:`set_start_method`. - The default will change in 3.14. + Available on POSIX systems. Currently the default on POSIX except macOS. *forkserver* When the program starts and selects the *forkserver* start method, @@ -144,6 +138,12 @@ to start a process. These *start methods* are Available on POSIX platforms which support passing file descriptors over Unix pipes such as Linux. +.. versionchanged:: 3.12 + Implicit use of the *fork* start method as the default now raises a + :exc:`DeprecationWarning`. Code that requires it should explicitly + specify *fork* via :func:`get_context` or :func:`set_start_method`. + The default will change away from *fork* in 3.14. + .. versionchanged:: 3.8 On macOS, the *spawn* start method is now the default. The *fork* start From 846e9a06b11cb1bfa5c9bbcdc0e6762b936ffb8b Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Wed, 1 Feb 2023 14:40:16 -0800 Subject: [PATCH 20/22] Remove an unused variable. --- Lib/multiprocessing/context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 003712e6096cb1..d4062fdb41d6e2 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -58,7 +58,6 @@ def Manager(self): from .managers import SyncManager ctx = self.get_context() m = SyncManager(ctx=ctx) - proc_class = ctx.Process m.start() return m From 7078986d2627b67d3738c86cbc5965a83f5e2d5b Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Wed, 1 Feb 2023 15:38:47 -0800 Subject: [PATCH 21/22] doc typo fix, unnecessary method change revert. --- Doc/library/concurrent.futures.rst | 2 +- Lib/multiprocessing/context.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e34931c63017b5..10cffdaee0bb8c 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -286,7 +286,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. control the lifetime of workers in the pool. .. versionchanged:: 3.12 - The implcit use of the :mod:`multiprocessing` *fork* start method as a + The implicit use of the :mod:`multiprocessing` *fork* start method as a platform default (see :ref:`multiprocessing-start-methods`) now raises a :exc:`DeprecationWarning`. The default will change in Python 3.14. Code that requires *fork* should explicitly specify that when creating diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index d4062fdb41d6e2..010a920540e844 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -56,8 +56,7 @@ def Manager(self): can be used to create shared objects. ''' from .managers import SyncManager - ctx = self.get_context() - m = SyncManager(ctx=ctx) + m = SyncManager(ctx=self.get_context()) m.start() return m From e3e3b3ce4998d8737b1a4e894581ca65a7e86644 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Thu, 2 Feb 2023 12:25:26 -0800 Subject: [PATCH 22/22] Add What's New entries. --- Doc/whatsnew/3.12.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a071159b800a34..e675fada339a1e 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -440,6 +440,11 @@ Deprecated warning at compile time. This field will be removed in Python 3.14. (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) +* Use of the implicit default ``'fork'`` start method for + :mod:`multiprocessing` and :class:`concurrent.futures.ProcessPoolExecutor` + now emits a :exc:`DeprecationWarning` on Linux and other non-macOS POSIX + systems. Avoid this by explicitly specifying a start method. + See :ref:`multiprocessing-start-methods`. Pending Removal in Python 3.13 ------------------------------ @@ -505,6 +510,9 @@ Pending Removal in Python 3.14 * Testing the truth value of an :class:`xml.etree.ElementTree.Element` is deprecated and will raise an exception in Python 3.14. +* The default :mod:`multiprocessing` start method will change to one of either + ``'forkserver'`` or ``'spawn'`` on all platforms for which ``'fork'`` remains + the default per :gh:`84559`. Pending Removal in Future Versions ----------------------------------