gh-155717: Determine multiprocessing's default start method lazily (GH-159254)

gh-155827 made `import multiprocessing` check whether the temporary
directory is writeable in order to choose between "forkserver" and
"spawn" as the default start method. That check ran at import time and
pulled in `multiprocessing.util`, `subprocess`, `tempfile`, `shutil`
and `random`, noticeably slowing down the import for every program,
including ones that never start a process and every spawned child,
the fork server and the resource tracker.

The default start method is now determined the first time it is needed
instead: on the first `get_context()`, `get_start_method()`,
`get_all_start_methods()` or default `Process` start.
`multiprocessing.util` goes back to importing `tempfile` only where it
is used.

Also include the fix from gh-158410 by Vyron Vasileiadis, along with a
test adapted from that PR: when `tempfile` finds no usable temporary
directory at all, the check now reports that instead of raising
`FileNotFoundError`, so that "spawn" is chosen.

The regression was reported, and the lazy approach suggested, by
Leandro Damascena in
#155717 (comment)

Co-authored-by: Vyron Vasileiadis <hi@fedonman.com>

Author: gpshead

Doc/library/multiprocessing.rst

@@ -160,11 +160,21 @@ to start a process.  These *start methods* are
     No unnecessary resources are inherited.
 
     Available on POSIX platforms which support passing file descriptors over
-    Unix pipes such as Linux.  The default on those.
+    Unix pipes such as Linux.  The default on those, unless there is no
+    writable temporary directory (see :func:`tempfile.gettempdir`), which
+    the fork server requires; *spawn* is the default in that case.
 
     .. versionchanged:: 3.14
        This became the default start method on POSIX platforms.
 
+    .. versionchanged:: next
+       *spawn* is now the default start method when there is no writable
+       temporary directory, for example on a read-only file system.
+       Previously *forkserver* remained the default there and starting a
+       process failed with an :exc:`OSError`.  The default is determined
+       when it is first needed rather than when :mod:`!multiprocessing`
+       is imported.  See :gh:`155717`.
+
 .. versionchanged:: 3.4
    *spawn* added on all POSIX platforms, and *forkserver* added for
    some POSIX platforms.

Lib/multiprocessing/context.py

@@ -1,10 +1,10 @@
+import functools
 import os
 import sys
 import threading
 
 from . import process
 from . import reduction
-from . import util
 
 __all__ = ()
 
@@ -240,10 +240,19 @@ def _after_fork():
 class DefaultContext(BaseContext):
     Process = Process
 
-    def __init__(self, context):
-        self._default_context = context
+    def __init__(self, context=None):
+        if context is not None:
+            self._default_context = context
         self._actual_context = None
 
+    # The platform default is only determined when first needed, as doing
+    # so can be costly; importing multiprocessing must stay cheap.
+    # A cached_property rather than a property so that the attribute can
+    # still be assigned, deleted and patched like a plain one.
+    @functools.cached_property
+    def _default_context(self):
+        return _get_platform_default_context()
+
     def get_context(self, method=None):
         if method is None:
             if self._actual_context is None:
@@ -331,18 +340,21 @@ def _check_available(self):
     if reduction.HAVE_SEND_HANDLE:
         _concrete_contexts['forkserver'] = ForkServerContext()
 
-    # bpo-33725: running arbitrary code after fork() is no longer reliable
-    # on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
-    # gh-84559: We changed everyones default to a thread safeish one in 3.14.
-    if (
-        reduction.HAVE_SEND_HANDLE
-        and sys.platform != 'darwin'
-        # gh-155717: forkserver requires to write temporary files
-        and util._has_writeable_tempdir()
-    ):
-        _default_context = DefaultContext(_concrete_contexts['forkserver'])
-    else:
-        _default_context = DefaultContext(_concrete_contexts['spawn'])
+    def _get_platform_default_context():
+        # bpo-33725: running arbitrary code after fork() is no longer reliable
+        # on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
+        # gh-84559: We changed everyones default to a thread safeish one in 3.14.
+        if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
+            # gh-155717: forkserver requires to write temporary files.
+            #
+            # Imported here rather than at the top level as util is costly
+            # to import and is not otherwise needed to import this module.
+            from . import util
+            if util._has_writeable_tempdir():
+                return _concrete_contexts['forkserver']
+        return _concrete_contexts['spawn']
+
+    _default_context = DefaultContext()
 
 else:  # Windows
 
@@ -365,7 +377,10 @@ class SpawnContext(BaseContext):
     _concrete_contexts = {
         'spawn': SpawnContext(),
     }
-    _default_context = DefaultContext(_concrete_contexts['spawn'])
+    def _get_platform_default_context():
+        return _concrete_contexts['spawn']
+
+    _default_context = DefaultContext()
 
 #
 # Force the start method

Lib/multiprocessing/util.py

@@ -10,7 +10,6 @@
 import os
 import itertools
 import sys
-import tempfile
 import weakref
 import atexit
 import threading        # we want threading to install it's
@@ -155,7 +154,7 @@ def _remove_temp_dir(rmtree, tempdir):
         current_process._config['tempdir'] = None
 
 
-def _get_base_temp_dir():
+def _get_base_temp_dir(tempfile):
     """Get a temporary directory where socket files will be created.
 
     To prevent additional imports, pass a pre-imported 'tempfile' module.
@@ -216,8 +215,9 @@ def get_temp_dir():
     # get name of a temp directory which will be automatically cleaned up
     tempdir = process.current_process()._config.get('tempdir')
     if tempdir is None:
-        import shutil
-        base_tempdir = _get_base_temp_dir()
+        # Deferred until first use to keep importing this module cheap.
+        import shutil, tempfile
+        base_tempdir = _get_base_temp_dir(tempfile)
         tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
         info('created temp directory %s', tempdir)
         # keep a strong reference to shutil.rmtree(), since the finalizer
@@ -234,7 +234,14 @@ def _has_writeable_tempdir():
     #
     # See: https://github.com/python/cpython/issues/155717.
 
-    path = _get_base_temp_dir()
+    # Deferred until first use to keep importing this module cheap.
+    import tempfile
+
+    try:
+        path = _get_base_temp_dir(tempfile)
+    except OSError:
+        # No usable temporary directory at all.
+        return False
     if path is None:
         return False
 

Lib/test/_test_multiprocessing.py

@@ -6516,6 +6516,62 @@ def test_forkserver_requires_writeable_tempdir(self, mode):
             _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
         self.assertEqual(out.decode().strip(), "spawn")
 
+    @unittest.skipIf(os.name == "nt", "requires POSIX")
+    @support.requires_non_root_user
+    def test_default_start_method_without_usable_tempdir(self):
+        # Neither importing multiprocessing nor determining the default
+        # start method may fail when tempfile finds no usable temporary
+        # directory at all, and the default start method must be 'spawn'.
+        #
+        # See https://github.com/python/cpython/issues/155717.
+
+        cmd = '''if 1:
+            import os, tempfile
+            # Make TMPDIR the only candidate, so that tempfile.gettempdir()
+            # raises FileNotFoundError instead of falling back to /tmp.
+            tempfile._candidate_tempdir_list = lambda: [os.environ["TMPDIR"]]
+            try:
+                tempfile.gettempdir()
+            except FileNotFoundError:
+                pass
+            else:
+                raise AssertionError("TMPDIR is usable")
+
+            import multiprocessing
+            if __name__ == "__main__":
+                print(multiprocessing.get_start_method(),
+                      multiprocessing.get_all_start_methods()[0])
+        '''
+
+        with support.os_helper.temp_dir() as root:
+            TMPDIR = pathlib.Path(root, "TMPDIR")
+            TMPDIR.mkdir(mode=os.R_OK | os.X_OK)
+            _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
+        self.assertEqual(out.decode().split(), ["spawn", "spawn"])
+
+    def test_import_does_not_determine_default_start_method(self):
+        # Determining the default start method requires checking that the
+        # temporary directory is writeable. That must not happen, nor may
+        # the modules it needs be loaded, when importing multiprocessing.
+        import_helper.ensure_lazy_imports(
+            "multiprocessing", {"tempfile", "shutil", "subprocess"})
+
+        cmd = '''if 1:
+            import sys
+            import multiprocessing
+            print(multiprocessing.get_start_method(allow_none=True),
+                  "tempfile" in sys.modules,
+                  multiprocessing.get_start_method(),
+                  multiprocessing.get_all_start_methods()[0])
+        '''
+        # -S: site customizations could import tempfile themselves.
+        _, out, err = script_helper.assert_python_ok('-S', '-c', cmd)
+        unset, probed, method, first = out.decode().split()
+        self.assertEqual(unset, "None")
+        self.assertEqual(probed, "False")
+        self.assertEqual(method, first)
+        self.assertIn(method, multiprocessing.get_all_start_methods())
+
 
 @unittest.skipIf(sys.platform == "win32",
                  "test semantics don't make sense on Windows")