Merge remote-tracking branch 'upstream/main' into gh-113318-getset-rework

# Conflicts:
#	Tools/clinic/libclinic/dsl_parser.py
#	Tools/clinic/libclinic/parse_args.py

Author: serhiy-storchaka

Doc/library/asyncio-task.rst

@@ -843,17 +843,13 @@ Timeouts
    Wait for the *fut* :ref:`awaitable <asyncio-awaitables>`
    to complete with a timeout.
 
-   If *fut* is a coroutine it is automatically scheduled as a Task.
-
    *timeout* can either be ``None`` or a float or int number of seconds
    to wait for.  If *timeout* is ``None``, block until the future
    completes.
 
-   If a timeout occurs, it cancels the task and raises
-   :exc:`TimeoutError`.
+   If a timeout occurs, it cancels *fut* and raises :exc:`TimeoutError`.
 
-   To avoid the task :meth:`cancellation <Task.cancel>`,
-   wrap it in :func:`shield`.
+   To prevent *fut* from being cancelled, wrap it in :func:`shield`.
 
    The function will wait until the future is actually cancelled,
    so the total wait time may exceed the *timeout*. If an exception
@@ -894,6 +890,10 @@ Timeouts
    .. versionchanged:: 3.11
       Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`.
 
+   .. versionchanged:: 3.12
+      Implemented using :func:`asyncio.timeout`, a coroutine passed as *fut*
+      is no longer wrapped in a :class:`Task` when *timeout* is positive.
+
 
 Waiting primitives
 ==================

Doc/library/tarfile.rst

@@ -1112,6 +1112,10 @@ reused in custom filters:
     paths (in case the name is absolute
     even after stripping slashes, e.g. ``C:/foo`` on Windows).
     This raises :class:`~tarfile.AbsolutePathError`.
+  - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components
+    using :func:`os.path.normpath`.
+    Note that this removes internal ``..`` components, which may change the
+    meaning of the name if it traverses symbolic links.
   - :ref:`Refuse <tarfile-extraction-refuse>` to extract files whose absolute
     path (after following symlinks) would end up outside the destination.
     This raises :class:`~tarfile.OutsideDestinationError`.
@@ -1120,6 +1124,10 @@ reused in custom filters:
 
   Return the modified ``TarInfo`` member.
 
+  .. versionchanged:: next
+
+     Filenames containing ``..`` components are now normalized.
+
 .. function:: data_filter(member, path)
 
   Implements the ``'data'`` filter.

Include/internal/pycore_interpframe_structs.h

@@ -66,9 +66,9 @@ struct _PyInterpreterFrame {
     PyObject *prefix##_qualname;                                            \
     _PyErr_StackItem prefix##_exc_state;                                    \
     PyObject *prefix##_origin_or_finalizer;                                 \
-    char prefix##_hooks_inited;                                             \
-    char prefix##_closed;                                                   \
-    char prefix##_running_async;                                            \
+    int8_t prefix##_hooks_inited;                                           \
+    int8_t prefix##_closed;                                                 \
+    int8_t prefix##_running_async;                                          \
     /* The frame */                                                         \
     int8_t prefix##_frame_state;                                            \
     _PyInterpreterFrame prefix##_iframe;                                    \

Lib/asyncio/streams.py

@@ -239,7 +239,17 @@ def connection_made(self, transport):
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
             writer = StreamWriter(transport, self, reader, self._loop)
-            res = self._client_connected_cb(reader, writer)
+            try:
+                res = self._client_connected_cb(reader, writer)
+            except Exception as exc:
+                self._loop.call_exception_handler({
+                    'message': 'Unhandled exception in client_connected_cb',
+                    'exception': exc,
+                    'transport': transport,
+                })
+                transport.close()
+                self._strong_reader = None
+                return
             if coroutines.iscoroutine(res):
                 def callback(task):
                     if task.cancelled():

Lib/asyncio/tasks.py

@@ -440,15 +440,13 @@ def _release_waiter(waiter, *args):
 async def wait_for(fut, timeout):
     """Wait for the single Future or coroutine to complete, with timeout.
 
-    Coroutine will be wrapped in Task.
-
     Returns result of the Future or coroutine.  When a timeout occurs,
-    it cancels the task and raises TimeoutError.  To avoid the task
-    cancellation, wrap it in shield().
+    it cancels fut and raises TimeoutError.  To prevent fut from being
+    cancelled, wrap it in shield().
 
-    If the wait is cancelled, the task is also cancelled.
+    If the wait is cancelled, fut is also cancelled.
 
-    If the task suppresses the cancellation and returns a value instead,
+    If fut suppresses the cancellation and returns a value instead,
     that value is returned.
 
     This function is a coroutine.

Lib/platform.py

@@ -427,11 +427,16 @@ def _win32_ver(version, csd, ptype):
 
     winver = getwindowsversion()
     is_client = (getattr(winver, 'product_type', 1) == 1)
-    try:
-        version = _syscmd_ver()[2]
-        major, minor, build = map(int, version.split('.'))
-    except ValueError:
-        major, minor, build = winver.platform_version or winver[:3]
+
+    if winver.device_family == "Desktop":
+        try:
+            version = _syscmd_ver()[2]
+            major, minor, build = map(int, version.split('.'))
+        except ValueError:
+            major, minor, build = winver.platform_version or winver[:3]
+            version = '{0}.{1}.{2}'.format(major, minor, build)
+    else:
+        major, minor, build = winver[:3]
         version = '{0}.{1}.{2}'.format(major, minor, build)
 
     # getwindowsversion() reflect the compatibility mode Python is

Lib/tarfile.py

@@ -833,6 +833,13 @@ def _get_filtered_attrs(member, dest_path, for_data=True):
         # For example, 'C:/foo' on Windows.
         raise AbsolutePathError(member)
     # Ensure we stay in the destination
+    if '..' in name.replace(os.sep, '/').split('/'):
+        # Directories are created from the name as given, so a name that
+        # leaves the destination part-way through would create them
+        # outside it even if the resolved path stays inside.
+        normalized = os.path.normpath(name)
+        if normalized != name:
+            name = new_attrs['name'] = normalized
     target_path = os.path.realpath(os.path.join(dest_path, name),
                                    strict=os.path.ALLOW_MISSING)
     if os.path.commonpath([target_path, dest_path]) != dest_path:

Lib/test/support/__init__.py

@@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None:
 
 
 def _memory_watchdog(pid):
-    """Return a function printing the memory usage of process *pid*."""
+    """Return a function printing the memory usage of process *pid*.
+
+    The largest value it saw is kept in its ``peak`` attribute.
+    """
     # Imported here: test.support does not depend on test.libregrtest.
     from test.libregrtest.utils import get_process_memory_usage
 
     def watch():
         mem = get_process_memory_usage(pid)
         if mem is not None:
+            watch.peak = max(watch.peak, mem)
             print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
                   flush=True)
+    watch.peak = 0
     return watch
 
 
@@ -1333,7 +1338,23 @@ def wrapper(self):
                 qualname = f'{cls.__qualname__}.{f.__name__}'
                 proc = isolation._start_test(cls.__module__, qualname)
                 watchdog = _memory_watchdog(proc.pid) if verbose else None
-                isolation._replay_test(self, *proc.wait(tick=watchdog))
+                payload, output, returncode = proc.wait(tick=watchdog)
+                if watchdog:
+                    # The subprocess measures its own peak exactly.  What the
+                    # parent sampled is only a lower bound.
+                    maxrss = payload and payload.get('maxrss')
+                    peak = maxrss or watchdog.peak
+                    if peak:
+                        print(f" ... peak memory use: "
+                              f"{peak / (1024 ** 3):.1f} GiB"
+                              f"{'' if maxrss else ' or more'}", flush=True)
+                    majflt = payload and payload.get('majflt')
+                    if majflt:
+                        # The test did not fit in memory, so its timing means
+                        # little.
+                        print(f" ... {majflt} major page faults: the test "
+                              f"waited for the disk", flush=True)
+                isolation._replay_test(self, payload, output, returncode)
                 return
 
             return f(self, maxsize)

Lib/test/support/subprocess_runner.py

@@ -72,7 +72,45 @@ def _outcome(kind, test, detail):
              for t, tb in result.expectedFailures]
 outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped]
 
-payload = {'outcomes': outcomes, 'durations': result.id_durations}
+def _usage():
+    """What this process used: peak resident set size in bytes, and the
+    number of major page faults it took, either of which can be None.
+
+    A major page fault is served from disk, so a non-zero count means swapping.
+
+    The modules are imported here, after the test has run, so that the test
+    does not see them.
+    """
+    try:
+        import resource
+    except ImportError:
+        pass
+    else:
+        usage = resource.getrusage(resource.RUSAGE_SELF)
+        # Solaris and illumos leave these fields at 0, which no live process
+        # has, so treat it as "not supported".
+        if not usage.ru_maxrss:
+            return {'maxrss': None, 'majflt': None}
+        # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs.
+        maxrss = usage.ru_maxrss
+        return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024,
+                'majflt': usage.ru_majflt}
+    try:
+        import os
+        import _winapi
+        handle = _winapi.OpenProcess(
+            _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid())
+    except (ImportError, OSError):
+        return {'maxrss': None, 'majflt': None}
+    try:
+        info = _winapi.GetProcessMemoryInfo(handle)
+    finally:
+        _winapi.CloseHandle(handle)
+    # PageFaultCount counts all faults, not only the ones served from disk.
+    return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None}
+
+
+payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()}
 with open(outfile, 'wb') as f:
     marshal.dump(payload, f)
 

Lib/test/test_asyncgen.py

@@ -619,6 +619,33 @@ async def agenfn():
         with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"):
             gen.close()
 
+    def test_async_gen_athrow_send_non_none(self):
+        # gh-120321: sending a non-None value to a just-started athrow()
+        # awaitable must not claim the generator, so the generator stays
+        # usable and the awaitable can still be awaited afterwards.
+        class MyExc(Exception):
+            pass
+
+        async def agenfn():
+            try:
+                yield 1
+            except MyExc:
+                yield 2
+
+        agen = agenfn()
+        with self.assertRaises(StopIteration):
+            agen.asend(None).send(None)
+
+        gen = agen.athrow(MyExc)
+        with self.assertRaisesRegex(RuntimeError, "non-None value"):
+            gen.send(42)
+        self.assertFalse(agen.ag_running)
+
+        # The awaitable is still in its initial state and works normally.
+        with self.assertRaises(StopIteration) as cm:
+            gen.send(None)
+        self.assertEqual(cm.exception.value, 2)
+
 
 class AsyncGenAsyncioTest(unittest.TestCase):
 
@@ -1950,6 +1977,41 @@ class MyException(Exception):
         ):
             nxt.throw(MyException)
 
+    def test_async_gen_send_same_athrow_coro_after_completion(self):
+        # gh-120321: an athrow() awaitable that needs more than one send()
+        # to complete must be closed on completion; sending to it again
+        # must raise instead of resuming the generator.
+        class YieldOnce:
+            def __await__(self):
+                yield
+
+        async def async_iterate():
+            try:
+                yield 1
+            except ValueError:
+                await YieldOnce()
+            yield 2
+
+        it = async_iterate()
+        with self.assertRaises(StopIteration):
+            it.__anext__().send(None)
+
+        nxt = it.athrow(ValueError)
+        # The exception handler suspends before the operation completes.
+        nxt.send(None)
+        with self.assertRaises(StopIteration) as cm:
+            nxt.send(None)
+        self.assertEqual(cm.exception.value, 2)
+
+        with self.assertRaisesRegex(
+            RuntimeError,
+            r"cannot reuse already awaited aclose\(\)/athrow\(\)"
+        ):
+            nxt.send(None)
+
+        with self.assertRaises(StopIteration):
+            it.aclose().send(None)
+
     def test_async_gen_aclose_twice_with_different_coros(self):
         # Regression test for https://bugs.python.org/issue39606
         async def async_iterate():