Merge branch 'main' into bigmemtest-rlimit

The memory watchdog now both reports the peak (GH-156024) and kills a test
which uses more memory than it declares.

Author: serhiy-storchaka

Doc/library/asyncio-task.rst

@@ -843,17 +843,13 @@ Timeouts
    Wait for the *fut* :ref:`awaitable <asyncio-awaitables>`
    to complete with a timeout.
 
-   If *fut* is a coroutine it is automatically scheduled as a Task.
-
    *timeout* can either be ``None`` or a float or int number of seconds
    to wait for.  If *timeout* is ``None``, block until the future
    completes.
 
-   If a timeout occurs, it cancels the task and raises
-   :exc:`TimeoutError`.
+   If a timeout occurs, it cancels *fut* and raises :exc:`TimeoutError`.
 
-   To avoid the task :meth:`cancellation <Task.cancel>`,
-   wrap it in :func:`shield`.
+   To prevent *fut* from being cancelled, wrap it in :func:`shield`.
 
    The function will wait until the future is actually cancelled,
    so the total wait time may exceed the *timeout*. If an exception
@@ -894,6 +890,10 @@ Timeouts
    .. versionchanged:: 3.11
       Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`.
 
+   .. versionchanged:: 3.12
+      Implemented using :func:`asyncio.timeout`, a coroutine passed as *fut*
+      is no longer wrapped in a :class:`Task` when *timeout* is positive.
+
 
 Waiting primitives
 ==================

Doc/library/tarfile.rst

@@ -1112,6 +1112,10 @@ reused in custom filters:
     paths (in case the name is absolute
     even after stripping slashes, e.g. ``C:/foo`` on Windows).
     This raises :class:`~tarfile.AbsolutePathError`.
+  - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components
+    using :func:`os.path.normpath`.
+    Note that this removes internal ``..`` components, which may change the
+    meaning of the name if it traverses symbolic links.
   - :ref:`Refuse <tarfile-extraction-refuse>` to extract files whose absolute
     path (after following symlinks) would end up outside the destination.
     This raises :class:`~tarfile.OutsideDestinationError`.
@@ -1120,6 +1124,10 @@ reused in custom filters:
 
   Return the modified ``TarInfo`` member.
 
+  .. versionchanged:: next
+
+     Filenames containing ``..`` components are now normalized.
+
 .. function:: data_filter(member, path)
 
   Implements the ``'data'`` filter.

Doc/tutorial/stdlib.rst

@@ -36,7 +36,7 @@ aids for working with large modules like :mod:`os`::
    <returns an extensive manual page created from the module's docstrings>
 
 For daily file and directory management tasks, the :mod:`shutil` module provides
-a higher level interface that is easier to use::
+a higher-level interface that is easier to use::
 
    >>> import shutil
    >>> shutil.copyfile('data.db', 'archive.db')
@@ -63,7 +63,7 @@ wildcard searches::
 Command-line arguments
 ======================
 
-Common utility scripts often need to process command line arguments. These
+Common utility scripts often need to process command-line arguments. These
 arguments are stored in the :mod:`sys` module's *argv* attribute as a list.  For
 instance, let's take the following :file:`demo.py` file::
 
@@ -77,7 +77,7 @@ line::
    ['demo.py', 'one', 'two', 'three']
 
 The :mod:`argparse` module provides a more sophisticated mechanism to process
-command line arguments.  The following script extracts one or more filenames
+command-line arguments.  The following script extracts one or more filenames
 and an optional number of lines to be displayed::
 
     import argparse

Include/internal/pycore_interpframe_structs.h

@@ -66,9 +66,9 @@ struct _PyInterpreterFrame {
     PyObject *prefix##_qualname;                                            \
     _PyErr_StackItem prefix##_exc_state;                                    \
     PyObject *prefix##_origin_or_finalizer;                                 \
-    char prefix##_hooks_inited;                                             \
-    char prefix##_closed;                                                   \
-    char prefix##_running_async;                                            \
+    int8_t prefix##_hooks_inited;                                           \
+    int8_t prefix##_closed;                                                 \
+    int8_t prefix##_running_async;                                          \
     /* The frame */                                                         \
     int8_t prefix##_frame_state;                                            \
     _PyInterpreterFrame prefix##_iframe;                                    \

Lib/asyncio/streams.py

@@ -239,7 +239,17 @@ def connection_made(self, transport):
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
             writer = StreamWriter(transport, self, reader, self._loop)
-            res = self._client_connected_cb(reader, writer)
+            try:
+                res = self._client_connected_cb(reader, writer)
+            except Exception as exc:
+                self._loop.call_exception_handler({
+                    'message': 'Unhandled exception in client_connected_cb',
+                    'exception': exc,
+                    'transport': transport,
+                })
+                transport.close()
+                self._strong_reader = None
+                return
             if coroutines.iscoroutine(res):
                 def callback(task):
                     if task.cancelled():

Lib/asyncio/tasks.py

@@ -440,15 +440,13 @@ def _release_waiter(waiter, *args):
 async def wait_for(fut, timeout):
     """Wait for the single Future or coroutine to complete, with timeout.
 
-    Coroutine will be wrapped in Task.
-
     Returns result of the Future or coroutine.  When a timeout occurs,
-    it cancels the task and raises TimeoutError.  To avoid the task
-    cancellation, wrap it in shield().
+    it cancels fut and raises TimeoutError.  To prevent fut from being
+    cancelled, wrap it in shield().
 
-    If the wait is cancelled, the task is also cancelled.
+    If the wait is cancelled, fut is also cancelled.
 
-    If the task suppresses the cancellation and returns a value instead,
+    If fut suppresses the cancellation and returns a value instead,
     that value is returned.
 
     This function is a coroutine.

Lib/platform.py

@@ -427,11 +427,16 @@ def _win32_ver(version, csd, ptype):
 
     winver = getwindowsversion()
     is_client = (getattr(winver, 'product_type', 1) == 1)
-    try:
-        version = _syscmd_ver()[2]
-        major, minor, build = map(int, version.split('.'))
-    except ValueError:
-        major, minor, build = winver.platform_version or winver[:3]
+
+    if winver.device_family == "Desktop":
+        try:
+            version = _syscmd_ver()[2]
+            major, minor, build = map(int, version.split('.'))
+        except ValueError:
+            major, minor, build = winver.platform_version or winver[:3]
+            version = '{0}.{1}.{2}'.format(major, minor, build)
+    else:
+        major, minor, build = winver[:3]
         version = '{0}.{1}.{2}'.format(major, minor, build)
 
     # getwindowsversion() reflect the compatibility mode Python is

Lib/tarfile.py

@@ -833,6 +833,13 @@ def _get_filtered_attrs(member, dest_path, for_data=True):
         # For example, 'C:/foo' on Windows.
         raise AbsolutePathError(member)
     # Ensure we stay in the destination
+    if '..' in name.replace(os.sep, '/').split('/'):
+        # Directories are created from the name as given, so a name that
+        # leaves the destination part-way through would create them
+        # outside it even if the resolved path stays inside.
+        normalized = os.path.normpath(name)
+        if normalized != name:
+            name = new_attrs['name'] = normalized
     target_path = os.path.realpath(os.path.join(dest_path, name),
                                    strict=os.path.ALLOW_MISSING)
     if os.path.commonpath([target_path, dest_path]) != dest_path:

Lib/test/support/__init__.py

@@ -1307,9 +1307,9 @@ def _limit_address_space(nbytes):
 def _memory_watchdog(proc, limit):
     """Return a function watching the memory used by the test in *proc*.
 
-    It reports the usage in verbose mode, and kills the test if it uses more
-    than *limit* bytes.  This is the only limit where the address space cannot
-    be limited.
+    It reports the usage in verbose mode, keeps the largest value it saw in
+    its ``peak`` attribute, and kills the test if it uses more than *limit*
+    bytes.  This is the only limit where the address space cannot be limited.
     """
     # Imported here: test.support does not depend on test.libregrtest.
     from test.libregrtest.utils import get_process_memory_usage
@@ -1318,12 +1318,14 @@ def watch():
         mem = get_process_memory_usage(proc.pid)
         if mem is None:
             return
+        watch.peak = max(watch.peak, mem)
         if verbose:
             print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
                   flush=True)
         if limit is not None and mem > limit:
             watch.exceeded = mem
             proc.kill()
+    watch.peak = 0
     watch.exceeded = None
     return watch
 
@@ -1393,6 +1395,22 @@ def wrapper(self):
                         f'the test used {watchdog.exceeded / _1G:.1f} GiB, '
                         f'more than the {size * memuse / _1G:.1f} GiB '
                         f'it declares')
+                if verbose:
+                    # The subprocess measures its own peak exactly.  What the
+                    # parent sampled is only a lower bound.
+                    maxrss = payload and payload.get('maxrss')
+                    peak = maxrss or watchdog.peak
+                    if peak:
+                        print(f" ... peak memory use: "
+                              f"{peak / (1024 ** 3):.1f} GiB"
+                              f"{'' if maxrss else ' or more'}",
+                              flush=True)
+                    majflt = payload and payload.get('majflt')
+                    if majflt:
+                        # The test did not fit in memory, so its
+                        # timing means little.
+                        print(f" ... {majflt} major page faults: the test "
+                              f"waited for the disk", flush=True)
                 isolation._replay_test(self, payload, output, returncode)
                 return
 

Lib/test/support/subprocess_runner.py

@@ -72,7 +72,45 @@ def _outcome(kind, test, detail):
              for t, tb in result.expectedFailures]
 outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped]
 
-payload = {'outcomes': outcomes, 'durations': result.id_durations}
+def _usage():
+    """What this process used: peak resident set size in bytes, and the
+    number of major page faults it took, either of which can be None.
+
+    A major page fault is served from disk, so a non-zero count means swapping.
+
+    The modules are imported here, after the test has run, so that the test
+    does not see them.
+    """
+    try:
+        import resource
+    except ImportError:
+        pass
+    else:
+        usage = resource.getrusage(resource.RUSAGE_SELF)
+        # Solaris and illumos leave these fields at 0, which no live process
+        # has, so treat it as "not supported".
+        if not usage.ru_maxrss:
+            return {'maxrss': None, 'majflt': None}
+        # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs.
+        maxrss = usage.ru_maxrss
+        return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024,
+                'majflt': usage.ru_majflt}
+    try:
+        import os
+        import _winapi
+        handle = _winapi.OpenProcess(
+            _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid())
+    except (ImportError, OSError):
+        return {'maxrss': None, 'majflt': None}
+    try:
+        info = _winapi.GetProcessMemoryInfo(handle)
+    finally:
+        _winapi.CloseHandle(handle)
+    # PageFaultCount counts all faults, not only the ones served from disk.
+    return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None}
+
+
+payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()}
 with open(outfile, 'wb') as f:
     marshal.dump(payload, f)