Merge branch 'main' into wasi-no-tracebacks

Author: brettcannon

Doc/library/tarfile.rst

@@ -1112,6 +1112,10 @@ reused in custom filters:
     paths (in case the name is absolute
     even after stripping slashes, e.g. ``C:/foo`` on Windows).
     This raises :class:`~tarfile.AbsolutePathError`.
+  - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components
+    using :func:`os.path.normpath`.
+    Note that this removes internal ``..`` components, which may change the
+    meaning of the name if it traverses symbolic links.
   - :ref:`Refuse <tarfile-extraction-refuse>` to extract files whose absolute
     path (after following symlinks) would end up outside the destination.
     This raises :class:`~tarfile.OutsideDestinationError`.
@@ -1120,6 +1124,10 @@ reused in custom filters:
 
   Return the modified ``TarInfo`` member.
 
+  .. versionchanged:: next
+
+     Filenames containing ``..`` components are now normalized.
+
 .. function:: data_filter(member, path)
 
   Implements the ``'data'`` filter.

Include/internal/pycore_interpframe_structs.h

@@ -66,9 +66,9 @@ struct _PyInterpreterFrame {
     PyObject *prefix##_qualname;                                            \
     _PyErr_StackItem prefix##_exc_state;                                    \
     PyObject *prefix##_origin_or_finalizer;                                 \
-    char prefix##_hooks_inited;                                             \
-    char prefix##_closed;                                                   \
-    char prefix##_running_async;                                            \
+    int8_t prefix##_hooks_inited;                                           \
+    int8_t prefix##_closed;                                                 \
+    int8_t prefix##_running_async;                                          \
     /* The frame */                                                         \
     int8_t prefix##_frame_state;                                            \
     _PyInterpreterFrame prefix##_iframe;                                    \

Lib/tarfile.py

@@ -833,6 +833,13 @@ def _get_filtered_attrs(member, dest_path, for_data=True):
         # For example, 'C:/foo' on Windows.
         raise AbsolutePathError(member)
     # Ensure we stay in the destination
+    if '..' in name.replace(os.sep, '/').split('/'):
+        # Directories are created from the name as given, so a name that
+        # leaves the destination part-way through would create them
+        # outside it even if the resolved path stays inside.
+        normalized = os.path.normpath(name)
+        if normalized != name:
+            name = new_attrs['name'] = normalized
     target_path = os.path.realpath(os.path.join(dest_path, name),
                                    strict=os.path.ALLOW_MISSING)
     if os.path.commonpath([target_path, dest_path]) != dest_path:

Lib/test/support/__init__.py

@@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None:
 
 
 def _memory_watchdog(pid):
-    """Return a function printing the memory usage of process *pid*."""
+    """Return a function printing the memory usage of process *pid*.
+
+    The largest value it saw is kept in its ``peak`` attribute.
+    """
     # Imported here: test.support does not depend on test.libregrtest.
     from test.libregrtest.utils import get_process_memory_usage
 
     def watch():
         mem = get_process_memory_usage(pid)
         if mem is not None:
+            watch.peak = max(watch.peak, mem)
             print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
                   flush=True)
+    watch.peak = 0
     return watch
 
 
@@ -1333,7 +1338,23 @@ def wrapper(self):
                 qualname = f'{cls.__qualname__}.{f.__name__}'
                 proc = isolation._start_test(cls.__module__, qualname)
                 watchdog = _memory_watchdog(proc.pid) if verbose else None
-                isolation._replay_test(self, *proc.wait(tick=watchdog))
+                payload, output, returncode = proc.wait(tick=watchdog)
+                if watchdog:
+                    # The subprocess measures its own peak exactly.  What the
+                    # parent sampled is only a lower bound.
+                    maxrss = payload and payload.get('maxrss')
+                    peak = maxrss or watchdog.peak
+                    if peak:
+                        print(f" ... peak memory use: "
+                              f"{peak / (1024 ** 3):.1f} GiB"
+                              f"{'' if maxrss else ' or more'}", flush=True)
+                    majflt = payload and payload.get('majflt')
+                    if majflt:
+                        # The test did not fit in memory, so its timing means
+                        # little.
+                        print(f" ... {majflt} major page faults: the test "
+                              f"waited for the disk", flush=True)
+                isolation._replay_test(self, payload, output, returncode)
                 return
 
             return f(self, maxsize)

Lib/test/support/subprocess_runner.py

@@ -72,7 +72,45 @@ def _outcome(kind, test, detail):
              for t, tb in result.expectedFailures]
 outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped]
 
-payload = {'outcomes': outcomes, 'durations': result.id_durations}
+def _usage():
+    """What this process used: peak resident set size in bytes, and the
+    number of major page faults it took, either of which can be None.
+
+    A major page fault is served from disk, so a non-zero count means swapping.
+
+    The modules are imported here, after the test has run, so that the test
+    does not see them.
+    """
+    try:
+        import resource
+    except ImportError:
+        pass
+    else:
+        usage = resource.getrusage(resource.RUSAGE_SELF)
+        # Solaris and illumos leave these fields at 0, which no live process
+        # has, so treat it as "not supported".
+        if not usage.ru_maxrss:
+            return {'maxrss': None, 'majflt': None}
+        # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs.
+        maxrss = usage.ru_maxrss
+        return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024,
+                'majflt': usage.ru_majflt}
+    try:
+        import os
+        import _winapi
+        handle = _winapi.OpenProcess(
+            _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid())
+    except (ImportError, OSError):
+        return {'maxrss': None, 'majflt': None}
+    try:
+        info = _winapi.GetProcessMemoryInfo(handle)
+    finally:
+        _winapi.CloseHandle(handle)
+    # PageFaultCount counts all faults, not only the ones served from disk.
+    return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None}
+
+
+payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()}
 with open(outfile, 'wb') as f:
     marshal.dump(payload, f)
 

Lib/test/test_asyncgen.py

@@ -619,6 +619,33 @@ async def agenfn():
         with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"):
             gen.close()
 
+    def test_async_gen_athrow_send_non_none(self):
+        # gh-120321: sending a non-None value to a just-started athrow()
+        # awaitable must not claim the generator, so the generator stays
+        # usable and the awaitable can still be awaited afterwards.
+        class MyExc(Exception):
+            pass
+
+        async def agenfn():
+            try:
+                yield 1
+            except MyExc:
+                yield 2
+
+        agen = agenfn()
+        with self.assertRaises(StopIteration):
+            agen.asend(None).send(None)
+
+        gen = agen.athrow(MyExc)
+        with self.assertRaisesRegex(RuntimeError, "non-None value"):
+            gen.send(42)
+        self.assertFalse(agen.ag_running)
+
+        # The awaitable is still in its initial state and works normally.
+        with self.assertRaises(StopIteration) as cm:
+            gen.send(None)
+        self.assertEqual(cm.exception.value, 2)
+
 
 class AsyncGenAsyncioTest(unittest.TestCase):
 
@@ -1950,6 +1977,41 @@ class MyException(Exception):
         ):
             nxt.throw(MyException)
 
+    def test_async_gen_send_same_athrow_coro_after_completion(self):
+        # gh-120321: an athrow() awaitable that needs more than one send()
+        # to complete must be closed on completion; sending to it again
+        # must raise instead of resuming the generator.
+        class YieldOnce:
+            def __await__(self):
+                yield
+
+        async def async_iterate():
+            try:
+                yield 1
+            except ValueError:
+                await YieldOnce()
+            yield 2
+
+        it = async_iterate()
+        with self.assertRaises(StopIteration):
+            it.__anext__().send(None)
+
+        nxt = it.athrow(ValueError)
+        # The exception handler suspends before the operation completes.
+        nxt.send(None)
+        with self.assertRaises(StopIteration) as cm:
+            nxt.send(None)
+        self.assertEqual(cm.exception.value, 2)
+
+        with self.assertRaisesRegex(
+            RuntimeError,
+            r"cannot reuse already awaited aclose\(\)/athrow\(\)"
+        ):
+            nxt.send(None)
+
+        with self.assertRaises(StopIteration):
+            it.aclose().send(None)
+
     def test_async_gen_aclose_twice_with_different_coros(self):
         # Regression test for https://bugs.python.org/issue39606
         async def async_iterate():