gh-75876: Limit the address space of a bigmem test

Doc/library/test.rst

@@ -835,7 +835,7 @@ The :mod:`!test.support` module defines the following functions:
    the trace function.
 
 
-.. decorator:: bigmemtest(size, memuse, dry_run=True)
+.. decorator:: bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True)
 
    Decorator for bigmem tests.
 
@@ -849,6 +849,12 @@ The :mod:`!test.support` module defines the following functions:
    method may be less than the requested value.  If *dry_run* is ``False``, it
    means the test doesn't support dummy runs when ``-M`` is not specified.
 
+   A test which really allocates the memory it asks for runs in a separate
+   process, whose address space is limited to what the test declares plus a
+   margin.
+   Set *limit_address_space* to ``False`` for a test which reserves much more
+   address space than it uses, for example one which starts many threads.
+
 
 .. decorator:: bigaddrspacetest
 

Lib/test/support/__init__.py

@@ -1270,25 +1270,67 @@ def set_memlimit(limit: str) -> None:
     max_memuse = memlimit
 
 
-def _memory_watchdog(pid):
-    """Return a function printing the memory usage of process *pid*.
+def _memory_limit(nbytes):
+    """How much memory a test declaring *nbytes* may use.
 
-    The largest value it saw is kept in its ``peak`` attribute.
+    The interpreter itself reserves about 250 MiB, whatever the test asks for.
+    """
+    return int(nbytes) + 512 * _1M
+
+
+def _limit_address_space(nbytes):
+    """Limit the address space of this process to what a test may use.
+
+    A test which uses much more memory than it declares then fails with a
+    MemoryError instead of making the machine swap.
+    """
+    if check_sanitizer(address=True):
+        # AddressSanitizer reserves terabytes of address space for its shadow
+        # memory, so any limit stops the interpreter from starting.
+        return
+    if sys.platform == 'darwin':
+        # macOS reserves much more address space than it uses.
+        return
+    try:
+        import resource
+        rlimit = resource.RLIMIT_AS
+    except (ImportError, AttributeError):
+        return
+    limit = _memory_limit(nbytes)
+    soft, hard = resource.getrlimit(rlimit)
+    for current in soft, hard:
+        if current != resource.RLIM_INFINITY:
+            limit = min(limit, current)
+    resource.setrlimit(rlimit, (limit, hard))
+
+
+def _memory_watchdog(proc, limit):
+    """Return a function watching the memory used by the test in *proc*.
+
+    It reports the usage in verbose mode, keeps the largest value it saw in
+    its ``peak`` attribute, and kills the test if it uses more than *limit*
+    bytes.  This is the only limit where the address space cannot be limited.
     """
     # Imported here: test.support does not depend on test.libregrtest.
     from test.libregrtest.utils import get_process_memory_usage
 
     def watch():
-        mem = get_process_memory_usage(pid)
-        if mem is not None:
-            watch.peak = max(watch.peak, mem)
+        mem = get_process_memory_usage(proc.pid)
+        if mem is None:
+            return
+        watch.peak = max(watch.peak, mem)
+        if verbose:
             print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
                   flush=True)
+        if limit is not None and mem > limit:
+            watch.exceeded = mem
+            proc.kill()
     watch.peak = 0
+    watch.exceeded = None
     return watch
 
 
-def bigmemtest(size, memuse, dry_run=True):
+def bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True):
     """Decorator for bigmem tests.
 
     'size' is a requested size for the test (in arbitrary, test-interpreted
@@ -1304,6 +1346,12 @@ def bigmemtest(size, memuse, dry_run=True):
     A test that actually allocates the requested memory (that is, one run with
     -M) runs in a subprocess, so that the memory it uses and the address space
     it fragments are released when it ends.  A dummy run stays in the process.
+
+    The address space of that subprocess is limited to what the test declares
+    plus a margin, so that a test which uses much more memory than it declares
+    fails instead of making the machine swap.  Pass 'limit_address_space' as
+    false for a test which reserves much more address space than it uses, for
+    example one which starts many threads.
     """
     def decorator(f):
         from test.support import isolation
@@ -1337,26 +1385,40 @@ def wrapper(self):
                 cls = type(self)
                 qualname = f'{cls.__qualname__}.{f.__name__}'
                 proc = isolation._start_test(cls.__module__, qualname)
-                watchdog = _memory_watchdog(proc.pid) if verbose else None
+                # Watched even if the address space is not limited: this
+                # counts the memory really used.
+                watchdog = _memory_watchdog(proc,
+                                            _memory_limit(size * memuse))
                 payload, output, returncode = proc.wait(tick=watchdog)
-                if watchdog:
+                if watchdog.exceeded:
+                    raise AssertionError(
+                        f'the test used {watchdog.exceeded / _1G:.1f} GiB, '
+                        f'more than the {size * memuse / _1G:.1f} GiB '
+                        f'it declares')
+                if verbose:
                     # The subprocess measures its own peak exactly.  What the
                     # parent sampled is only a lower bound.
                     maxrss = payload and payload.get('maxrss')
                     peak = maxrss or watchdog.peak
                     if peak:
                         print(f" ... peak memory use: "
                               f"{peak / (1024 ** 3):.1f} GiB"
-                              f"{'' if maxrss else ' or more'}", flush=True)
+                              f"{'' if maxrss else ' or more'}",
+                              flush=True)
                     majflt = payload and payload.get('majflt')
                     if majflt:
-                        # The test did not fit in memory, so its timing means
-                        # little.
+                        # The test did not fit in memory, so its
+                        # timing means little.
                         print(f" ... {majflt} major page faults: the test "
                               f"waited for the disk", flush=True)
                 isolation._replay_test(self, payload, output, returncode)
                 return
 
+            if (real_max_memuse and limit_address_space
+                    and isolation.runningInSubprocess):
+                # Only in the subprocess: the limit is never lifted.
+                _limit_address_space(size * memuse)
+
             return f(self, maxsize)
 
         wrapper.size = size

Lib/test/support/isolation.py

@@ -123,6 +123,10 @@ def __init__(self, proc, result_path):
     def pid(self):
         return self._proc.pid
 
+    def kill(self):
+        """Kill the test, for example when it uses too much memory."""
+        self._proc.kill()
+
     def wait(self, timeout=None, tick=None, interval=1.0):
         """Wait for the test to finish, calling *tick* every *interval* seconds.
 

Lib/test/test_bigmem.py

@@ -373,7 +373,7 @@ def test_split_small(self, size):
     # suffer for the list size. (Otherwise, it'd cost another 48 times
     # size in bytes!) Nevertheless, a list of size takes
     # 8*size bytes.
-    @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size)
+    @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 3 + pointer_size)
     def test_split_large(self, size):
         _ = self.from_latin1
         s = _(' a') * size + _(' ')
@@ -936,11 +936,11 @@ def basic_test_repr(self, size):
         self.assertEqual(s[:10], '(False, Fa')
         self.assertEqual(s[-10:], 'se, False)')
 
-    @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
+    @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
+    @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 
@@ -1115,11 +1115,11 @@ def basic_test_repr(self, size):
         self.assertEqual(s[-10:], 'se, False]')
         self.assertEqual(s.count('F'), size)
 
-    @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
+    @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
+    @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 

Lib/test/test_interpreters/test_stress.py

@@ -26,7 +26,8 @@ def test_create_many_sequential(self):
         support.gc_collect()
 
     @threading_helper.requires_working_threading()
-    @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False)
+    @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False,
+                        limit_address_space=False)
     def test_create_many_threaded(self, size):
         alive = []
         start = threading.Event()
@@ -43,7 +44,8 @@ def task():
         support.gc_collect()
 
     @threading_helper.requires_working_threading()
-    @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False)
+    @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False,
+                        limit_address_space=False)
     def test_many_threads_running_interp_in_other_interp(self, size):
         start = threading.Event()
         interp = interpreters.create()

Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst

@@ -0,0 +1,5 @@
+A test decorated with :func:`~test.support.bigmemtest` now runs with the
+address space limited to what it declares plus a margin, so that a test which
+uses much more memory than it declares fails instead of making the machine
+swap.  Pass ``limit_address_space=False`` for a test which reserves much more
+address space than it uses.