diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 72ffc2a0b760ea..837e6b0b8fa459 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -835,7 +835,7 @@ The :mod:`!test.support` module defines the following functions: the trace function. -.. decorator:: bigmemtest(size, memuse, dry_run=True) +.. decorator:: bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True) Decorator for bigmem tests. @@ -849,6 +849,12 @@ The :mod:`!test.support` module defines the following functions: method may be less than the requested value. If *dry_run* is ``False``, it means the test doesn't support dummy runs when ``-M`` is not specified. + A test which really allocates the memory it asks for runs in a separate + process, whose address space is limited to what the test declares plus a + margin. + Set *limit_address_space* to ``False`` for a test which reserves much more + address space than it uses, for example one which starts many threads. + .. decorator:: bigaddrspacetest diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index be71575a6ea06a..f0bd8bd0d5fdcb 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1270,25 +1270,67 @@ def set_memlimit(limit: str) -> None: max_memuse = memlimit -def _memory_watchdog(pid): - """Return a function printing the memory usage of process *pid*. +def _memory_limit(nbytes): + """How much memory a test declaring *nbytes* may use. - The largest value it saw is kept in its ``peak`` attribute. + The interpreter itself reserves about 250 MiB, whatever the test asks for. + """ + return int(nbytes) + 512 * _1M + + +def _limit_address_space(nbytes): + """Limit the address space of this process to what a test may use. + + A test which uses much more memory than it declares then fails with a + MemoryError instead of making the machine swap. + """ + if check_sanitizer(address=True): + # AddressSanitizer reserves terabytes of address space for its shadow + # memory, so any limit stops the interpreter from starting. + return + if sys.platform == 'darwin': + # macOS reserves much more address space than it uses. + return + try: + import resource + rlimit = resource.RLIMIT_AS + except (ImportError, AttributeError): + return + limit = _memory_limit(nbytes) + soft, hard = resource.getrlimit(rlimit) + for current in soft, hard: + if current != resource.RLIM_INFINITY: + limit = min(limit, current) + resource.setrlimit(rlimit, (limit, hard)) + + +def _memory_watchdog(proc, limit): + """Return a function watching the memory used by the test in *proc*. + + It reports the usage in verbose mode, keeps the largest value it saw in + its ``peak`` attribute, and kills the test if it uses more than *limit* + bytes. This is the only limit where the address space cannot be limited. """ # Imported here: test.support does not depend on test.libregrtest. from test.libregrtest.utils import get_process_memory_usage def watch(): - mem = get_process_memory_usage(pid) - if mem is not None: - watch.peak = max(watch.peak, mem) + mem = get_process_memory_usage(proc.pid) + if mem is None: + return + watch.peak = max(watch.peak, mem) + if verbose: print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB", flush=True) + if limit is not None and mem > limit: + watch.exceeded = mem + proc.kill() watch.peak = 0 + watch.exceeded = None return watch -def bigmemtest(size, memuse, dry_run=True): +def bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True): """Decorator for bigmem tests. 'size' is a requested size for the test (in arbitrary, test-interpreted @@ -1304,6 +1346,12 @@ def bigmemtest(size, memuse, dry_run=True): A test that actually allocates the requested memory (that is, one run with -M) runs in a subprocess, so that the memory it uses and the address space it fragments are released when it ends. A dummy run stays in the process. + + The address space of that subprocess is limited to what the test declares + plus a margin, so that a test which uses much more memory than it declares + fails instead of making the machine swap. Pass 'limit_address_space' as + false for a test which reserves much more address space than it uses, for + example one which starts many threads. """ def decorator(f): from test.support import isolation @@ -1337,9 +1385,17 @@ def wrapper(self): cls = type(self) qualname = f'{cls.__qualname__}.{f.__name__}' proc = isolation._start_test(cls.__module__, qualname) - watchdog = _memory_watchdog(proc.pid) if verbose else None + # Watched even if the address space is not limited: this + # counts the memory really used. + watchdog = _memory_watchdog(proc, + _memory_limit(size * memuse)) payload, output, returncode = proc.wait(tick=watchdog) - if watchdog: + if watchdog.exceeded: + raise AssertionError( + f'the test used {watchdog.exceeded / _1G:.1f} GiB, ' + f'more than the {size * memuse / _1G:.1f} GiB ' + f'it declares') + if verbose: # The subprocess measures its own peak exactly. What the # parent sampled is only a lower bound. maxrss = payload and payload.get('maxrss') @@ -1347,16 +1403,22 @@ def wrapper(self): if peak: print(f" ... peak memory use: " f"{peak / (1024 ** 3):.1f} GiB" - f"{'' if maxrss else ' or more'}", flush=True) + f"{'' if maxrss else ' or more'}", + flush=True) majflt = payload and payload.get('majflt') if majflt: - # The test did not fit in memory, so its timing means - # little. + # The test did not fit in memory, so its + # timing means little. print(f" ... {majflt} major page faults: the test " f"waited for the disk", flush=True) isolation._replay_test(self, payload, output, returncode) return + if (real_max_memuse and limit_address_space + and isolation.runningInSubprocess): + # Only in the subprocess: the limit is never lifted. + _limit_address_space(size * memuse) + return f(self, maxsize) wrapper.size = size diff --git a/Lib/test/support/isolation.py b/Lib/test/support/isolation.py index 3cfd406b0f2f0e..259d26959746aa 100644 --- a/Lib/test/support/isolation.py +++ b/Lib/test/support/isolation.py @@ -123,6 +123,10 @@ def __init__(self, proc, result_path): def pid(self): return self._proc.pid + def kill(self): + """Kill the test, for example when it uses too much memory.""" + self._proc.kill() + def wait(self, timeout=None, tick=None, interval=1.0): """Wait for the test to finish, calling *tick* every *interval* seconds. diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 3c09cdb8640890..d8b635dd391041 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -373,7 +373,7 @@ def test_split_small(self, size): # suffer for the list size. (Otherwise, it'd cost another 48 times # size in bytes!) Nevertheless, a list of size takes # 8*size bytes. - @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size) + @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 3 + pointer_size) def test_split_large(self, size): _ = self.from_latin1 s = _(' a') * size + _(' ') @@ -936,11 +936,11 @@ def basic_test_repr(self, size): self.assertEqual(s[:10], '(False, Fa') self.assertEqual(s[-10:], 'se, False)') - @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_large(self, size): return self.basic_test_repr(size) @@ -1115,11 +1115,11 @@ def basic_test_repr(self, size): self.assertEqual(s[-10:], 'se, False]') self.assertEqual(s.count('F'), size) - @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_large(self, size): return self.basic_test_repr(size) diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index 50d2444a4c72d3..eedbbfe1182fad 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -26,7 +26,8 @@ def test_create_many_sequential(self): support.gc_collect() @threading_helper.requires_working_threading() - @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False) + @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False, + limit_address_space=False) def test_create_many_threaded(self, size): alive = [] start = threading.Event() @@ -43,7 +44,8 @@ def task(): support.gc_collect() @threading_helper.requires_working_threading() - @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False) + @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False, + limit_address_space=False) def test_many_threads_running_interp_in_other_interp(self, size): start = threading.Event() interp = interpreters.create() diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst new file mode 100644 index 00000000000000..44d6c81d08768b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst @@ -0,0 +1,5 @@ +A test decorated with :func:`~test.support.bigmemtest` now runs with the +address space limited to what it declares plus a margin, so that a test which +uses much more memory than it declares fails instead of making the machine +swap. Pass ``limit_address_space=False`` for a test which reserves much more +address space than it uses.