From 9043fa5e5966c36570971c2eff0d30fa3ac5e103 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Aug 2026 22:22:49 +0300 Subject: [PATCH 1/4] gh-75876: Limit the address space of a bigmem test A test run with -M now runs in a subprocess whose address space is limited to what it declares plus 512 MiB. A test which uses much more memory than it declares fails with a MemoryError instead of making the machine swap. Pass limit_address_space=False for a test which reserves much more address space than it uses. The two threaded tests in test_interpreters do: glibc reserves an arena per thread, up to 8 per CPU, and this alone exceeds the declared size on a machine with many cores. --- Doc/library/test.rst | 8 ++++- Lib/test/support/__init__.py | 34 ++++++++++++++++++- Lib/test/test_interpreters/test_stress.py | 6 ++-- ...6-08-18-19-36-04.gh-issue-75876.5AhLHL.rst | 5 +++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 72ffc2a0b760ead..837e6b0b8fa4595 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -835,7 +835,7 @@ The :mod:`!test.support` module defines the following functions: the trace function. -.. decorator:: bigmemtest(size, memuse, dry_run=True) +.. decorator:: bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True) Decorator for bigmem tests. @@ -849,6 +849,12 @@ The :mod:`!test.support` module defines the following functions: method may be less than the requested value. If *dry_run* is ``False``, it means the test doesn't support dummy runs when ``-M`` is not specified. + A test which really allocates the memory it asks for runs in a separate + process, whose address space is limited to what the test declares plus a + margin. + Set *limit_address_space* to ``False`` for a test which reserves much more + address space than it uses, for example one which starts many threads. + .. decorator:: bigaddrspacetest diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 31e5508dd9b7907..09be2f1d55418b8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1270,6 +1270,27 @@ def set_memlimit(limit: str) -> None: max_memuse = memlimit +def _limit_address_space(nbytes): + """Limit the address space of this process to *nbytes* plus a margin. + + A test which uses much more memory than it declares then fails with a + MemoryError instead of making the machine swap. + """ + try: + import resource + rlimit = resource.RLIMIT_AS + except (ImportError, AttributeError): + return + # The margin does not grow with the declared size: the interpreter itself + # reserves about 250 MiB, whatever the test asks for. + limit = int(nbytes) + 512 * _1M + soft, hard = resource.getrlimit(rlimit) + for current in soft, hard: + if current != resource.RLIM_INFINITY: + limit = min(limit, current) + resource.setrlimit(rlimit, (limit, hard)) + + def _memory_watchdog(pid): """Return a function printing the memory usage of process *pid*.""" # Imported here: test.support does not depend on test.libregrtest. @@ -1283,7 +1304,7 @@ def watch(): return watch -def bigmemtest(size, memuse, dry_run=True): +def bigmemtest(size, memuse, dry_run=True, *, limit_address_space=True): """Decorator for bigmem tests. 'size' is a requested size for the test (in arbitrary, test-interpreted @@ -1299,6 +1320,12 @@ def bigmemtest(size, memuse, dry_run=True): A test that actually allocates the requested memory (that is, one run with -M) runs in a subprocess, so that the memory it uses and the address space it fragments are released when it ends. A dummy run stays in the process. + + The address space of that subprocess is limited to what the test declares + plus a margin, so that a test which uses much more memory than it declares + fails instead of making the machine swap. Pass 'limit_address_space' as + false for a test which reserves much more address space than it uses, for + example one which starts many threads. """ def decorator(f): from test.support import isolation @@ -1336,6 +1363,11 @@ def wrapper(self): isolation._replay_test(self, *proc.wait(tick=watchdog)) return + if (real_max_memuse and limit_address_space + and isolation.runningInSubprocess): + # Only in the subprocess: the limit is never lifted. + _limit_address_space(size * memuse) + return f(self, maxsize) wrapper.size = size diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index 50d2444a4c72d31..eedbbfe1182fad0 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -26,7 +26,8 @@ def test_create_many_sequential(self): support.gc_collect() @threading_helper.requires_working_threading() - @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False) + @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False, + limit_address_space=False) def test_create_many_threaded(self, size): alive = [] start = threading.Event() @@ -43,7 +44,8 @@ def task(): support.gc_collect() @threading_helper.requires_working_threading() - @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False) + @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False, + limit_address_space=False) def test_many_threads_running_interp_in_other_interp(self, size): start = threading.Event() interp = interpreters.create() diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst new file mode 100644 index 000000000000000..358b2752db0870c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst @@ -0,0 +1,5 @@ +A test decorated with :func:`~test.support.bigmemtest` now runs with the +address space limited to what it declares plus 512 MiB, so that a test which +uses much more memory than it declares fails instead of making the machine +swap. Pass ``limit_address_space=False`` for a test which reserves much more +address space than it uses. From 8bc2c018cad05ae720bc5729c43849381a95fcfa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Aug 2026 21:11:21 +0300 Subject: [PATCH 2/4] Do not limit the address space where it cannot work AddressSanitizer reserves terabytes of address space for its shadow memory, so any limit stops the interpreter from starting. macOS reserves much more address space than Linux does, so 512 MiB is not enough of a margin there. --- Lib/test/support/__init__.py | 9 +++++++-- .../Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 09be2f1d55418b8..7022fa5b88987ce 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1276,14 +1276,19 @@ def _limit_address_space(nbytes): A test which uses much more memory than it declares then fails with a MemoryError instead of making the machine swap. """ + if check_sanitizer(address=True): + # AddressSanitizer reserves terabytes of address space for its shadow + # memory, so any limit stops the interpreter from starting. + return try: import resource rlimit = resource.RLIMIT_AS except (ImportError, AttributeError): return # The margin does not grow with the declared size: the interpreter itself - # reserves about 250 MiB, whatever the test asks for. - limit = int(nbytes) + 512 * _1M + # reserves about 250 MiB, whatever the test asks for. macOS reserves more. + margin = _1G if sys.platform == 'darwin' else 512 * _1M + limit = int(nbytes) + margin soft, hard = resource.getrlimit(rlimit) for current in soft, hard: if current != resource.RLIM_INFINITY: diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst index 358b2752db0870c..44d6c81d08768b1 100644 --- a/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst +++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-36-04.gh-issue-75876.5AhLHL.rst @@ -1,5 +1,5 @@ A test decorated with :func:`~test.support.bigmemtest` now runs with the -address space limited to what it declares plus 512 MiB, so that a test which +address space limited to what it declares plus a margin, so that a test which uses much more memory than it declares fails instead of making the machine swap. Pass ``limit_address_space=False`` for a test which reserves much more address space than it uses. From 78b6deaeb8c8335331e5a12a15e0a55473c6001f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 20 Aug 2026 10:59:14 +0300 Subject: [PATCH 3/4] gh-75876: Correct the memory use declared by two bigmem tests repr() of a list or a tuple of 2**31 False takes 15.3 bytes per item, not 15, and str.split() takes 10.7 bytes per item and more for a larger string, not 10. The difference is small, but it grows with the size of the test: 614 MiB for test_repr_large. --- Lib/test/test_bigmem.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 3c09cdb86408909..d8b635dd391041a 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -373,7 +373,7 @@ def test_split_small(self, size): # suffer for the list size. (Otherwise, it'd cost another 48 times # size in bytes!) Nevertheless, a list of size takes # 8*size bytes. - @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size) + @bigmemtest(size=_2G + 5, memuse=ascii_char_size * 3 + pointer_size) def test_split_large(self, size): _ = self.from_latin1 s = _(' a') * size + _(' ') @@ -936,11 +936,11 @@ def basic_test_repr(self, size): self.assertEqual(s[:10], '(False, Fa') self.assertEqual(s[-10:], 'se, False)') - @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_large(self, size): return self.basic_test_repr(size) @@ -1115,11 +1115,11 @@ def basic_test_repr(self, size): self.assertEqual(s[-10:], 'se, False]') self.assertEqual(s.count('F'), size) - @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7) + @bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 8) def test_repr_large(self, size): return self.basic_test_repr(size) From 874c66af8291b5016d3eadb6bec635b2ae279ba0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 20 Aug 2026 11:04:08 +0300 Subject: [PATCH 4/4] gh-75876: Kill a bigmem test which uses more memory than it declares The address space cannot be limited everywhere: not under AddressSanitizer, which reserves terabytes for its shadow memory, and not on macOS, which reserves much more than it uses. The parent process already watches the memory of the subprocess running the test, so let it kill the test which uses more than it declares plus the same margin. This also catches a test which fills the memory without reserving that much address space. --- Lib/test/support/__init__.py | 50 +++++++++++++++++++++++++++-------- Lib/test/support/isolation.py | 4 +++ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 7022fa5b88987ce..2d6db6d392b6b32 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1270,8 +1270,16 @@ def set_memlimit(limit: str) -> None: max_memuse = memlimit +def _memory_limit(nbytes): + """How much memory a test declaring *nbytes* may use. + + The interpreter itself reserves about 250 MiB, whatever the test asks for. + """ + return int(nbytes) + 512 * _1M + + def _limit_address_space(nbytes): - """Limit the address space of this process to *nbytes* plus a margin. + """Limit the address space of this process to what a test may use. A test which uses much more memory than it declares then fails with a MemoryError instead of making the machine swap. @@ -1280,15 +1288,15 @@ def _limit_address_space(nbytes): # AddressSanitizer reserves terabytes of address space for its shadow # memory, so any limit stops the interpreter from starting. return + if sys.platform == 'darwin': + # macOS reserves much more address space than it uses. + return try: import resource rlimit = resource.RLIMIT_AS except (ImportError, AttributeError): return - # The margin does not grow with the declared size: the interpreter itself - # reserves about 250 MiB, whatever the test asks for. macOS reserves more. - margin = _1G if sys.platform == 'darwin' else 512 * _1M - limit = int(nbytes) + margin + limit = _memory_limit(nbytes) soft, hard = resource.getrlimit(rlimit) for current in soft, hard: if current != resource.RLIM_INFINITY: @@ -1296,16 +1304,27 @@ def _limit_address_space(nbytes): resource.setrlimit(rlimit, (limit, hard)) -def _memory_watchdog(pid): - """Return a function printing the memory usage of process *pid*.""" +def _memory_watchdog(proc, limit): + """Return a function watching the memory used by the test in *proc*. + + It reports the usage in verbose mode, and kills the test if it uses more + than *limit* bytes. This is the only limit where the address space cannot + be limited. + """ # Imported here: test.support does not depend on test.libregrtest. from test.libregrtest.utils import get_process_memory_usage def watch(): - mem = get_process_memory_usage(pid) - if mem is not None: + mem = get_process_memory_usage(proc.pid) + if mem is None: + return + if verbose: print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB", flush=True) + if limit is not None and mem > limit: + watch.exceeded = mem + proc.kill() + watch.exceeded = None return watch @@ -1364,8 +1383,17 @@ def wrapper(self): cls = type(self) qualname = f'{cls.__qualname__}.{f.__name__}' proc = isolation._start_test(cls.__module__, qualname) - watchdog = _memory_watchdog(proc.pid) if verbose else None - isolation._replay_test(self, *proc.wait(tick=watchdog)) + # Watched even if the address space is not limited: this + # counts the memory really used. + watchdog = _memory_watchdog(proc, + _memory_limit(size * memuse)) + payload, output, returncode = proc.wait(tick=watchdog) + if watchdog.exceeded: + raise AssertionError( + f'the test used {watchdog.exceeded / _1G:.1f} GiB, ' + f'more than the {size * memuse / _1G:.1f} GiB ' + f'it declares') + isolation._replay_test(self, payload, output, returncode) return if (real_max_memuse and limit_address_space diff --git a/Lib/test/support/isolation.py b/Lib/test/support/isolation.py index 3cfd406b0f2f0e1..259d26959746aac 100644 --- a/Lib/test/support/isolation.py +++ b/Lib/test/support/isolation.py @@ -123,6 +123,10 @@ def __init__(self, proc, result_path): def pid(self): return self._proc.pid + def kill(self): + """Kill the test, for example when it uses too much memory.""" + self._proc.kill() + def wait(self, timeout=None, tick=None, interval=1.0): """Wait for the test to finish, calling *tick* every *interval* seconds.