diff --git a/src/borg/archive.py b/src/borg/archive.py index 8017574b05..d0cd7763e3 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -16,7 +16,7 @@ from collections.abc import Iterator from shutil import get_terminal_size -from .platformflags import is_win32 +from .platformflags import is_win32, is_pypy from .logger import create_logger logger = create_logger() @@ -831,7 +831,19 @@ def extract_helper(self, item, path, hlm, *, dry_run=False): # backup_io would not catch) on platforms where os.link ignores follow_symlinks. with backup_io("link"): if os.link in os.supports_follow_symlinks: - os.link(link_target, path, follow_symlinks=False) + try: + os.link(link_target, path, follow_symlinks=False) + except OSError as os_error: + # pypy advertises follow_symlinks support, but raises EINVAL when + # the parameter is actually used (pypy bug) - emulate the secure + # follow_symlinks=False behaviour documented above. + if not (is_pypy and os_error.errno == errno.EINVAL): + raise + if os.path.islink(link_target): + # we cannot hardlink the symlink itself, so make an equal symlink. + os.symlink(os.readlink(link_target), path) + else: + os.link(link_target, path) else: os.link(link_target, path) hardlink_set = True diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 7c0e659f0e..52c8ff6da6 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -38,6 +38,8 @@ import hashlib import hmac from math import ceil +from ..platformflags import is_pypy + from cpython cimport PyMem_Malloc, PyMem_Free from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString @@ -658,6 +660,12 @@ cdef class CHACHA20_POLY1305(_AEAD_BASE): def hmac_sha256(key, data): + if is_pypy: + # pypy's hmac/_hashlib only accepts bytes (no memoryview, no bytearray) + if not isinstance(key, bytes): + key = bytes(key) + if not isinstance(data, bytes): + data = bytes(data) return hmac.digest(key, data, 'sha256') diff --git a/src/borg/platform/darwin.pyx b/src/borg/platform/darwin.pyx index c8d7e124df..a11f8d54c5 100644 --- a/src/borg/platform/darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -269,6 +269,9 @@ def set_flags(path, bsd_flags, fd=None): import errno as errno_mod import fcntl as fcntl_mod +# pypy's fcntl module does not expose F_FULLFSYNC (value 51 in macOS headers) +F_FULLFSYNC = getattr(fcntl_mod, "F_FULLFSYNC", 51) + def fdatasync(fd): """macOS fdatasync using F_FULLFSYNC for true data durability. @@ -278,7 +281,7 @@ def fdatasync(fd): Falls back to os.fsync() if F_FULLFSYNC is not supported (e.g. network fs). """ try: - fcntl_mod.fcntl(fd, fcntl_mod.F_FULLFSYNC) + fcntl_mod.fcntl(fd, F_FULLFSYNC) except OSError: # F_FULLFSYNC not supported (e.g. network filesystem), fall back os.fsync(fd) diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py index cc84f7c7ac..8af9ecfc9d 100644 --- a/src/borg/platformflags.py +++ b/src/borg/platformflags.py @@ -19,3 +19,6 @@ # MSYS2 (on Windows) is_msystem = is_win32 and "MSYSTEM" in os.environ + +# Python implementation +is_pypy = sys.implementation.name == "pypy" diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py index 05ef2bff91..29f5307803 100644 --- a/src/borg/testsuite/archiver/extract_cmd_test.py +++ b/src/borg/testsuite/archiver/extract_cmd_test.py @@ -21,7 +21,7 @@ from .. import changedir, same_ts_ns, granularity_sleep from .. import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported, is_birthtime_fully_supported from ...platform import get_birthtime_ns -from ...platformflags import is_darwin, is_freebsd, is_win32 +from ...platformflags import is_darwin, is_freebsd, is_win32, is_pypy from . import ( RK_ENCRYPTION, requires_hardlinks, @@ -132,7 +132,10 @@ def test_extract_hardlinked_symlink_does_not_leak_target(archivers, request): h_path = os.path.join(archiver.output_path, "h") # "h" is a hardlink to the symlink itself (same inode as "s"), a faithful restore ... assert os.path.islink(h_path) - assert os.stat(h_path, follow_symlinks=False).st_ino == os.stat(s_path, follow_symlinks=False).st_ino + if not is_pypy: + # pypy's os.link(follow_symlinks=False) is broken, borg emulates it there by + # creating an equal symlink, which shares no inode with "s". + assert os.stat(h_path, follow_symlinks=False).st_ino == os.stat(s_path, follow_symlinks=False).st_ino # ... and NOT a hardlink to the external victim file's inode (that would be the leak). assert os.stat(h_path, follow_symlinks=False).st_ino != os.stat(victim_file).st_ino # the out-of-tree victim is untouched diff --git a/src/borg/testsuite/archiver/lock_cmds_test.py b/src/borg/testsuite/archiver/lock_cmds_test.py index 81e9dcccd8..be760470bc 100644 --- a/src/borg/testsuite/archiver/lock_cmds_test.py +++ b/src/borg/testsuite/archiver/lock_cmds_test.py @@ -28,13 +28,14 @@ def test_with_lock(tmp_path): print("sys.path: %r" % sys.path) print("PYTHONPATH: %s" % env.get("PYTHONPATH", "")) print("PATH: %s" % env.get("PATH", "")) - command0 = "python3", "-m", "borg", "repo-create", "--encryption=none" + python = sys.executable or "python3" + command0 = python, "-m", "borg", "repo-create", "--encryption=none" # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock, # so that lock acquisition for command2 fails as the test expects it. lock_wait = 2 - command1 = ("python3", "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') - command2 = "python3", "-c", 'print("second command - should never get executed")' - borgwl = "python3", "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" + command1 = (python, "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') + command2 = python, "-c", 'print("second command - should never get executed")' + borgwl = python, "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" popen_options = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) subprocess.run(command0, env=env, check=True, text=True, capture_output=True) assert repo_path.exists() @@ -50,7 +51,8 @@ def test_with_lock(tmp_path): assert "Failed to create/acquire the lock" in err_out assert p2.returncode == 73 # LockTimeout: could not acquire the lock, p1 already has it out, err_out = p1.communicate(input="") # Unblock command1 and read output - assert not err_out + # ignore the pure-python msgpack warning borg emits on pypy + assert not [line for line in err_out.splitlines() if "pure-python msgpack" not in line] assert p1.returncode == 0 diff --git a/src/borg/testsuite/helpers/msgpack_test.py b/src/borg/testsuite/helpers/msgpack_test.py index 41960da784..cd505a5d45 100644 --- a/src/borg/testsuite/helpers/msgpack_test.py +++ b/src/borg/testsuite/helpers/msgpack_test.py @@ -13,6 +13,9 @@ def expected_py_mp_slow_combination(): # msgpack is slow on Cygwin if is_cygwin: return True + # pypy only has the pure-python msgpack (which pypy's jit hopefully makes fast enough) + if sys.implementation.name == "pypy": + return True # msgpack < 1.0.6 did not have Python 3.12 wheels if sys.version_info[:2] == (3, 12) and msgpack.version < (1, 0, 6): return True diff --git a/src/borg/testsuite/item_test.py b/src/borg/testsuite/item_test.py index 9c6a5b796c..51d3de5356 100644 --- a/src/borg/testsuite/item_test.py +++ b/src/borg/testsuite/item_test.py @@ -4,6 +4,7 @@ from ..item import Item, chunks_contents_equal from ..helpers import StableDict from ..helpers.msgpack import Timestamp +from ..platformflags import is_pypy def test_item_empty(): @@ -131,6 +132,7 @@ def test_item_dict_property(): assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}} +@pytest.mark.xfail(is_pypy, reason="setting undeclared attributes on cdef class instances is not blocked on pypy") def test_unknown_property(): # We do not want the user to be able to set unknown attributes — # they will not appear in the .as_dict() result dictionary. diff --git a/src/borg/testsuite/platform/darwin_test.py b/src/borg/testsuite/platform/darwin_test.py index c69a7e1452..6e68147def 100644 --- a/src/borg/testsuite/platform/darwin_test.py +++ b/src/borg/testsuite/platform/darwin_test.py @@ -68,7 +68,7 @@ def mock_fcntl(fd, cmd, *args): tmp.flush() darwin.fdatasync(tmp.fileno()) - assert any(cmd == fcntl_mod.F_FULLFSYNC for _, cmd in calls), "fdatasync should call fcntl with F_FULLFSYNC" + assert any(cmd == darwin.F_FULLFSYNC for _, cmd in calls), "fdatasync should call fcntl with F_FULLFSYNC" def test_fdatasync_falls_back_to_fsync(monkeypatch): @@ -79,7 +79,7 @@ def test_fdatasync_falls_back_to_fsync(monkeypatch): fsync_calls = [] def mock_fcntl(fd, cmd, *args): - if cmd == fcntl_mod.F_FULLFSYNC: + if cmd == darwin.F_FULLFSYNC: raise OSError("F_FULLFSYNC not supported") return 0