fix(index): validate paths at native index boundaries

<!-- Byron -->
rubber-stamp, after checking diff quickly.

<!-- agent -->
Native tree merging and index serialization could install paths that Git's
own index writers reject. A subsequent checkout trusts that index and can
write outside the working tree or into Git metadata (finding 21506 and
the shared write-side cause of finding 21501).

Validate complete paths when importing tree entries, reading or writing
index entries, and building trees from the index. Reject traversal,
absolute/drive paths, embedded NULs, and Git metadata aliases on NTFS/HFS
while retaining ordinary POSIX filename characters. Validate before
creating trees, and preserve index writer rollback on failure.

Refresh the on-disk index before checkout, including when entries were
cached. Parse full sentinel-length names and check their terminators and
padding so validation sees the same paths as Git. Reject unsupported
mandatory extensions such as split indexes: validating only their inline
entries would overlook paths from external shared indexes.

QA review follow-up: reject unsupported index versions with an explicit
exception, preserving `AssertionError` for compatibility. Python's `-O`
mode removed the former assertion, allowing version 4 prefix-compressed
paths to be parsed with version 2/3 rules before native checkout.
Validation must never accept a format whose paths Git interprets
differently. Optimized subprocess regressions for unsupported versions
fail before this fix and pass afterward. All 76 selected index tests
pass, with two platform skips; locked Basedpyright, mypy, Codespell, and
Ruff checks pass.

Author: codex

git/index/base.py

@@ -1390,6 +1390,9 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik
 
         # END stderr handler
 
+        # Read and validate the index before Git trusts its paths for checkout.
+        self._delete_entries_cache()
+        self.entries  # noqa: B018
         if paths is None:
             args.append("--all")
             kwargs["as_process"] = 1

git/index/fun.py

@@ -34,7 +34,7 @@
     traverse_trees_recursive,
     tree_to_stream,
 )
-from git.util import IndexFileSHA1Writer, finalize_process
+from git.util import IndexFileSHA1Writer, finalize_process, _validate_repo_path
 
 from .typ import CE_EXTENDED, BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT
 from .util import pack, unpack
@@ -279,13 +279,13 @@ def write_cache(
 
     # Body
     for entry in entries:
+        _validate_repo_path(entry.path)
         beginoffset = tell()
         write(entry.ctime_bytes)  # ctime
         write(entry.mtime_bytes)  # mtime
-        path_str = str(entry.path)
+        path_str = os.fspath(entry.path)
         path: bytes = force_bytes(path_str, encoding=defenc)
-        plen = len(path) & CE_NAMEMASK  # Path length
-        assert plen == len(path), "Path %s too long to fit into index" % entry.path
+        plen = min(len(path), CE_NAMEMASK)  # Longer names use a sentinel.
         flags = plen | (entry.flags & CE_NAMEMASK_INV)  # Clear possible previous values.
         if entry.extended_flags:
             flags |= CE_EXTENDED
@@ -325,7 +325,8 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]:
     unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2)))
     version, num_entries = unpacked
 
-    assert version in (1, 2, 3), "Unsupported git index version %i, only 1, 2, and 3 are supported" % version
+    if version not in (1, 2, 3):
+        raise AssertionError("Unsupported git index version %i, only 1, 2, and 3 are supported" % version)
     return version, num_entries
 
 
@@ -382,10 +383,23 @@ def read_cache(
         if flags & CE_EXTENDED:
             extended_flags = unpack(">H", read(2))[0]
         path_size = flags & CE_NAMEMASK
-        path = read(path_size).decode(defenc)
-
-        real_size = (tell() - beginoffset + 8) & ~7
-        read((beginoffset + real_size) - tell())
+        path_bytes = bytearray(read(path_size))
+        if len(path_bytes) != path_size:
+            raise ValueError("Truncated index entry path")
+        terminator = read(1)
+        if path_size == CE_NAMEMASK:
+            while terminator and terminator != b"\0":
+                path_bytes.extend(terminator)
+                terminator = read(1)
+        if terminator != b"\0":
+            raise ValueError("Unterminated index entry path")
+        path = path_bytes.decode(defenc)
+        _validate_repo_path(path)
+
+        real_size = (tell() - beginoffset + 7) & ~7
+        padding_size = beginoffset + real_size - tell()
+        if read(padding_size) != b"\0" * padding_size:
+            raise ValueError("Invalid index entry padding")
         entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size, extended_flags))
         # entry_key would be the method to use, but we save the effort.
         entries[(path, entry.stage)] = entry
@@ -408,6 +422,18 @@ def read_cache(
     # Truncate the sha in the end as we will dynamically create it anyway.
     extension_data = extension_data[:-20]
 
+    offset = 0
+    while offset < len(extension_data):
+        header = extension_data[offset : offset + 8]
+        if len(header) != 8:
+            raise ValueError("Truncated index extension header")
+        signature, size = unpack(">4sL", header)
+        if not b"A" <= signature[:1] <= b"Z":
+            raise ValueError("Unsupported mandatory index extension %r" % signature)
+        offset += 8 + size
+        if offset > len(extension_data):
+            raise ValueError("Truncated index extension %r" % signature)
+
     return (version, entries, extension_data, content_sha)
 
 
@@ -434,6 +460,9 @@ def write_tree_from_cache(
 
         A tuple of a sha and a list of tree entries being a tuple of hexsha, mode, name.
     """
+    if si == 0:
+        for entry in entries[sl]:
+            _validate_repo_path(entry.path)
     tree_items: List["TreeCacheTup"] = []
 
     ci = sl.start
@@ -481,6 +510,7 @@ def write_tree_from_cache(
 
 
 def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> BaseIndexEntry:
+    _validate_repo_path(tree_entry[2])
     return BaseIndexEntry((tree_entry[1], tree_entry[0], stage << CE_STAGESHIFT, tree_entry[2]))
 
 

git/util.py

@@ -34,6 +34,7 @@
 from functools import wraps
 import getpass
 import logging
+import ntpath
 import os
 import os.path as osp
 from pathlib import Path
@@ -380,6 +381,33 @@ def _to_relative_path(root: PathLike, path: PathLike) -> str:
     return relative_path
 
 
+_HFS_IGNORABLES = str.maketrans(
+    "", "", "\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u206a\u206b\u206c\u206d\u206e\u206f\ufeff"
+)
+
+
+def _validate_repo_path(path: PathLike) -> None:
+    """Reject unsafe tree/index paths without normalizing away their components.
+
+    Protect Git metadata aliases on NTFS and HFS even when writing on another
+    platform. Other POSIX filename characters, including newlines, remain valid.
+    """
+    name = os.fspath(path)
+    if not name or "\0" in name or ntpath.splitdrive(name)[0] or name.startswith("/"):
+        raise ValueError("Invalid repository path %r" % name)
+    if os.name == "nt" and "\\" in name:
+        raise ValueError("Index paths must use '/' separators: %r" % name)
+    for component in name.split("/"):
+        if component in ("", ".", ".."):
+            raise ValueError("Invalid repository path %r" % name)
+        # NTFS recognizes backslashes and alternate data streams in metadata names.
+        for part in component.split("\\"):
+            ntfs_name = part.split(":", 1)[0].rstrip(" .").lower()
+            hfs_name = part.translate(_HFS_IGNORABLES).lower()
+            if ntfs_name in (".git", "git~1") or hfs_name == ".git":
+                raise ValueError("Repository path aliases Git metadata: %r" % name)
+
+
 def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool:
     """Make sure that the directory pointed to by path exists.
 

test/test_index.py

@@ -4,26 +4,28 @@
 # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
 
 import contextlib
-from dataclasses import dataclass
-from io import BytesIO
 import logging
 import os
 import os.path as osp
-from pathlib import Path
 import re
 import shutil
-from stat import S_ISLNK, ST_MODE
+import struct
 import subprocess
 import sys
 import tempfile
+from dataclasses import dataclass
+from hashlib import sha1
+from io import BytesIO
+from itertools import product
+from pathlib import Path
+from stat import S_ISLNK, ST_MODE
 from unittest import mock
 
-from gitdb.base import IStream
-
 import ddt
 import pytest
+from gitdb.base import IStream
 
-from git import BlobFilter, Diff, Git, IndexFile, NULL_TREE, Object, Repo, Tree
+from git import NULL_TREE, BlobFilter, Diff, Git, IndexFile, Object, Repo, Tree
 from git.diff import NULL_TREE_SHA
 from git.exc import (
     CheckoutError,
@@ -33,13 +35,12 @@
     UnmergedEntriesError,
     UnsafeOptionError,
 )
-from git.index.fun import _git_for_windows_bash, _which_from_path, hook_path, run_commit_hook
+from git.index.fun import _git_for_windows_bash, _which_from_path, hook_path, read_cache, run_commit_hook, write_cache
 from git.index.typ import BaseIndexEntry, IndexEntry
 from git.index.util import TemporaryFileSwap
 from git.objects import Blob
 from git.util import Actor, cwd, hex_to_bin, rmtree
-
-from test.lib import TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo, PathLikeMock
+from test.lib import PathLikeMock, TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo
 from test.lib.helper import symlinks_supported, xfail_if_raises
 
 HOOKS_SHEBANG = "#!/usr/bin/env sh\n"
@@ -188,6 +189,15 @@ def _make_hook(git_dir, name, content, make_exec=True):
     return hp
 
 
+def _raw_index(path):
+    """Build an index without using the writer under test."""
+    name = path.encode("utf-8")
+    entry = struct.pack(">10L20sH", 0, 0, 0, 0, 0, 0, 0o100644, 0, 0, 0, b"a" * 20, min(len(name), 0xFFF)) + name
+    entry += b"\0" * (8 - len(entry) % 8)
+    data = b"DIRC" + struct.pack(">LL", 2, 1) + entry
+    return data + sha1(data).digest()
+
+
 @ddt.ddt
 class TestIndex(TestBase):
     @with_rw_repo("HEAD")
@@ -303,6 +313,54 @@ def test_index_file_base(self):
             self.assertEqual(fp.read(), fixture("index_merge"))
         os.remove(tmpfile)
 
+    @ddt.data(
+        "",
+        ".",
+        "..",
+        "../outside",
+        "a/../outside",
+        "/absolute",
+        "C:relative",
+        "a//b",
+        "a/./b",
+        "a/",
+        "nul\0name",
+        ".git/config",
+        "a/.GiT/hooks/hook",
+        "git~1/config",
+        ".git. /config",
+        ".git:stream",
+        ".g\u200cit/config",
+        "a\\.git\\config",
+    )
+    def test_index_reader_and_writer_reject_unsafe_paths(self, path):
+        with pytest.raises(ValueError):
+            read_cache(BytesIO(_raw_index(path)))
+        entry = IndexEntry((0o100644, b"a" * 20, 0, path))
+        with pytest.raises(ValueError):
+            write_cache([entry], BytesIO())
+
+    def test_valid_unusual_index_names_round_trip(self):
+        names = ["a b", "a\nb", "a\tb", "name:value", "dir/.gitignore", "café"]
+        if os.name != "nt":
+            names.append("a\\b")
+        entries = [IndexEntry((0o100644, b"a" * 20, 0, name)) for name in sorted(names)]
+        stream = BytesIO()
+        write_cache(entries, stream)
+        stream.seek(0)
+        assert [entry.path for entry in read_cache(stream)[1].values()] == sorted(names)
+
+    def test_long_index_names_are_fully_validated(self):
+        prefix = "a/" + "nested/" * 650
+        with pytest.raises(ValueError):
+            read_cache(BytesIO(_raw_index(prefix + "../outside")))
+        name = prefix + "file"
+        assert next(iter(read_cache(BytesIO(_raw_index(name)))[1])) == (name, 0)
+        stream = BytesIO()
+        write_cache([IndexEntry((0o100644, b"a" * 20, 0, name))], stream)
+        stream.seek(0)
+        assert next(iter(read_cache(stream)[1])) == (name, 0)
+
     def _cmp_tree_index(self, tree, index):
         # Fail unless both objects contain the same paths and blobs.
         if isinstance(tree, str):
@@ -637,6 +695,30 @@ def _count_existing(self, repo, files):
 
     # END num existing helper
 
+    @ddt.data("write", "write_tree", "checkout", "cached_checkout")
+    @with_rw_directory
+    def test_index_boundaries_reject_injected_entries_before_side_effects(self, rw_dir, operation):
+        tmp_path = Path(rw_dir) / "repo"
+        with Repo.init(tmp_path) as repo:
+            index_path = Path(repo.index.path)
+            if operation == "checkout":
+                index_path.write_bytes(_raw_index("../outside"))
+            else:
+                repo.index.write()
+            before = index_path.read_bytes()
+            index = repo.index
+            if operation == "cached_checkout":
+                assert not index.entries
+                index_path.write_bytes(_raw_index("../outside"))
+                before = index_path.read_bytes()
+                operation = "checkout"
+            elif operation != "checkout":
+                index.entries[("../outside", 0)] = IndexEntry((0o100644, b"a" * 20, 0, "../outside"))
+            with pytest.raises(ValueError):
+                getattr(index, operation)()
+            assert index_path.read_bytes() == before
+            assert not (tmp_path.parent / "outside").exists()
+
     @with_rw_repo("0.1.6")
     def test_index_mutation(self, rw_repo):
         with xfail_if_raises(
@@ -1035,6 +1117,20 @@ def test_index_new(self):
             assert isinstance(index, IndexFile)
         # END for each arg tuple
 
+    @ddt.data(*product(("../outside", ".git/hooks/pre-commit", "a/.GIT/config"), (1, 2, 3)))
+    @ddt.unpack
+    @with_rw_directory
+    def test_native_tree_merge_rejects_unsafe_paths(self, rw_dir, path, tree_count):
+        tmp_path = Path(rw_dir)
+        with Repo.init(tmp_path) as repo:
+            blob = repo.odb.store(IStream("blob", 4, BytesIO(b"data"))).binsha
+            data = b"100755 " + path.encode() + b"\0" + blob
+            tree = repo.odb.store(IStream("tree", len(data), BytesIO(data))).binsha
+            empty = repo.odb.store(IStream("tree", 0, BytesIO())).binsha
+            with pytest.raises(ValueError):
+                IndexFile.new(repo, *([empty] * (tree_count - 1) + [tree]))
+            assert not (tmp_path / ".git" / "index").exists()
+
     @with_rw_repo("HEAD", bare=True)
     def test_index_bare_add(self, rw_bare_repo):
         # Something is wrong after cloning to a bare repo, reading the property
@@ -1433,34 +1529,61 @@ def test_commit_msg_hook_fail(self, rw_repo):
 
     @with_rw_repo("HEAD")
     def test_index_add_pathlib(self, rw_repo):
-        git_dir = Path(rw_repo.git_dir)
+        worktree = Path(rw_repo.working_tree_dir)
 
-        file = git_dir / "file.txt"
+        file = worktree / "file.txt"
         file.touch()
 
         rw_repo.index.add(file)
 
     @with_rw_repo("HEAD")
     def test_index_add_pathlike(self, rw_repo):
-        git_dir = Path(rw_repo.git_dir)
+        worktree = Path(rw_repo.working_tree_dir)
 
-        file = git_dir / "file.txt"
+        file = worktree / "file.txt"
         file.touch()
 
         rw_repo.index.add(PathLikeMock(str(file)))
 
     @with_rw_repo("HEAD")
     def test_index_add_non_normalized_path(self, rw_repo):
-        git_dir = Path(rw_repo.git_dir)
+        worktree = Path(rw_repo.working_tree_dir)
 
-        file = git_dir / "file.txt"
+        file = worktree / "file.txt"
         file.touch()
         non_normalized_path = file.as_posix()
         if os.name != "nt":
             non_normalized_path = "/" + non_normalized_path[1:].replace("/", "//")
 
         rw_repo.index.add(non_normalized_path)
 
+    @ddt.data(0, 4, 5)
+    def test_unsupported_index_versions_fail_even_with_optimization(self, version):
+        data = b"DIRC" + struct.pack(">LL", version, 0)
+        data += sha1(data).digest()
+        with pytest.raises(AssertionError, match="Unsupported git index version"):
+            read_cache(BytesIO(data))
+        code = """
+from io import BytesIO
+import sys
+from git.index.fun import read_cache
+try:
+    read_cache(BytesIO(sys.stdin.buffer.read()))
+except AssertionError as error:
+    if "Unsupported git index version" not in str(error):
+        raise
+else:
+    raise SystemExit("Unsupported index version was accepted")
+"""
+        result = subprocess.run([sys.executable, "-O", "-c", code], input=data, capture_output=True, timeout=10)
+        assert result.returncode == 0, result.stderr.decode()
+
+    @ddt.data(b"link", b"sdir")
+    def test_unsupported_mandatory_index_extensions_fail_closed(self, signature):
+        data = b"DIRC" + struct.pack(">LL", 2, 0) + signature + struct.pack(">L", 0)
+        with pytest.raises(ValueError, match="extension"):
+            read_cache(BytesIO(data + sha1(data).digest()))
+
     def test_index_file_v3(self):
         index = IndexFile(self.rorepo, fixture_path("index_extended_flags"))
         assert index.entries