fix(windows): use extended paths in Python bootstraps

news/4071.fixed.md

@@ -0,0 +1,3 @@
+(windows) Fixed long path DLL loading in {obj}`py_binary` and
+{obj}`py_test` bootstraps
+([#4071](https://github.com/bazel-contrib/rules_python/pull/4071)).

python/private/python_bootstrap_template.txt

@@ -141,33 +141,22 @@ def get_windows_path_with_unc_prefix(path):
   if not IS_WINDOWS or sys.version_info[0] < 3:
     return path
 
-  # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been
-  # removed from common Win32 file and directory functions.
-  # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later
-  import platform
-  win32_version = None
-  # Windows 2022 with Python 3.12.8 gives flakey errors, so try a couple times.
-  for _ in range(3):
-    try:
-      win32_version = platform.win32_ver()[1]
-      break
-    except (ValueError, KeyError):
-      pass
-  if win32_version and win32_version >= '10.0.14393':
-    return path
-
   # import sysconfig only now to maintain python 2.6 compatibility
   import sysconfig
   if sysconfig.get_platform() == 'mingw':
     return path
 
-  # Lets start the unicode fun
-  unicode_prefix = '\\\\?\\'
-  if path.startswith(unicode_prefix):
+  # Implicit long-path support is not universal across the Win32 API. For
+  # example, DLL loading still requires an explicit extended-length prefix.
+  # abspath returns a normalized absolute path
+  path = abspath(path)
+  extended_path_prefix = '\\\\?\\'
+  if path.startswith(extended_path_prefix):
     return path
 
-  # abspath returns a normalized absolute path
-  return unicode_prefix + abspath(path)
+  if path.startswith('\\\\'):
+    return extended_path_prefix + 'UNC\\' + path[2:]
+  return extended_path_prefix + path
 
 def search_path(name):
   """Finds a file in a given search path."""
@@ -237,6 +226,9 @@ def find_runfiles_root(main_rel_path):
   # argv[0] may no longer point to a location inside the runfiles
   # directory. We should therefore respect RUNFILES_DIR and
   # RUNFILES_MANIFEST_FILE set by the caller.
+  if IS_WINDOWS and main_rel_path:
+    main_rel_path = main_rel_path.replace('/', os.sep)
+
   runfiles_dir = os.environ.get('RUNFILES_DIR', None)
   if not runfiles_dir:
     runfiles_manifest_file = os.environ.get('RUNFILES_MANIFEST_FILE', '')

python/private/site_init_template.py

@@ -35,6 +35,10 @@ def _is_verbose():
     return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE"))
 
 
+def _is_windows():
+    return os.name == "nt"
+
+
 def _print_verbose_coverage(*args):
     if os.environ.get("VERBOSE_COVERAGE") or _is_verbose():
         _print_verbose(*args)
@@ -66,9 +70,10 @@ def _find_runfiles_root():
 
     # Be defensive: the runfiles dir should contain ourselves. If it doesn't,
     # then it must not be our runfiles directory.
-    if runfiles_dir and os.path.exists(
-        os.path.join(runfiles_dir, _SELF_RUNFILES_RELATIVE_PATH)
-    ):
+    self_path = _SELF_RUNFILES_RELATIVE_PATH
+    if _is_windows():
+        self_path = self_path.replace("/", os.sep)
+    if runfiles_dir and os.path.exists(os.path.join(runfiles_dir, self_path)):
         return runfiles_dir
 
     num_dirs_to_runfiles_root = _SELF_RUNFILES_RELATIVE_PATH.count("/") + 1
@@ -83,45 +88,29 @@ def _find_runfiles_root():
 _print_verbose("runfiles_root:", _RUNFILES_ROOT)
 
 
-def _is_windows():
-    return os.name == "nt"
-
-
 def _get_windows_path_with_unc_prefix(path):
     path = path.strip()
     # No need to add prefix for non-Windows platforms.
     if not _is_windows() or sys.version_info[0] < 3:
         return path
 
-    # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been
-    # removed from common Win32 file and directory functions.
-    # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later
-    import platform
-
-    win32_version = None
-    # Windows 2022 with Python 3.12.8 gives flakey errors, so try a couple times.
-    for _ in range(3):
-        try:
-            win32_version = platform.win32_ver()[1]
-            break
-        except (ValueError, KeyError):
-            pass
-    if win32_version and win32_version >= "10.0.14393":
-        return path
-
     # import sysconfig only now to maintain python 2.6 compatibility
     import sysconfig
 
     if sysconfig.get_platform() == "mingw":
         return path
 
-    # Lets start the unicode fun
-    unicode_prefix = "\\\\?\\"
-    if path.startswith(unicode_prefix):
+    # Implicit long-path support is not universal across the Win32 API. For
+    # example, DLL loading still requires an explicit extended-length prefix.
+    # os.path.abspath returns a normalized absolute path
+    path = os.path.abspath(path)
+    extended_path_prefix = "\\\\?\\"
+    if path.startswith(extended_path_prefix):
         return path
 
-    # os.path.abspath returns a normalized absolute path
-    return unicode_prefix + os.path.abspath(path)
+    if path.startswith("\\\\"):
+        return extended_path_prefix + "UNC\\" + path[2:]
+    return extended_path_prefix + path
 
 
 def _search_path(name):

python/private/stage2_bootstrap_template.py

@@ -113,34 +113,23 @@ def get_windows_path_with_unc_prefix(path):
     if not IS_WINDOWS or sys.version_info[0] < 3:
         return path
 
-    # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been
-    # removed from common Win32 file and directory functions.
-    # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later
-    import platform
-
-    win32_version = None
-    # Windows 2022 with Python 3.12.8 gives flakey errors, so try a couple times.
-    for _ in range(3):
-        try:
-            win32_version = platform.win32_ver()[1]
-            break
-        except (ValueError, KeyError):
-            pass
-    if win32_version and win32_version >= "10.0.14393":
-        return path
-
     # import sysconfig only now to maintain python 2.6 compatibility
     import sysconfig
 
     if sysconfig.get_platform() == "mingw":
         return path
 
-    # Lets start the unicode fun
-    if path.startswith(unicode_prefix):  # noqa: F821
+    # Implicit long-path support is not universal across the Win32 API. For
+    # example, DLL loading still requires an explicit extended-length prefix.
+    # os.path.abspath returns a normalized absolute path
+    path = os.path.abspath(path)
+    extended_path_prefix = "\\\\?\\"
+    if path.startswith(extended_path_prefix):
         return path
 
-    # os.path.abspath returns a normalized absolute path
-    return unicode_prefix + os.path.abspath(path)  # noqa: F821
+    if path.startswith("\\\\"):
+        return extended_path_prefix + "UNC\\" + path[2:]
+    return extended_path_prefix + path
 
 
 def print_verbose(*args, mapping=None, values=None):
@@ -186,6 +175,9 @@ def find_runfiles_root(main_rel_path):
     # argv[0] may no longer point to a location inside the runfiles
     # directory. We should therefore respect RUNFILES_DIR and
     # RUNFILES_MANIFEST_FILE set by the caller.
+    if IS_WINDOWS and main_rel_path:
+        main_rel_path = main_rel_path.replace("/", os.sep)
+
     runfiles_dir = os.environ.get("RUNFILES_DIR", None)
     if not runfiles_dir:
         runfiles_manifest_file = os.environ.get("RUNFILES_MANIFEST_FILE", "")

python/private/zipapp/zip_main_template.py

@@ -103,27 +103,23 @@ def get_windows_path_with_unc_prefix(path):
     if not IS_WINDOWS or sys.version_info[0] < 3:
         return path
 
-    # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been
-    # removed from common Win32 file and directory functions.
-    # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later
-    import platform
-
-    if platform.win32_ver()[1] >= "10.0.14393":
-        return path
-
     # import sysconfig only now to maintain python 2.6 compatibility
     import sysconfig
 
     if sysconfig.get_platform() == "mingw":
         return path
 
-    # Lets start the unicode fun
-    unicode_prefix = "\\\\?\\"
-    if path.startswith(unicode_prefix):
+    # Implicit long-path support is not universal across the Win32 API. For
+    # example, DLL loading still requires an explicit extended-length prefix.
+    # os.path.abspath returns a normalized absolute path
+    path = os.path.abspath(path)
+    extended_path_prefix = "\\\\?\\"
+    if path.startswith(extended_path_prefix):
         return path
 
-    # os.path.abspath returns a normalized absolute path
-    return unicode_prefix + os.path.abspath(path)
+    if path.startswith("\\\\"):
+        return extended_path_prefix + "UNC\\" + path[2:]
+    return extended_path_prefix + path
 
 
 def search_path(name):
@@ -204,6 +200,8 @@ def extract_zip(zip_path, dest_dir):
                         # Directories aren't stored in zips, so a missing
                         # target means it points to a directory.
                         target_is_directory = True
+                    target = os.path.abspath(join(dirname(file_path), target))
+                    target = get_windows_path_with_unc_prefix(target)
                 else:
                     target_is_directory = False
                 os.symlink(target, file_path, target_is_directory=target_is_directory)
@@ -223,10 +221,10 @@ def create_runfiles_root():
             extract_root = join(EXTRACT_ROOT, extract_dir, hash_dir)
         else:
             extract_root = join(EXTRACT_ROOT, EXTRACT_DIR, APP_HASH)
-            extract_root = get_windows_path_with_unc_prefix(extract_root)
     else:
         extract_root = tempfile.mkdtemp("", "Bazel.runfiles_")
 
+    extract_root = get_windows_path_with_unc_prefix(extract_root)
     extract_zip(dirname(__file__), extract_root)
     print_verbose("extracted to:", extract_root)
     # IMPORTANT: Later code does `rm -fr` on dirname(runfiles_root) -- it's

python/runfiles/runfiles.py

@@ -26,6 +26,7 @@
 from __future__ import annotations
 
 import inspect
+import ntpath
 import os
 import pathlib
 import posixpath
@@ -496,6 +497,18 @@ def EnvVars(self) -> dict[str, str]:
         }
 
 
+def _normalize_windows_path(path: str) -> str:
+    """Strip extended path prefixes (\\\\?\\ and \\\\?\\UNC\\) and normalize separators on Windows."""
+    if not path:
+        return path
+    path = path.replace("/", "\\")
+    if path.startswith("\\\\?\\UNC\\"):
+        return "\\\\" + path[8:]
+    if path.startswith("\\\\?\\"):
+        return path[4:]
+    return path
+
+
 class Runfiles:
     """Returns the runtime location of runfiles.
 
@@ -640,47 +653,41 @@ def CurrentRepository(self, frame: int = 1) -> str:
             path is not contained in the Python runfiles tree
         """
         try:
-            # pylint: disable-next=protected-access
             caller_path = inspect.getfile(sys._getframe(frame))
         except (TypeError, ValueError) as exc:
             raise ValueError("failed to determine caller's file path") from exc
-        caller_runfiles_path = os.path.relpath(caller_path, self._python_runfiles_root)
-        if caller_runfiles_path.startswith(".." + os.path.sep):
-            # With Python 3.10 and earlier, sys.path contains the directory
-            # of the script, which can result in a module being loaded from
-            # outside the runfiles tree. In this case, assume that the module is
-            # located in the main repository.
-            # With Python 3.11 and higher, the Python launcher sets
-            # PYTHONSAFEPATH, which prevents this behavior.
-            # On Windows, the current toolchain being used has a buggy zip file
-            # bootstrap, which leaves RUNFILES_DIR pointing at the first stage
-            # path and not the module path. In this case too, assume that the
-            # module is located in the main repository.
-            # TODO: This doesn't cover the case of a script being run from an
-            #       external repository, which could be heuristically detected
-            #       by parsing the script's path.
-            if (sys.version_info.minor <= 10 or sys.platform == "win32") and sys.path[
-                0
-            ] != self._python_runfiles_root:
+        path_mod = ntpath if (sys.platform == "win32" or os.name == "nt") else os.path
+        python_runfiles_root = self._python_runfiles_root
+        if sys.platform == "win32" or os.name == "nt":
+            caller_path = _normalize_windows_path(caller_path)
+            python_runfiles_root = _normalize_windows_path(python_runfiles_root)
+        try:
+            caller_runfiles_path = path_mod.relpath(caller_path, python_runfiles_root)
+        except ValueError:
+            caller_runfiles_path = ".." + path_mod.sep
+        if caller_runfiles_path.startswith(".." + path_mod.sep):
+            sys_path_0 = sys.path[0] if sys.path else ""
+            if sys.platform == "win32" or os.name == "nt":
+                sys_path_0 = _normalize_windows_path(sys_path_0)
+                if (
+                    sys.version_info.minor <= 10
+                    or sys.platform == "win32"
+                    or os.name == "nt"
+                ) and path_mod.normcase(sys_path_0) != path_mod.normcase(
+                    python_runfiles_root
+                ):
+                    return ""
+            elif (sys.version_info.minor <= 10) and sys_path_0 != python_runfiles_root:
                 return ""
             raise ValueError(
                 "{} does not lie under the runfiles root {}".format(
                     caller_path, self._python_runfiles_root
                 )
             )
 
-        caller_runfiles_directory = caller_runfiles_path[
-            : caller_runfiles_path.find(os.path.sep)
-        ]
-        # With Bzlmod, the runfiles directory of the main repository is always
-        # named "_main". Without Bzlmod, the value returned by this function is
-        # never used, so we just assume Bzlmod is enabled.
+        caller_runfiles_directory = caller_runfiles_path.split(path_mod.sep, 1)[0]
         if caller_runfiles_directory == "_main":
-            # The canonical name of the main repository (also known as the
-            # workspace) is the empty string.
             return ""
-        # For all other repositories, the name of the runfiles directory is the
-        # canonical name.
         return caller_runfiles_directory
 
     # TODO: Update return type to Self when 3.11 is the min version

tests/bootstrap_impls/long_path/p1_pkg_dir_exceeding_max_path_limit_0123456789/p2_pkg_dir_exceeding_max_path_limit_0123456789/BUILD.bazel

@@ -0,0 +1,23 @@
+load("//python:py_test.bzl", "py_test")
+load("//python/cc:py_extension.bzl", "py_extension")
+
+package(
+    default_visibility = ["//tests/bootstrap_impls:__subpackages__"],
+)
+
+licenses(["notice"])
+
+py_extension(
+    name = "ext_long_path",
+    srcs = ["ext_long_path.c"],
+    target_compatible_with = ["@platforms//os:windows"],
+)
+
+py_test(
+    name = "py_extension_long_path_test",
+    srcs = ["py_extension_long_path_test.py"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        ":ext_long_path",
+    ],
+)

tests/bootstrap_impls/long_path/p1_pkg_dir_exceeding_max_path_limit_0123456789/p2_pkg_dir_exceeding_max_path_limit_0123456789/ext_long_path.c

@@ -0,0 +1,22 @@
+#include <Python.h>
+
+static PyObject* get_magic_number(PyObject* self, PyObject* args) {
+    return PyLong_FromLong(42);
+}
+
+static PyMethodDef ModuleMethods[] = {
+    {"get_magic_number", get_magic_number, METH_NOARGS, "Returns 42."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef ext_long_path_module = {
+    PyModuleDef_HEAD_INIT,
+    "ext_long_path",
+    NULL,
+    -1,
+    ModuleMethods
+};
+
+PyMODINIT_FUNC PyInit_ext_long_path(void) {
+    return PyModule_Create(&ext_long_path_module);
+}