Refresh colab snapshot

.github/scripts/lock_notebook.py

@@ -14,8 +14,12 @@
 their pin sets identical, so they are tested against one environment and are
 published together in one container image (see ../docker/README.md).
 
+Re-locking keeps each package at its currently pinned version unless the Colab
+snapshot or the requirements file forces a change; pass `--upgrade` to resolve
+everything to the newest allowed versions instead.
+
 Usage:
-    python .github/scripts/lock_notebook.py <notebook.ipynb> [...]
+    python .github/scripts/lock_notebook.py [--upgrade] <notebook.ipynb> [...]
 
 Assumes `uv` is on PATH and `nbformat` is importable.
 """
@@ -99,7 +103,15 @@ def overrides_in(requirements: Path) -> list[str]:
     ]
 
 
-def compile_pins(requirements: Path) -> list[str]:
+def compile_pins(requirements: Path, existing: list[str] | None = None) -> list[str]:
+    """Resolve `requirements` to a full pin set under the Colab constraints.
+
+    `existing` is the notebook's current pin set. It is handed to uv as the
+    previous lock, so packages keep their current version unless a constraint
+    or requirement forces a change. A refresh of the Colab snapshot then moves
+    only what Colab moved, and packages Colab does not ship (dandi, pynwb, ...)
+    stay where they were tested. Pass None to resolve everything afresh.
+    """
     overrides = overrides_in(requirements)
     constraint = CONSTRAINT
     if overrides:
@@ -120,11 +132,20 @@ def compile_pins(requirements: Path) -> list[str]:
         "--constraint", str(constraint),
         "--no-header", "--no-annotate",
     ]
+    previous = None
+    if existing:
+        # uv reads an existing --output-file as preferences for the new lock.
+        with tempfile.NamedTemporaryFile("w", suffix=".lock.txt", delete=False) as f:
+            f.write("\n".join(existing) + "\n")
+            previous = Path(f.name)
+        cmd += ["--output-file", str(previous)]
     try:
         r = subprocess.run(cmd, capture_output=True, text=True, stdin=subprocess.DEVNULL)
     finally:
         if constraint is not CONSTRAINT:
             constraint.unlink(missing_ok=True)
+        if previous is not None:
+            previous.unlink(missing_ok=True)
     if r.returncode != 0:
         raise RuntimeError(f"uv pip compile failed for {requirements}:\n{r.stderr}")
     pins = [
@@ -145,15 +166,17 @@ def install_cell_source(pins: list[str], helpers: list[str]) -> str:
     return "\n".join(lines)
 
 
-def lock(nb_path: Path) -> None:
+def lock(nb_path: Path, upgrade: bool = False) -> None:
     requirements = requirements_for(nb_path)
-    pins = compile_pins(requirements)
     nb = nbformat.read(nb_path, as_version=4)
 
     try:
-        _, helpers, install_idx = find_install_cell(nb)
+        existing, helpers, install_idx = find_install_cell(nb)
     except RuntimeError:
-        helpers, install_idx = [], None
+        existing, helpers, install_idx = [], [], None
+    # Only `name==version` entries are usable as preferences (not git pins).
+    existing = [p for p in existing if re.fullmatch(r"[A-Za-z0-9._\[\],-]+==\S+", p)]
+    pins = compile_pins(requirements, None if upgrade else existing)
 
     if install_idx is not None:
         nb.cells[install_idx].source = install_cell_source(pins, helpers)
@@ -182,11 +205,14 @@ def lock(nb_path: Path) -> None:
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("notebooks", nargs="+", type=Path)
+    parser.add_argument("--upgrade", action="store_true",
+                        help="Ignore the notebook's current pins and resolve every "
+                             "package to the newest version the constraints allow")
     args = parser.parse_args()
     failures = 0
     for nb_path in args.notebooks:
         try:
-            lock(nb_path)
+            lock(nb_path, upgrade=args.upgrade)
         except Exception as e:
             print(f"error: {nb_path}: {e}", file=sys.stderr)
             failures += 1

.github/scripts/refresh_colab_snapshot.py

@@ -0,0 +1,133 @@
+"""Refresh `.github/colab-preinstalled.txt` from Colab's published pip freeze.
+
+Colab rebuilds its runtime image every week or two, and each rebuild bumps some
+of the preinstalled packages. The notebooks' install cells are locked against
+the snapshot in `.github/colab-preinstalled.txt`, so once the snapshot is stale
+the install cell downgrades those packages back to the old pins, which is slow
+and forces a runtime restart. This script rewrites the snapshot from
+googlecolab/backend-info and, with `--relock`, re-locks every notebook that
+already has a bootstrap install cell and is tested by CI.
+
+Only `name==version` lines are kept. Colab's freeze also lists direct-URL
+installs (torch, google-colab, ...), which cannot be used as constraints.
+
+Usage:
+    python .github/scripts/refresh_colab_snapshot.py [--relock]
+
+Exits 0 whether or not anything changed; prints `changed=true|false` and, when
+$GITHUB_OUTPUT is set, writes the same line there.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+from pathlib import Path
+
+import nbformat
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from list_notebooks import REPO_ROOT, is_excluded, load_exclusions  # noqa: E402
+from lock_notebook import CONSTRAINT, requirements_for  # noqa: E402
+from run_notebook import find_install_cell  # noqa: E402
+
+SOURCE_URL = "https://raw.githubusercontent.com/googlecolab/backend-info/main/pip-freeze.txt"
+PIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*==\S+$")
+REFRESHED_RE = re.compile(r"^# Last refreshed: .*$", re.MULTILINE)
+
+
+def fetch_pins() -> list[str]:
+    with urllib.request.urlopen(SOURCE_URL, timeout=60) as r:
+        text = r.read().decode()
+    pins = [line.strip() for line in text.splitlines() if PIN_RE.match(line.strip())]
+    # Guard against an empty or truncated response replacing a good snapshot.
+    if len(pins) < 300:
+        raise RuntimeError(f"Only {len(pins)} pins in {SOURCE_URL}; refusing to refresh")
+    return pins
+
+
+def split_snapshot(text: str) -> tuple[str, list[str]]:
+    lines = text.splitlines()
+    header = [line for line in lines if line.startswith("#") or not line.strip()]
+    pins = [line.strip() for line in lines if line.strip() and not line.startswith("#")]
+    return "\n".join(header).rstrip("\n"), pins
+
+
+def describe_changes(old: list[str], new: list[str]) -> list[str]:
+    old_map = dict(p.split("==", 1) for p in old)
+    new_map = dict(p.split("==", 1) for p in new)
+    out = []
+    for name in sorted(set(old_map) | set(new_map), key=str.lower):
+        before, after = old_map.get(name), new_map.get(name)
+        if before != after:
+            out.append(f"{name}: {before or '(absent)'} -> {after or '(removed)'}")
+    return out
+
+
+def bootstrapped_notebooks() -> list[Path]:
+    """Notebooks that carry a locked install cell and a requirements file.
+
+    Notebooks excluded from CI are left alone: their pins cannot be verified
+    after a re-lock, and several are held on an older stack on purpose.
+    """
+    exclusions = load_exclusions()
+    found = []
+    for nb_path in sorted(REPO_ROOT.rglob("*.ipynb")):
+        if ".ipynb_checkpoints" in nb_path.parts:
+            continue
+        if is_excluded(str(nb_path.relative_to(REPO_ROOT)), exclusions):
+            continue
+        try:
+            find_install_cell(nbformat.read(nb_path, as_version=4))
+            requirements_for(nb_path)
+        except Exception:
+            continue
+        found.append(nb_path)
+    return found
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--relock", action="store_true",
+                        help="Re-lock every bootstrapped notebook when the snapshot changed")
+    parser.add_argument("--force-relock", action="store_true",
+                        help="Re-lock even when the snapshot did not change")
+    args = parser.parse_args()
+
+    header, old_pins = split_snapshot(CONSTRAINT.read_text())
+    new_pins = fetch_pins()
+    changes = describe_changes(old_pins, new_pins)
+    changed = bool(changes)
+
+    if changed:
+        today = datetime.date.today().isoformat()
+        header = REFRESHED_RE.sub(f"# Last refreshed: {today}", header)
+        CONSTRAINT.write_text(header + "\n" + "\n".join(new_pins) + "\n")
+        print(f"Refreshed {CONSTRAINT.relative_to(REPO_ROOT)}: {len(changes)} package(s) changed")
+        for line in changes:
+            print(f"  {line}")
+    else:
+        print("Snapshot already matches Colab's current pip freeze")
+
+    rc = 0
+    if (changed and args.relock) or args.force_relock:
+        notebooks = bootstrapped_notebooks()
+        print(f"Re-locking {len(notebooks)} notebook(s)")
+        rc = subprocess.run(
+            [sys.executable, str(Path(__file__).with_name("lock_notebook.py")), *map(str, notebooks)]
+        ).returncode
+
+    print(f"changed={'true' if changed else 'false'}")
+    if os.environ.get("GITHUB_OUTPUT"):
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+            f.write(f"changed={'true' if changed else 'false'}\n")
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/refresh-colab-snapshot.yml

@@ -0,0 +1,80 @@
+name: Refresh Colab snapshot (weekly)
+
+# Colab rebuilds its runtime image every week or two. When its preinstalled
+# versions move past `.github/colab-preinstalled.txt`, the notebooks' install
+# cells start downgrading packages on Colab. This job compares the snapshot with
+# googlecolab/backend-info and, when they differ, opens (or updates) one PR that
+# refreshes the snapshot and re-locks the notebooks against it.
+
+on:
+  schedule:
+    # Mondays at 05:00 UTC, an hour before the weekly notebook test run
+    - cron: '0 5 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: refresh-colab-snapshot
+  cancel-in-progress: false
+
+env:
+  BRANCH: bot/refresh-colab-snapshot
+
+jobs:
+  refresh:
+    name: Refresh snapshot and re-lock
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # A PAT lets the PR trigger "Test changed notebooks"; PRs opened with
+          # the default GITHUB_TOKEN do not trigger other workflows.
+          token: ${{ secrets.COLAB_REFRESH_TOKEN || github.token }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install tooling
+        run: pip install --no-cache-dir uv nbformat
+
+      - name: Refresh snapshot and re-lock notebooks
+        id: refresh
+        run: python .github/scripts/refresh_colab_snapshot.py --relock | tee refresh.log
+
+      - name: Open or update the PR
+        if: steps.refresh.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.COLAB_REFRESH_TOKEN || github.token }}
+        run: |
+          set -euo pipefail
+          {
+            echo "Colab's preinstalled package versions have moved past \`.github/colab-preinstalled.txt\`, so the notebooks' install cells were downgrading those packages on Colab. This refreshes the snapshot from [googlecolab/backend-info](https://github.com/googlecolab/backend-info) and re-locks every CI-tested notebook against it. Packages that Colab does not ship keep their current pins."
+            echo
+            echo "<details><summary>Snapshot changes</summary>"
+            echo
+            echo '```'
+            grep -E '^  \S+: ' refresh.log || true
+            echo '```'
+            echo
+            echo "</details>"
+          } > pr-body.md
+          rm -f refresh.log
+
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A -- .github/colab-preinstalled.txt '*.ipynb'
+          git commit -m "Refresh the Colab snapshot and re-lock notebooks"
+          git push --force origin "$BRANCH"
+
+          if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            gh pr edit "$BRANCH" --body-file pr-body.md
+          else
+            gh pr create --head "$BRANCH" --base "${{ github.event.repository.default_branch }}" \
+              --title "Refresh the Colab snapshot and re-lock notebooks" \
+              --body-file pr-body.md
+          fi

000108/chunglab/demo/2021-09-27_dandi-demo.ipynb

@@ -48,10 +48,11 @@
     "    \"cffi==2.1.1\" \\\n",
     "    \"charset-normalizer==3.4.9\" \\\n",
     "    \"ci-info==0.4.0\" \\\n",
-    "    \"click==8.4.2\" \\\n",
+    "    \"click==8.5.0\" \\\n",
     "    \"click-didyoumean==0.3.1\" \\\n",
+    "    \"cloudpickle==3.1.2\" \\\n",
     "    \"contourpy==1.3.3\" \\\n",
-    "    \"cryptography==50.0.0\" \\\n",
+    "    \"cryptography==50.0.1\" \\\n",
     "    \"cycler==0.12.1\" \\\n",
     "    \"dandi==0.77.0\" \\\n",
     "    \"dandischema==0.14.0\" \\\n",
@@ -61,16 +62,16 @@
     "    \"email-validator==2.3.0\" \\\n",
     "    \"etelemetry==0.3.1\" \\\n",
     "    \"fasteners==0.20\" \\\n",
-    "    \"fonttools==4.63.0\" \\\n",
+    "    \"fonttools==4.64.0\" \\\n",
     "    \"fqdn==1.5.1\" \\\n",
     "    \"frozenlist==1.8.0\" \\\n",
     "    \"fscacher==0.4.4\" \\\n",
-    "    \"fsspec==2025.3.0\" \\\n",
+    "    \"fsspec==2025.12.0\" \\\n",
     "    \"google-crc32c==1.8.0\" \\\n",
     "    \"h5py==3.16.0\" \\\n",
     "    \"hdmf==6.2.0\" \\\n",
     "    \"humanize==4.16.0\" \\\n",
-    "    \"idna==3.18\" \\\n",
+    "    \"idna==3.19\" \\\n",
     "    \"imageio==2.37.4\" \\\n",
     "    \"interleave==0.3.0\" \\\n",
     "    \"isodate==0.7.2\" \\\n",
@@ -81,20 +82,20 @@
     "    \"jeepney==0.9.0\" \\\n",
     "    \"jinja2==3.1.6\" \\\n",
     "    \"jinxed==2.1.0\" \\\n",
-    "    \"joblib==1.5.3\" \\\n",
+    "    \"joblib==1.6.0\" \\\n",
     "    \"jsonpointer==3.1.1\" \\\n",
     "    \"jsonschema==4.26.0\" \\\n",
     "    \"jsonschema-specifications==2025.9.1\" \\\n",
     "    \"keyring==25.7.0\" \\\n",
     "    \"keyrings-alt==5.0.2\" \\\n",
-    "    \"kiwisolver==1.5.0\" \\\n",
+    "    \"kiwisolver==1.5.1\" \\\n",
     "    \"lazy-loader==0.5\" \\\n",
     "    \"markupsafe==3.0.3\" \\\n",
     "    \"matplotlib==3.10.0\" \\\n",
     "    \"ml-dtypes==0.6.0\" \\\n",
     "    \"more-itertools==10.8.0\" \\\n",
     "    \"multidict==6.7.1\" \\\n",
-    "    \"narwhals==2.24.0\" \\\n",
+    "    \"narwhals==2.25.0\" \\\n",
     "    \"natsort==8.4.0\" \\\n",
     "    \"networkx==3.6.1\" \\\n",
     "    \"numcodecs==0.16.5\" \\\n",
@@ -103,12 +104,12 @@
     "    \"packaging==26.3\" \\\n",
     "    \"pandas==2.2.3\" \\\n",
     "    \"pillow==11.3.0\" \\\n",
-    "    \"platformdirs==4.11.3\" \\\n",
+    "    \"platformdirs==4.11.7\" \\\n",
     "    \"propcache==0.5.2\" \\\n",
     "    \"pycparser==3.0\" \\\n",
     "    \"pycryptodomex==3.23.0\" \\\n",
-    "    \"pydantic==2.13.4\" \\\n",
-    "    \"pydantic-core==2.46.4\" \\\n",
+    "    \"pydantic==2.13.5\" \\\n",
+    "    \"pydantic-core==2.46.5\" \\\n",
     "    \"pydantic-settings==2.15.0\" \\\n",
     "    \"pynwb==4.1.0\" \\\n",
     "    \"pyout==0.8.1\" \\\n",
@@ -131,15 +132,15 @@
     "    \"six==1.17.0\" \\\n",
     "    \"tenacity==9.1.4\" \\\n",
     "    \"tensorstore==0.1.85\" \\\n",
-    "    \"tifffile==2026.8.16\" \\\n",
+    "    \"tifffile==2026.8.23\" \\\n",
     "    \"tornado==6.5.7\" \\\n",
     "    \"tqdm==4.67.3\" \\\n",
     "    \"typing-extensions==4.16.0\" \\\n",
     "    \"typing-inspection==0.4.4\" \\\n",
     "    \"tzdata==2026.3\" \\\n",
     "    \"uri-template==1.3.0\" \\\n",
     "    \"urllib3==2.5.0\" \\\n",
-    "    \"wcwidth==0.8.2\" \\\n",
+    "    \"wcwidth==0.8.3\" \\\n",
     "    \"webcolors==25.10.0\" \\\n",
     "    \"xyzservices==2026.3.0\" \\\n",
     "    \"yarl==1.24.5\" \\\n",