diff --git a/config/jupyterhub/01-spawner.py b/config/jupyterhub/01-spawner.py index 917ee4b..95560bd 100644 --- a/config/jupyterhub/01-spawner.py +++ b/config/jupyterhub/01-spawner.py @@ -330,7 +330,12 @@ def _setup_trust_bundle(spawner): # ``kubespawner_override`` accepts any valid KubeSpawner trait so deployers # can add node_selector, image, extra_resource_limits (GPU), etc. without # code changes. Empty list = no profile selector (single-instance mode). -# Keys used only for group gating; KubeSpawner must never see them. +# Keys used only for group gating; KubeSpawner must never see them. They are +# stripped per user in _filter_profiles because gating is a per-user decision. +# ``image-variant`` is the other chart-only key — it is stripped once at load +# in _resolve_image_variants because its effect (image injection) is the same +# for every user and jhub-apps reads the resolved list too. A future custom key +# belongs in whichever of the two matches how it is evaluated. _PROFILE_GATING_KEYS = ("access", "groups", "users") @@ -442,7 +447,78 @@ async def _render_profile_list(spawner): return visible -_profiles = get_config("custom.profiles", []) +def _resolve_image_variants(profiles, base_name, base_tag, overrides): + """Inject a variant of the singleuser image into ``image-variant`` profiles. + + Every jupyterlab image variant (today only ``gpu``) is built from the same + commit as the CPU image and shares its ``sha-`` tag, so a profile + marked ``image-variant: gpu`` resolves to + ``-gpu:`` and stays current + across pack updates without a hardcoded SHA in the overlay (issue #230). + + Order of precedence for the injected image: + 1. the profile's own ``kubespawner_override.image`` — never touched + 2. ``custom.image-variants.`` — deployer override, full ref + (mirrored registries, a variant published elsewhere) + 3. ``-:`` — derived + 4. nothing — ``base_name``/``base_tag`` empty (schema-valid in z2jh); + the profile falls back to the CPU default image and the hub warns. + + The ``image-variant`` key is stripped whatever its value — KubeSpawner + must never see it. Returns new dicts; the input list is left untouched. + + Two cases only warn, because raising here would break hub startup (and + therefore login) for every user: the empty-ref fallback above, and a + profile that also declares ``profile_options.image`` — KubeSpawner applies + the selected choice's ``kubespawner_override`` AFTER the profile-level one + and replaces rather than merges, so the choice's image silently wins over + the injected one at spawn time (while jhub-apps still displays the + injected one). + """ + resolved = [] + for profile in profiles: + if "image-variant" not in profile: + resolved.append(profile) + continue + name = profile.get("slug") or profile.get("display_name") + variant = profile["image-variant"] + profile = {k: v for k, v in profile.items() if k != "image-variant"} + if not variant: + resolved.append(profile) + continue + override = dict(profile.get("kubespawner_override") or {}) + if not override.get("image"): + image = (overrides or {}).get(variant) + if not image and base_name and base_tag: + image = f"{base_name}-{variant}:{base_tag}" + if image: + override["image"] = image + log.info("profiles: %r image-variant %r — injected image %s", name, variant, image) + else: + log.warning( + "profiles: %r has image-variant %r but no image could be derived " + "(singleuser.image.name/tag empty?) and custom.image-variants.%s is " + "unset — the profile will spawn the CPU default image", + name, variant, variant, + ) + if "image" in (profile.get("profile_options") or {}): + log.warning( + "profiles: %r has image-variant %r but declares profile_options.image; " + "the selected choice's image overrides the injected one at spawn time — " + "drop the option or point its choices at the variant image", + name, variant, + ) + profile["kubespawner_override"] = override + resolved.append(profile) + return resolved + + +_profiles = _resolve_image_variants( + get_config("custom.profiles", []), + get_config("singleuser.image.name", ""), + get_config("singleuser.image.tag", ""), + get_config("custom.image-variants", {}) or {}, +) if _profiles: c.KubeSpawner.profile_list = _render_profile_list log.info( diff --git a/docs/src/content/docs/server-profiles.md b/docs/src/content/docs/server-profiles.md index 7693945..f3024bc 100644 --- a/docs/src/content/docs/server-profiles.md +++ b/docs/src/content/docs/server-profiles.md @@ -150,6 +150,72 @@ kubectl -n data-science logs deploy/hub | grep -i "profiles:\|groups" ## GPU profiles +### The GPU image, without hardcoding a SHA + +Mark a profile `image-variant: gpu` and the chart injects the matching GPU JupyterLab image +(`nebari-data-science-pack-jupyterlab-gpu`) at the chart's current tag: + +```yaml + - slug: gpu-instance + display_name: "G4 GPU Instance" + image-variant: gpu + access: yaml + groups: + - gpu-access + kubespawner_override: + # no image needed — the -gpu image is injected automatically + node_selector: + node.kubernetes.io/instance-type: g4dn.xlarge + extra_resource_limits: + nvidia.com/gpu: 1 +``` + +Both JupyterLab images are built from the same commit and share the same `sha-` tag, so +the derived ref (`-gpu:`) exists on +`quay.io/nebari` for every release and GPU profiles track pack updates exactly like CPU +profiles — no more stale SHAs in the overlay +([issue #230](https://github.com/nebari-dev/data-science-pack/issues/230)). + +The derivation is generic — `image-variant: ` resolves to +`-:` — but `gpu` is the only variant +published today. Like the gating keys, `image-variant` is stripped (whatever its value) before +the profile reaches KubeSpawner. The hub logs the injected ref at startup: + +```bash +kubectl -n data-science logs deploy/hub | grep "profiles:.*gpu" +``` + +:::caution[Mirrored registries] +The derivation only rewrites the image *name*. If you point `jupyterhub.singleuser.image.name` +at a mirror (ECR, an airgapped registry), the chart derives `-gpu:`, which does +not exist unless you mirrored it too. Nothing validates the ref at `helm upgrade` time — the +first GPU spawn fails with `ImagePullBackOff`. Either mirror the `-gpu` image under that name +or map it under `jupyterhub.custom.image-variants`: + +```yaml + image-variants: + gpu: 123456789012.dkr.ecr.us-east-1.amazonaws.com/lab-gpu:sha-abc1234 +``` + +The `-gpu` image is built for `linux/amd64` only; the CPU image is multi-arch. +::: + +Two things override the injection: + +- **An explicit `kubespawner_override.image`** always wins. Note that + `scripts/bump_image_tags.py` only rewrites the CPU image ref, so a hand-pinned `-gpu` + image stays frozen across releases — the exact problem `image-variant` exists to solve. Prefer + the key over pinning. +- **`profile_options.image`.** KubeSpawner applies the selected choice's + `kubespawner_override` *after* the profile-level one and replaces rather than merges, so + an image choice silently puts the CPU image on the GPU node (while jhub-apps' Create App + still displays the injected GPU image). The shipped CPU profiles carry such an option — + do not copy it onto an `image-variant` profile. The hub logs a warning if you do. + +`jupyterhub.custom.image-variants.` changes which image gets injected for a variant chart-wide. + +### Scheduling onto GPU nodes + A GPU profile requests the resource through `extra_resource_limits`, but scheduling onto a tainted GPU node group also needs a toleration — whether you must add it yourself depends on whether the cluster runs the `ExtendedResourceToleration` admission controller (EKS and GKE diff --git a/docs/src/content/docs/values-reference.md b/docs/src/content/docs/values-reference.md index 0e1b61d..ee143d5 100644 --- a/docs/src/content/docs/values-reference.md +++ b/docs/src/content/docs/values-reference.md @@ -190,6 +190,7 @@ always win. | `external-url` | `""` *(derived)* | Hub bind hostname. | | `nebi-image` | `""` *(derived)* | `repository:tag` copied into user pods. | | `nebi-image-pull-policy` | `IfNotPresent` | — | +| `image-variants` | `{}` | Per-variant image overrides for `image-variant: ` profiles. Default derivation is `-:` — see [Server profiles](/server-profiles/#gpu-profiles). | | `jhub-app-proxy-version` | `v0.2.3` | Installed at app-spawn time. Must be ≥ v0.2.3 for apps to run inside a Nebi (pixi) environment; older versions only activate conda and fall back to the base env. | | `nebi-remote-url` | `""` *(derived)* | Browser-facing Nebi URL. | | `nebi-internal-url` | `""` *(derived)* | In-cluster Nebi URL. | diff --git a/docs/superpowers/specs/2026-08-24-gpu-profile-image-design.md b/docs/superpowers/specs/2026-08-24-gpu-profile-image-design.md new file mode 100644 index 0000000..85d6c7d --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-gpu-profile-image-design.md @@ -0,0 +1,116 @@ +# GPU profile image auto-derivation + +Issue: https://github.com/nebari-dev/data-science-pack/issues/230 + +## Problem + +Deployers who add GPU JupyterLab profiles must hardcode +`quay.io/nebari/nebari-data-science-pack-jupyterlab-gpu:sha-` in their +overlay. CPU profiles inherit the chart's image tag on every pack update; GPU +profiles silently fall behind until someone remembers to bump the SHA. + +Both `nebari-data-science-pack-jupyterlab` and +`nebari-data-science-pack-jupyterlab-gpu` are built by the same +`build-images.yaml` workflow from the same commit, so they always share the +same `sha-` tag. The chart therefore already knows the correct GPU +image ref: `-gpu:`. + +## Design + +A new per-profile key `image-variant: ` in `jupyterhub.custom.profiles`: + +```yaml +- slug: gpu + display_name: "GPU Access" + image-variant: gpu + access: yaml + groups: [gpu-access] + kubespawner_override: + # no image needed — injected automatically + node_selector: {node.kubernetes.io/instance-type: g4dn.xlarge} + extra_resource_limits: {nvidia.com/gpu: 1} +``` + +### Components + +1. **Spawner config** (`config/jupyterhub/01-spawner.py`): at load, + `_resolve_image_variants(profiles, base_name, base_tag, overrides)` + walks `custom.profiles`, reading `singleuser.image.name`/`.tag` and + `custom.image-variants` from z2jh. For each `image-variant: `: + * no `kubespawner_override.image` → inject, in order of precedence, + `custom.image-variants.` if set, else + `-:`; if neither can be produced + (`singleuser.image.name`/`tag` empty, schema-valid in z2jh) warn and + fall back to the CPU default image + * explicit image → leave it alone (explicit wins) + * the key is always stripped so KubeSpawner never sees it + * `profile_options.image` present → warn (see Rejected alternatives) + + Resolution happens once at module load, before `_render_profile_list` + filtering, so jhub-apps' server-types endpoint sees the same image. + +2. **Override map** `jupyterhub.custom.image-variants: {}` in `values.yaml` + — full refs per variant, for mirrored registries or a variant published + elsewhere. Read directly via z2jh `get_config` (it is a mapping, so the + `get_chart_config` empty-string convention does not apply). + +No Helm-side change: the derivation needs the variant name, which only the +profile knows, so it lives in Python where both inputs are available. + +### Why `image-variant: ` rather than a boolean `gpu: true` + +* The image naming already has a variant axis (`-gpu`), and a `-rocm` or + arm64 runtime image is plausible. A boolean would have to coexist with a + string key forever once it shipped in `values.yaml`. +* Same amount of code today; the derivation string is the only place the + variant name appears. +* Cost: `image-variant: gpu` reads slightly less naturally than `gpu: true` + in an overlay. Mitigated by the example in `values.yaml` and the docs. + +### Rejected alternatives + +* **Boolean `gpu: true`.** The first draft of this PR. Rejected in review + for the reason above: it is an API surface we cannot drop without a + deprecation cycle, and the string key costs nothing extra. +* **Deriving in Helm (`_CHART_DERIVED["gpu-image"]`).** Also the first + draft. Works for one hardcoded variant but cannot be generic — Helm does + not see the profile list the z2jh subchart consumes, so it cannot know + which variant names are in use. +* **Injecting into `profile_options.image.choices` too.** Choices always + carry an explicit image (that is their purpose), so injecting there + would overwrite explicit deployer values and contradict "explicit wins". + Instead the hub warns when a variant profile declares + `profile_options.image`, and the docs say not to combine them. +* **Raising on an empty derived image.** Would break hub startup, and + therefore login, for every user. A warning plus fallback to the CPU + default image is the right level. + +### Automatic tag currency + +`scripts/bump_image_tags.py` already bumps `jupyterhub.singleuser.image.tag` +every release; the derived GPU ref follows with zero script changes. + +### Out of scope (YAGNI) + +* Auto-injecting `extra_resource_limits` / tolerations — cluster-specific, + already documented. +* Rewriting `profile_options.image.choices` for GPU profiles — explicit + choices keep winning at spawn time; the hub warns when a variant profile + declares that option (see Rejected alternatives). +* Teaching `scripts/bump_image_tags.py` to bump explicitly pinned `-gpu` + refs, and recording the build invariant (both jupyterlab images from one + `build-images.yaml` run) — follow-ups. + +## Testing + +* Unit (`tests/unit/test_spawner_profiles.py`): derivation, generic + variant names, override map, explicit-image precedence, key stripping + (including empty variant), empty-base fallback + warning, + `profile_options.image` warning (+ negative), input non-mutation, + load-time wiring from z2jh keys, load-time log naming the injected ref. + +## Docs + +* `docs/src/content/docs/server-profiles.md` GPU section: document `image-variant`. +* `docs/src/content/docs/values-reference.md`: `image-variants` row. +* `values.yaml` comments for `image-variants` + profile example. diff --git a/tests/unit/test_chart_derived.py b/tests/unit/test_chart_derived.py index fe94183..ffc2ca9 100644 --- a/tests/unit/test_chart_derived.py +++ b/tests/unit/test_chart_derived.py @@ -148,3 +148,4 @@ def test_get_chart_config_explicit_override_wins(rendered_chart_derived): ) got = ns["get_chart_config"]("external-url") assert got == "explicit.example.com" + diff --git a/tests/unit/test_spawner_profiles.py b/tests/unit/test_spawner_profiles.py index 4d1a48f..ee4ad53 100644 --- a/tests/unit/test_spawner_profiles.py +++ b/tests/unit/test_spawner_profiles.py @@ -268,6 +268,235 @@ def test_render_profile_list_hides_restricted_profile_from_outsider(): assert [p["slug"] for p in visible] == ["small"] +BASE_NAME = "quay.io/nebari/nebari-data-science-pack-jupyterlab" +BASE_TAG = "sha-5dfee5e" +GPU_IMAGE = f"{BASE_NAME}-gpu:{BASE_TAG}" + + +def _resolve(mod, profiles, base_name=BASE_NAME, base_tag=BASE_TAG, overrides=None): + return mod._resolve_image_variants(profiles, base_name, base_tag, overrides or {}) + + +def test_variant_profile_gets_derived_image_injected(): + """An ``image-variant: gpu`` profile with no explicit image gets + ``-gpu:``, so deployers stop + hardcoding SHAs (issue #230).""" + mod, _ = _load() + + profiles = [{"slug": "gpu", "image-variant": "gpu", "kubespawner_override": {"cpu_limit": 4}}] + resolved = _resolve(mod, profiles) + + override = resolved[0]["kubespawner_override"] + assert override["image"] == GPU_IMAGE, ( + f"expected the derived variant image to be injected, got {override!r}" + ) + assert override["cpu_limit"] == 4, "other kubespawner_override keys must survive" + + +def test_variant_name_is_generic(): + """The derivation is ``-:`` for any variant string — + a future -rocm or arm64 build needs no code change.""" + mod, _ = _load() + + resolved = _resolve(mod, [{"slug": "amd", "image-variant": "rocm"}]) + + got = resolved[0]["kubespawner_override"]["image"] + assert got == f"{BASE_NAME}-rocm:{BASE_TAG}", f"unexpected derived ref {got!r}" + + +def test_variant_override_map_wins_over_derivation(): + """``custom.image-variants.`` replaces the derived ref chart-wide — + the escape hatch for mirrored registries.""" + mod, _ = _load() + + resolved = _resolve( + mod, + [{"slug": "gpu", "image-variant": "gpu"}], + overrides={"gpu": "mirror.example.com/lab-gpu:v1"}, + ) + + got = resolved[0]["kubespawner_override"]["image"] + assert got == "mirror.example.com/lab-gpu:v1", f"override map ignored, got {got!r}" + + +def test_variant_profile_explicit_image_wins(): + """An explicit ``kubespawner_override.image`` is never replaced — the + deployer opted out of derivation for that profile.""" + mod, _ = _load() + + profiles = [{"slug": "gpu", "image-variant": "gpu", "kubespawner_override": {"image": "custom:1"}}] + resolved = _resolve(mod, profiles, overrides={"gpu": "override:1"}) + + got = resolved[0]["kubespawner_override"]["image"] + assert got == "custom:1", f"explicit image was overwritten with {got!r}" + + +def test_variant_key_is_stripped_before_kubespawner(): + """The ``image-variant`` marker is chart-only: KubeSpawner must never see + it, and a profile with no ``kubespawner_override`` still gets the image.""" + mod, _ = _load() + + profiles = [ + {"slug": "gpu", "image-variant": "gpu"}, + {"slug": "gpu2", "image-variant": "gpu", "kubespawner_override": {"image": "custom:1"}}, + {"slug": "cpu", "image-variant": ""}, + ] + resolved = _resolve(mod, profiles) + + assert all("image-variant" not in p for p in resolved), f"key leaked: {resolved!r}" + assert resolved[0]["kubespawner_override"]["image"] == GPU_IMAGE, ( + "a variant profile without kubespawner_override should still get the image" + ) + assert resolved[2] == {"slug": "cpu"}, ( + f"an empty variant must be stripped without injecting, got {resolved[2]!r}" + ) + + +def test_non_variant_profile_is_untouched(): + """Profiles without the ``image-variant`` key pass through byte-for-byte.""" + mod, _ = _load() + + profiles = [{"slug": "small", "kubespawner_override": {"cpu_limit": 1}}] + resolved = _resolve(mod, profiles) + + assert resolved == profiles, f"plain profile was modified: {resolved!r}" + + +def test_variant_without_base_image_falls_back_to_default(caplog): + """When ``singleuser.image.name``/``tag`` is empty (schema-valid in z2jh) + and no override is set, the key is still stripped, no image is injected, + and the hub warns: this silently lands the CPU image on a GPU node, but + raising would break hub startup and therefore login.""" + mod, _ = _load() + + profiles = [{"slug": "gpu", "image-variant": "gpu", "kubespawner_override": {"cpu_limit": 4}}] + with caplog.at_level("WARNING"): + resolved = _resolve(mod, profiles, base_name="", base_tag=BASE_TAG) + + assert "image-variant" not in resolved[0], "key must be stripped even without an image" + assert "image" not in resolved[0]["kubespawner_override"], ( + "no image should be injected when the ref cannot be derived" + ) + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert any("gpu" in w and "image-variants" in w for w in warnings), ( + f"expected a warning naming the profile and custom.image-variants, got {warnings!r}" + ) + + +def test_variant_profile_with_image_choices_warns(caplog): + """KubeSpawner applies ``profile_options.image.choices.*.kubespawner_override`` + AFTER the profile-level override and replaces rather than merges, so an + image choice silently defeats the injection. The hub must say so.""" + mod, _ = _load() + + profiles = [ + { + "slug": "gpu", + "image-variant": "gpu", + "profile_options": { + "image": { + "display_name": "Image", + "choices": { + "default": { + "display_name": "cpu-lab:sha-1", + "default": True, + "kubespawner_override": {"image": "cpu-lab:sha-1"}, + } + }, + } + }, + } + ] + with caplog.at_level("WARNING"): + _resolve(mod, profiles) + + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert any("profile_options" in w and "gpu" in w for w in warnings), ( + f"expected a warning that profile_options.image overrides the variant image, got {warnings!r}" + ) + + +def test_variant_profile_without_image_choices_does_not_warn(caplog): + """The choices warning is specific: a plain variant profile (or one with + non-image profile_options) stays quiet.""" + mod, _ = _load() + + profiles = [ + {"slug": "gpu", "image-variant": "gpu"}, + {"slug": "gpu2", "image-variant": "gpu", "profile_options": {"size": {"choices": {}}}}, + ] + with caplog.at_level("WARNING"): + _resolve(mod, profiles) + + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert warnings == [], f"unexpected warnings: {warnings!r}" + + +def test_variant_resolution_does_not_mutate_input_profiles(): + """Resolution returns new dicts; the z2jh-provided list is left intact.""" + mod, _ = _load() + + profiles = [{"slug": "gpu", "image-variant": "gpu", "kubespawner_override": {"cpu_limit": 4}}] + _resolve(mod, profiles) + + assert profiles == [ + {"slug": "gpu", "image-variant": "gpu", "kubespawner_override": {"cpu_limit": 4}} + ], f"input profiles were mutated: {profiles!r}" + + +def _load_with_variant_profile(overrides=None): + """Load 01-spawner.py with one ``image-variant: gpu`` profile and the + z2jh ``singleuser.image`` values the hub reads in production.""" + z2jh = sys.modules["z2jh"] + prior = z2jh.get_config + + def fake_get_config(key, default=None): + return { + "custom.profiles": [{"slug": "gpu", "image-variant": "gpu"}], + "custom.image-variants": overrides or {}, + "singleuser.image.name": BASE_NAME, + "singleuser.image.tag": BASE_TAG, + }.get(key, default) + + z2jh.get_config = fake_get_config + try: + return _load() + finally: + z2jh.get_config = prior + + +def test_variant_image_injected_at_load_time(): + """Module load resolves variant profiles from custom.profiles + + singleuser.image + custom.image-variants, so both the spawner and + jhub-apps see the injected image.""" + mod, _ = _load_with_variant_profile() + + assert mod._profiles == [{"slug": "gpu", "kubespawner_override": {"image": GPU_IMAGE}}], ( + f"load-time resolution did not inject the variant image: {mod._profiles!r}" + ) + + +def test_variant_override_map_applied_at_load_time(): + """``custom.image-variants`` is read from z2jh at load and wins over + the derivation.""" + mod, _ = _load_with_variant_profile(overrides={"gpu": "mirror.example.com/lab-gpu:v1"}) + + got = mod._profiles[0]["kubespawner_override"]["image"] + assert got == "mirror.example.com/lab-gpu:v1", f"override map ignored at load, got {got!r}" + + +def test_load_log_names_the_injected_variant_image(caplog): + """``kubectl logs deploy/hub`` must be able to answer which image a variant + profile got — the load-time info line carries the derived ref.""" + with caplog.at_level("INFO"): + _load_with_variant_profile() + + infos = [r.getMessage() for r in caplog.records if r.levelname == "INFO"] + assert any(GPU_IMAGE in m and "gpu" in m for m in infos), ( + f"expected an info line naming the injected image, got {infos!r}" + ) + + def test_profile_list_is_the_filtering_callable_when_profiles_configured(): """When profiles exist, KubeSpawner.profile_list is wired to the per-user callable, not the raw static list.""" diff --git a/values.yaml b/values.yaml index de2f7ce..4ea31c0 100644 --- a/values.yaml +++ b/values.yaml @@ -390,6 +390,20 @@ jupyterhub: # If empty, derived as `:`. nebi-image: "" nebi-image-pull-policy: "IfNotPresent" + # Per-variant image overrides (full refs) for profiles marked + # ``image-variant: `` (see the profiles docs below). By default a + # variant resolves to ``-:`` + # — every variant is built from the same commit as the CPU image, so the tags + # always match and variant profiles track pack updates automatically. ``gpu`` + # is the only variant published today (linux/amd64 only; the CPU image is + # multi-arch). + # If you mirror ``singleuser.image.name`` into a private/airgapped registry, + # mirror the ``-gpu`` image too or map it here — the derived ref is not + # validated at install time, so a missing mirror only shows up as + # ImagePullBackOff on the first GPU spawn. + # image-variants: + # gpu: 123456789012.dkr.ecr.us-east-1.amazonaws.com/lab-gpu:sha-abc1234 + image-variants: {} # jhub-app-proxy version installed at app-spawn time. Must be >= v0.2.3 for # apps to run inside a Nebi (pixi) environment; older versions only support # conda activation and fall back to the base env (missing packages). @@ -486,9 +500,23 @@ jupyterhub: # access: yaml # groups: # - gpu-access + # image-variant: gpu # inject the -gpu jupyterlab image automatically # kubespawner_override: # extra_resource_limits: # nvidia.com/gpu: 1 + # Profiles marked ``image-variant: `` get ``kubespawner_override.image`` + # set to ``-:`` (or the + # ``image-variants`` override above) unless an explicit image is given, so + # deployers never hardcode a -gpu SHA. The ``image-variant`` key is stripped + # (whatever its value) before the profile reaches KubeSpawner. Two caveats: + # * an explicit ``kubespawner_override.image`` is never touched — and + # scripts/bump_image_tags.py only rewrites the CPU image, so a pinned + # -gpu ref stays frozen across releases (the exact problem image-variant + # exists to avoid). Prefer the key over pinning. + # * do not combine ``image-variant`` with ``profile_options.image`` — the + # selected choice's image replaces the injected one at spawn time + # (KubeSpawner applies choice overrides after profile overrides). The + # hub logs a warning if you do. # NOTE: when bumping singleuser.image.tag below, also bump the # ``image: ...`` lines inside each profile_options.image.choices.default # entry so the profile selector shows the right tag. (z2jh values.yaml