From 1ef138b2555e95dc9dff764ea0144361f833599e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:43:26 +0100 Subject: [PATCH 01/13] ci: add k8s stack PR preview via labeled deploy + Cloudflare Tunnel Labeling a PR with deploy-preview spins up the full NIC platform stack plus this chart in an ephemeral k3d cluster, then exposes JupyterHub through a per-PR Cloudflare Tunnel gated by Cloudflare Access (GitHub org SSO), posting the link as a sticky PR comment. Includes a standalone tunnel-only smoketest workflow for validating the Cloudflare plumbing without the full stack deploy. --- .../k8s-preview-tunnel-smoketest.yaml | 138 ++++++++ .github/workflows/k8s-preview.yaml | 309 ++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 .github/workflows/k8s-preview-tunnel-smoketest.yaml create mode 100644 .github/workflows/k8s-preview.yaml diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml new file mode 100644 index 0000000..4392d0d --- /dev/null +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -0,0 +1,138 @@ +name: K8s Preview Tunnel Smoketest +# Manual, on-demand check of the Cloudflare Tunnel + DNS + Access plumbing +# that k8s-preview.yaml depends on — without paying for a full k3d + +# platform-stack + helm deploy on every iteration. Serves a static +# index.html instead of JupyterHub; same tunnel/DNS create+configure+ +# delete API calls as the real workflow, same secrets. Run this first +# when validating the Cloudflare-side setup (token scopes, Access +# application, GitHub identity provider); once a visit to the printed +# URL round-trips through GitHub SSO successfully, k8s-preview.yaml's +# tunnel plumbing is known-good and any remaining issue is in the +# k3d/helm/chart side, not Cloudflare. + +on: + workflow_dispatch: + +env: + PREVIEW_DOMAIN: dspack.iakte.ch + CLOUDFLARED_VERSION: "2026.7.3" + CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" + +jobs: + smoketest: + name: Tunnel smoketest + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Serve a trivial static page + run: | + mkdir -p /tmp/preview-test + cat > /tmp/preview-test/index.html <<'EOF' + +

Tunnel smoketest OK

+ EOF + python3 -m http.server 8000 --directory /tmp/preview-test \ + > /tmp/http-server.log 2>&1 & + + - name: Install cloudflared + run: | + curl -fsSL -o /tmp/cloudflared \ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64" + echo "${CLOUDFLARED_SHA256} /tmp/cloudflared" | sha256sum -c - + chmod +x /tmp/cloudflared + + - name: Create Cloudflare Tunnel for this run + id: cf_tunnel + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} + run: | + tunnel_secret=$(openssl rand -base64 32) + echo "::add-mask::${tunnel_secret}" + + create_resp=$(curl -fsS -X POST \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg name "smoketest-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") + tunnel_id=$(jq -r '.result.id' <<< "$create_resp") + if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then + echo "::error::Tunnel creation failed: $create_resp" + exit 1 + fi + echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" + + token_resp=$(curl -fsS \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/token" \ + -H "Authorization: Bearer ${CF_API_TOKEN}") + tunnel_token=$(jq -r '.result' <<< "$token_resp") + echo "::add-mask::${tunnel_token}" + echo "TUNNEL_TOKEN=${tunnel_token}" >> "$GITHUB_ENV" + + curl -fsS -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" \ + '{config: {ingress: [{hostname: $host, service: "http://localhost:8000"}, {service: "http_status:404"}]}}')" \ + > /dev/null + + - name: Point DNS at the tunnel + id: cf_dns + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} + run: | + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') + if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then + echo "::error::Could not resolve zone id for iakte.ch" + exit 1 + fi + echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" + + record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ + '{type: "CNAME", name: $host, content: $target, proxied: true}')") + record_id=$(jq -r '.result.id' <<< "$record_resp") + if [ -z "$record_id" ] || [ "$record_id" = "null" ]; then + echo "::error::DNS record creation failed: $record_resp" + exit 1 + fi + echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" + + echo "## Smoketest URL" >> "$GITHUB_STEP_SUMMARY" + echo "https://${PREVIEW_HOSTNAME}" >> "$GITHUB_STEP_SUMMARY" + echo "Visiting it should challenge you with Cloudflare Access GitHub SSO," >> "$GITHUB_STEP_SUMMARY" + echo "then show 'Tunnel smoketest OK' once you're through." >> "$GITHUB_STEP_SUMMARY" + echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" + + - name: Run tunnel until the job times out + run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + + - name: Delete DNS record + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + run: | + [ -n "${ZONE_ID:-}" ] && [ -n "${DNS_RECORD_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Delete Cloudflare Tunnel + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + run: | + [ -n "${TUNNEL_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml new file mode 100644 index 0000000..cbf510f --- /dev/null +++ b/.github/workflows/k8s-preview.yaml @@ -0,0 +1,309 @@ +name: K8s Stack Preview +# Deploys the full Nebari platform stack (Keycloak + nic-operator + Envoy +# Gateway, via nebari-dev/action-nebari-sandbox's `platform` profile) plus +# this PR's chart into an ephemeral k3d cluster on the runner, then exposes +# JupyterHub through a per-PR Cloudflare Tunnel behind Cloudflare Access +# (GitHub-org SSO) so a reviewer can click a link and use it. +# +# This repo is public, so a plain shared secret posted in the PR comment +# (a quick-tunnel URL, a basic-auth password) is readable by anyone who +# opens the PR, not just intended reviewers. Access closes that gap by +# authenticating the *person*, not a string in the comment: Cloudflare +# challenges every request to *.dspack.iakte.ch with a GitHub SSO +# login and only lets it through to cloudflared if the signed-in account +# is a member of this GitHub org. The PR comment only ever contains a URL. +# +# iakte.ch is a placeholder domain used temporarily because it lives in +# the same Cloudflare account as the Tunnel and Zero Trust setup; swap +# PREVIEW_DOMAIN below once a permanent domain is available in that +# account. +# +# Scope, deliberately: the tunnel points straight at the `proxy-public` +# service (dummy-authenticator login, same as local Tilt dev), not the +# operator-provisioned NebariApp/OIDC route — that route exists for its +# own Keycloak realm and redirect URIs, unrelated to Access's GitHub SSO. +# The NebariApp CRD is still installed (auth.enabled=true) purely so the +# operator/Keycloak reconcile is exercised and its Ready condition +# reported in the run; it is not what the reviewer clicks through. +# +# The link only lives for the run's duration (bounded by timeout-minutes +# below) — it is not a persistent per-PR environment. Each run creates its +# own Cloudflare Tunnel + DNS record (so concurrent previews on different +# PRs don't collide on the same route) and deletes both on cleanup. +# +# One-time setup this workflow assumes already exists (Cloudflare Zero +# Trust dashboard, done by a repo admin, not scripted here). The tunnel, +# the `iakte.ch` zone, and Zero Trust/Access all live in ONE Cloudflare +# account (aktechlabs) — not the account behind CLOUDFLARE_API_TOKEN / +# CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for Pages: +# - Zone `iakte.ch` in that account, with an Access self-hosted +# application for `*.dspack.iakte.ch`, GitHub as identity provider, +# policy scoped to this org. +# - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the +# `cfd_tunnel` API is account-scoped; can't be derived from the +# token alone). +# - Secret CLOUDFLARE_TUNNEL_API_TOKEN: a custom token scoped to +# EXACTLY three permissions, nothing broader: +# * Account -> Cloudflare Tunnel -> Edit +# * Zone -> Zone -> Read (to resolve the zone id by name) +# * Zone -> DNS -> Edit (to create/delete the CNAME record) +# Zone Resources: Include -> Specific zone -> iakte.ch. +# Account Resources: Include -> Specific account -> aktechlabs. +# +# Only runs when a maintainer/collaborator adds the `deploy-preview` label +# (GitHub restricts who can label a PR) — arbitrary PR authors, including +# from forks, cannot trigger this themselves. Even so, a fork PR still runs +# attacker-authored code once labeled; the comment flags this so whoever +# labels it is doing so knowingly. +# +# Residual risk not covered here: k3d pods share the runner's Docker daemon +# rather than being hardware-isolated, and k3d's default CNI (flannel) +# does not enforce NetworkPolicy, so a container escape or outbound abuse +# from inside a spawned notebook pod is not blocked at the network layer. +# Per-job GITHUB_TOKEN permissions are scoped to the minimum each job +# needs so a compromised runner in the exposed 90-minute window can't use +# an ambient token to touch other workflows or repo state. + +on: + pull_request: + types: [labeled, unlabeled, synchronize] + +concurrency: + group: k8s-preview-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + PREVIEW_LABEL: deploy-preview + PREVIEW_DOMAIN: dspack.iakte.ch + CLOUDFLARED_VERSION: "2026.7.3" + # sha256 of cloudflared-linux-amd64 for the pinned version above, + # computed from the official release asset at + # https://github.com/cloudflare/cloudflared/releases/tag/2026.7.3 + CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" + +jobs: + deploy-preview: + if: contains(github.event.pull_request.labels.*.name, 'deploy-preview') && github.event.action != 'unlabeled' + name: Deploy preview + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Provision sandbox (k3d + full NIC platform stack) + id: sandbox + uses: nebari-dev/action-nebari-sandbox@dbdb054c16efee3b29e26aee406c12fb9639bfd0 # v2.2.0 + with: + profile: platform + cluster-name: pr-preview-${{ github.event.pull_request.number }} + + - name: Build hub image from this PR + run: docker build --target jupyterhub -t nebari-data-science-pack-jupyterhub:preview images/ + + - name: Install k3d CLI (for image import) + run: | + curl -fsSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | \ + TAG=v5.8.3 bash + + - name: Side-load hub image into the sandbox cluster + run: k3d image import nebari-data-science-pack-jupyterhub:preview -c ${{ steps.sandbox.outputs.cluster-name }} + + - name: Deploy chart + id: deploy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + helm upgrade --install preview . \ + --namespace pr-preview --create-namespace \ + --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ + --set jupyterhub.hub.image.tag=preview \ + --set nebariapp.enabled=true \ + --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}.${{ steps.sandbox.outputs.domain }}" \ + --set nebariapp.auth.enabled=true \ + --wait --timeout 5m + + - name: Wait for hub + proxy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview rollout status deployment/hub --timeout=180s + kubectl -n pr-preview rollout status deployment/proxy --timeout=180s + + - name: Report NebariApp reconcile status + if: always() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview get nebariapp -o wide || true + kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true + + - name: Port-forward JupyterHub proxy + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview port-forward svc/proxy-public 8000:80 \ + > /tmp/port-forward.log 2>&1 & + echo "PORT_FORWARD_PID=$!" >> "$GITHUB_ENV" + + - name: Install cloudflared + run: | + curl -fsSL -o /tmp/cloudflared \ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64" + echo "${CLOUDFLARED_SHA256} /tmp/cloudflared" | sha256sum -c - + chmod +x /tmp/cloudflared + + # A per-run named Tunnel (not the anonymous quick-tunnel) so: + # (a) it can sit behind an Access application (quick tunnels have no + # account/zone attached, so no policy can be bound to them), and + # (b) each PR gets its own tunnel + hostname, so two PRs previewing + # at once don't share one route and cross-talk. + - name: Create Cloudflare Tunnel for this PR + id: cf_tunnel + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + run: | + tunnel_secret=$(openssl rand -base64 32) + echo "::add-mask::${tunnel_secret}" + + create_resp=$(curl -fsS -X POST \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg name "pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") + tunnel_id=$(jq -r '.result.id' <<< "$create_resp") + if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then + echo "::error::Tunnel creation failed: $create_resp" + exit 1 + fi + echo "tunnel_id=${tunnel_id}" >> "$GITHUB_OUTPUT" + echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" + + token_resp=$(curl -fsS \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/token" \ + -H "Authorization: Bearer ${CF_API_TOKEN}") + tunnel_token=$(jq -r '.result' <<< "$token_resp") + echo "::add-mask::${tunnel_token}" + echo "TUNNEL_TOKEN=${tunnel_token}" >> "$GITHUB_ENV" + + curl -fsS -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${tunnel_id}/configurations" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" \ + '{config: {ingress: [{hostname: $host, service: "http://localhost:8000"}, {service: "http_status:404"}]}}')" \ + > /dev/null + + - name: Point DNS at the tunnel + id: cf_dns + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} + run: | + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') + if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then + echo "::error::Could not resolve zone id for iakte.ch" + exit 1 + fi + echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" + + record_resp=$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg host "$PREVIEW_HOSTNAME" --arg target "${TUNNEL_ID}.cfargotunnel.com" \ + '{type: "CNAME", name: $host, content: $target, proxied: true}')") + record_id=$(jq -r '.result.id' <<< "$record_resp") + if [ -z "$record_id" ] || [ "$record_id" = "null" ]; then + echo "::error::DNS record creation failed: $record_resp" + exit 1 + fi + echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" + echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + + - name: Comment preview link on PR + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **K8s stack preview** for `${{ github.event.pull_request.head.ref }}`: + ${{ steps.cf_dns.outputs.url }} + + ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} + + You'll be asked to sign in via Cloudflare Access (GitHub SSO) before + reaching JupyterHub — only members of this GitHub org get through. + **Then JupyterHub login:** dummy authenticator, any username + any password. + + This goes straight to JupyterHub's proxy and skips the operator-provisioned + OIDC/Keycloak route (unrelated realm/redirect URIs to Access's GitHub SSO). + The `NebariApp` resource is still deployed alongside it so the operator/ + Keycloak reconcile itself is exercised — see the "Report NebariApp reconcile + status" step in this run for its `Ready` condition. + + Live only for this job's run (up to 90 min), or until the `deploy-preview` + label is removed. Push a new commit or re-add the label to redeploy. + + - name: Run tunnel until the job times out + run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + + - name: Delete DNS record + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + run: | + [ -n "${ZONE_ID:-}" ] && [ -n "${DNS_RECORD_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${DNS_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Delete Cloudflare Tunnel + if: always() + env: + CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_TUNNEL_ACCOUNT_ID }} + run: | + [ -n "${TUNNEL_ID:-}" ] || exit 0 + curl -fsS -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" || true + + - name: Cleanup cluster + if: always() + run: k3d cluster delete ${{ steps.sandbox.outputs.cluster-name }} || true + + cleanup-preview: + if: github.event.action == 'unlabeled' && github.event.label.name == 'deploy-preview' + name: Stop preview + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: write + steps: + - name: Cancel the in-flight preview run for this PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_id=$(gh api "repos/${{ github.repository }}/actions/runs?event=pull_request&status=in_progress" \ + --jq '.workflow_runs[] | select(.name == "K8s Stack Preview") | select(.pull_requests[]?.number == ${{ github.event.pull_request.number }}) | .id' \ + | head -1) + if [ -n "$run_id" ]; then + gh run cancel "$run_id" --repo "${{ github.repository }}" + fi + + - name: Comment that the preview stopped + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **K8s stack preview** stopped — the `deploy-preview` label was removed. + + Add it again to redeploy. From 72dab4607f8aa7a54d6032f1d1527273f3f82127 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:47:01 +0100 Subject: [PATCH 02/13] ci: temporarily trigger smoketest on this PR for iteration pull_request/synchronize on this one file, scoped narrowly, so the tunnel-only smoketest can run without waiting for main to see the workflow_dispatch trigger. Remove before merging. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 4392d0d..64cef48 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,6 +12,14 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: + # TEMPORARY, for iterating on this PR before workflow_dispatch is + # reachable (it needs the file on the default branch first). Scoped to + # this file only so it can't fire on unrelated PRs. Remove this + # `pull_request` block before merging. + pull_request: + types: [synchronize] + paths: + - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: PREVIEW_DOMAIN: dspack.iakte.ch From 52d9f86e8cd1c95f739c48e5a0f617e2878761e4 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 14:52:25 +0100 Subject: [PATCH 03/13] fix: move cloudflared --no-autoupdate flag before the run subcommand --no-autoupdate is a tunnel-level flag, not a run-subcommand flag; placed after `run` it errored with "flag provided but not defined" and the tunnel process exited immediately without ever connecting. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 2 +- .github/workflows/k8s-preview.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 64cef48..e639af0 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -122,7 +122,7 @@ jobs: echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" - name: Run tunnel until the job times out - run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - name: Delete DNS record if: always() diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index cbf510f..5abe013 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -252,7 +252,7 @@ jobs: label is removed. Push a new commit or re-add the label to redeploy. - name: Run tunnel until the job times out - run: /tmp/cloudflared tunnel run --token "${TUNNEL_TOKEN}" --no-autoupdate + run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" - name: Delete DNS record if: always() From 2818f4844b18388fca56546452a5b347b161d4f4 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:00:12 +0100 Subject: [PATCH 04/13] ci: use single-level github.fyi hostnames for the preview tunnel Two-level hostnames (pr-.dspack.iakte.ch) aren't covered by Cloudflare's default Universal SSL, which only auto-issues a cert for the zone apex plus one wildcard level. Switching to a dedicated single-level domain (pr-.github.fyi) avoids needing the paid Advanced Certificate Manager add-on. --- .../k8s-preview-tunnel-smoketest.yaml | 6 ++-- .github/workflows/k8s-preview.yaml | 34 +++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index e639af0..cc7b601 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -22,7 +22,7 @@ on: - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: - PREVIEW_DOMAIN: dspack.iakte.ch + PREVIEW_DOMAIN: github.fyi CLOUDFLARED_VERSION: "2026.7.3" CLOUDFLARED_SHA256: "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17" @@ -95,10 +95,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: smoketest-${{ github.run_id }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for iakte.ch" + echo "::error::Could not resolve zone id for github.fyi" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 5abe013..4d48889 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -9,14 +9,15 @@ name: K8s Stack Preview # (a quick-tunnel URL, a basic-auth password) is readable by anyone who # opens the PR, not just intended reviewers. Access closes that gap by # authenticating the *person*, not a string in the comment: Cloudflare -# challenges every request to *.dspack.iakte.ch with a GitHub SSO -# login and only lets it through to cloudflared if the signed-in account -# is a member of this GitHub org. The PR comment only ever contains a URL. +# challenges every request to *.github.fyi with a GitHub SSO login and +# only lets it through to cloudflared if the signed-in account is a +# member of this GitHub org. The PR comment only ever contains a URL. # -# iakte.ch is a placeholder domain used temporarily because it lives in -# the same Cloudflare account as the Tunnel and Zero Trust setup; swap -# PREVIEW_DOMAIN below once a permanent domain is available in that -# account. +# Hostnames are single-level (pr-.github.fyi, not pr-.dspack.github.fyi) +# deliberately: Cloudflare's free Universal SSL only auto-covers the zone +# apex plus one wildcard level (github.fyi + *.github.fyi); a second level +# needs the paid Advanced Certificate Manager add-on, which this setup +# doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` # service (dummy-authenticator login, same as local Tilt dev), not the @@ -33,12 +34,15 @@ name: K8s Stack Preview # # One-time setup this workflow assumes already exists (Cloudflare Zero # Trust dashboard, done by a repo admin, not scripted here). The tunnel, -# the `iakte.ch` zone, and Zero Trust/Access all live in ONE Cloudflare +# the `github.fyi` zone, and Zero Trust/Access all live in ONE Cloudflare # account (aktechlabs) — not the account behind CLOUDFLARE_API_TOKEN / # CLOUDFLARE_ACCOUNT_ID, which docs.yml uses for Pages: -# - Zone `iakte.ch` in that account, with an Access self-hosted -# application for `*.dspack.iakte.ch`, GitHub as identity provider, -# policy scoped to this org. +# - Zone `github.fyi` in that account, with an Access self-hosted +# application for `*.github.fyi`, GitHub as identity provider, +# policy scoped to this org. Note this wildcard covers ANY +# single-label subdomain of github.fyi, not just previews — fine as +# long as github.fyi isn't also hosting unrelated services outside +# this org's control. # - Secret CLOUDFLARE_TUNNEL_ACCOUNT_ID: that account's id (the # `cfd_tunnel` API is account-scoped; can't be derived from the # token alone). @@ -47,7 +51,7 @@ name: K8s Stack Preview # * Account -> Cloudflare Tunnel -> Edit # * Zone -> Zone -> Read (to resolve the zone id by name) # * Zone -> DNS -> Edit (to create/delete the CNAME record) -# Zone Resources: Include -> Specific zone -> iakte.ch. +# Zone Resources: Include -> Specific zone -> github.fyi. # Account Resources: Include -> Specific account -> aktechlabs. # # Only runs when a maintainer/collaborator adds the `deploy-preview` label @@ -74,7 +78,7 @@ concurrency: env: PREVIEW_LABEL: deploy-preview - PREVIEW_DOMAIN: dspack.iakte.ch + PREVIEW_DOMAIN: github.fyi CLOUDFLARED_VERSION: "2026.7.3" # sha256 of cloudflared-linux-amd64 for the pinned version above, # computed from the official release asset at @@ -206,10 +210,10 @@ jobs: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_API_TOKEN }} PREVIEW_HOSTNAME: pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }} run: | - zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=iakte.ch" \ + zone_id=$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=github.fyi" \ -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id') if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then - echo "::error::Could not resolve zone id for iakte.ch" + echo "::error::Could not resolve zone id for github.fyi" exit 1 fi echo "ZONE_ID=${zone_id}" >> "$GITHUB_ENV" From fedb68f84d84002722c74b6f32cb685e5cd0a7a0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:12:40 +0100 Subject: [PATCH 05/13] ci: drop temporary pull_request trigger from smoketest workflow workflow_dispatch is now registered and reachable; the iteration workaround is no longer needed. --- .github/workflows/k8s-preview-tunnel-smoketest.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index cc7b601..61786f0 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,14 +12,6 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: - # TEMPORARY, for iterating on this PR before workflow_dispatch is - # reachable (it needs the file on the default branch first). Scoped to - # this file only so it can't fire on unrelated PRs. Remove this - # `pull_request` block before merging. - pull_request: - types: [synchronize] - paths: - - .github/workflows/k8s-preview-tunnel-smoketest.yaml env: PREVIEW_DOMAIN: github.fyi From b22f710106a2bb39ff78744806696ddbb8ea5930 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:18:56 +0100 Subject: [PATCH 06/13] fix: fetch chart dependencies before helm install in preview workflow charts/ is gitignored, so the vendored jupyterhub-4.3.2.tgz seen in local dev never reaches a fresh CI checkout. helm dependency build resolves it against the already-committed Chart.lock digest. --- .github/workflows/k8s-preview.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 4d48889..3dde5ea 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -116,6 +116,13 @@ jobs: - name: Side-load hub image into the sandbox cluster run: k3d image import nebari-data-science-pack-jupyterhub:preview -c ${{ steps.sandbox.outputs.cluster-name }} + # charts/ is gitignored (dependency .tgz files aren't committed), so a + # fresh checkout needs this before `helm upgrade --install` can find + # the jupyterhub subchart. Resolves against the version/digest already + # pinned in the committed Chart.lock, not a new or bumped dependency. + - name: Fetch chart dependencies + run: helm dependency build . + - name: Deploy chart id: deploy env: From 1c7618ad43de5534e0c74eb522eda3e5a467abb0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:21:03 +0100 Subject: [PATCH 07/13] ci: post smoketest URL as a PR comment via optional pr_number input workflow_dispatch has no PR context to auto-detect, so the comment step uses number_force (sticky-pull-request-comment's any-event PR number override, distinct from number which is push-event only). --- .../k8s-preview-tunnel-smoketest.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index 61786f0..ac08747 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -12,6 +12,10 @@ name: K8s Preview Tunnel Smoketest on: workflow_dispatch: + inputs: + pr_number: + description: 'PR number to post the smoketest URL to (optional; skips the comment if blank)' + required: false env: PREVIEW_DOMAIN: github.fyi @@ -25,6 +29,7 @@ jobs: timeout-minutes: 15 permissions: contents: read + pull-requests: write steps: - name: Serve a trivial static page run: | @@ -112,6 +117,21 @@ jobs: echo "Visiting it should challenge you with Cloudflare Access GitHub SSO," >> "$GITHUB_STEP_SUMMARY" echo "then show 'Tunnel smoketest OK' once you're through." >> "$GITHUB_STEP_SUMMARY" echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" + echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + + - name: Comment smoketest link on PR + if: github.event.inputs.pr_number != '' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + header: k8s-preview-smoketest + number_force: ${{ github.event.inputs.pr_number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + **Tunnel smoketest** (static page, not the real stack): + ${{ steps.cf_dns.outputs.url }} + + Sign-in via Cloudflare Access (GitHub SSO), then should show "Tunnel smoketest OK". + Live for up to 15 minutes from this run. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" From 0a75683f6c906071d52b2b806d865e0cd5e69336 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:28:51 +0100 Subject: [PATCH 08/13] fix: register the jupyterhub chart repo before dependency build helm dependency build resolves against locally-registered repos, not just the Chart.lock digest; a fresh runner has none configured. Added helm repo add before the build step. Verified against a clean clone. --- .github/workflows/k8s-preview.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 3dde5ea..23c8428 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -121,7 +121,9 @@ jobs: # the jupyterhub subchart. Resolves against the version/digest already # pinned in the committed Chart.lock, not a new or bumped dependency. - name: Fetch chart dependencies - run: helm dependency build . + run: | + helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ + helm dependency build . - name: Deploy chart id: deploy From 1eec127f9e1f77806be0b3e1592c9cb1700d7cc6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 15:43:30 +0100 Subject: [PATCH 09/13] fix: opt preview namespace into nic-operator management, add failure debugging nic-operator ignores any namespace without nebari.dev/managed=true, so the NebariApp CRD never reconciled. Also adds a static pod/job/event dump and an interactive tmate SSH session (actor-restricted, 20min cap) on deploy failure, since the cluster is deleted right after and static logs alone weren't enough to diagnose the last two failures. --- .github/workflows/k8s-preview.yaml | 43 +++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 23c8428..4d9e6bd 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -125,13 +125,23 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . + # nic-operator's CoreReconciler ignores any namespace without this + # label (NamespaceNotOptedIn) — without it the NebariApp never + # reconciles regardless of what the chart deploys. + - name: Create and opt-in the preview namespace + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl create namespace pr-preview + kubectl label namespace pr-preview nebari.dev/managed=true + - name: Deploy chart id: deploy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | helm upgrade --install preview . \ - --namespace pr-preview --create-namespace \ + --namespace pr-preview \ --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=true \ @@ -154,6 +164,37 @@ jobs: kubectl -n pr-preview get nebariapp -o wide || true kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true + # Cleanup deletes the whole cluster next, so this is the only chance + # to see why a pod/job didn't reach Ready if `helm --wait` timed out. + - name: Debug pod/job status on deploy failure + if: failure() + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + kubectl -n pr-preview get pods -o wide || true + kubectl -n pr-preview get jobs || true + kubectl -n pr-preview get events --sort-by=.lastTimestamp || true + for pod in $(kubectl -n pr-preview get pods -o name 2>/dev/null); do + echo "--- describe $pod ---" + kubectl -n pr-preview describe "$pod" || true + echo "--- logs $pod ---" + kubectl -n pr-preview logs "$pod" --all-containers --tail=100 || true + done + + # Interactive SSH debug session into the live runner (cluster still + # up, KUBECONFIG still valid) instead of guessing blind from static + # logs. limit-access-to-actor restricts the SSH session to whoever + # triggered this run — required on a public repo. Bounded to 20min + # so a forgotten session doesn't eat the whole 90min job timeout. + - name: Debug via tmate SSH on deploy failure + if: failure() + uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24 + timeout-minutes: 20 + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + with: + limit-access-to-actor: true + - name: Port-forward JupyterHub proxy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} From 17f723b74dd9240479290a59cc7f4b9c8e6da493 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 16:03:16 +0100 Subject: [PATCH 10/13] fix: disable nebariapp for preview deploy, hub crash-loops otherwise 00-gateway-auth.py reads /etc/oauth/issuer-url unconditionally at import time. That file only exists once the operator's async Keycloak client provisioning finishes, which Helm doesn't wait for, so the hub pod crash-loops immediately when nebariapp.auth.enabled=true. Confirmed via kubectl logs during a failed run. Dropping nebariapp entirely for this preview; operator/OIDC reconcile isn't exercised here. --- .github/workflows/k8s-preview.yaml | 45 ++++++++---------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index 4d9e6bd..e780243 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -20,12 +20,13 @@ name: K8s Stack Preview # doesn't use. # # Scope, deliberately: the tunnel points straight at the `proxy-public` -# service (dummy-authenticator login, same as local Tilt dev), not the -# operator-provisioned NebariApp/OIDC route — that route exists for its -# own Keycloak realm and redirect URIs, unrelated to Access's GitHub SSO. -# The NebariApp CRD is still installed (auth.enabled=true) purely so the -# operator/Keycloak reconcile is exercised and its Ready condition -# reported in the run; it is not what the reviewer clicks through. +# service (dummy-authenticator login, same as local Tilt dev). The chart +# deploys with nebariapp.enabled=false — nebariapp.auth.enabled=true +# was tried to additionally exercise the operator/Keycloak reconcile, +# but 00-gateway-auth.py reads /etc/oauth/issuer-url unconditionally at +# import time, before the operator's async client provisioning can ever +# populate it, so the hub pod crash-loops. Not worth chasing for a +# preview link; the operator/OIDC path stays untested here. # # The link only lives for the run's duration (bounded by timeout-minutes # below) — it is not a persistent per-PR environment. Each run creates its @@ -125,28 +126,16 @@ jobs: helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ helm dependency build . - # nic-operator's CoreReconciler ignores any namespace without this - # label (NamespaceNotOptedIn) — without it the NebariApp never - # reconciles regardless of what the chart deploys. - - name: Create and opt-in the preview namespace - env: - KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl create namespace pr-preview - kubectl label namespace pr-preview nebari.dev/managed=true - - name: Deploy chart id: deploy env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} run: | helm upgrade --install preview . \ - --namespace pr-preview \ + --namespace pr-preview --create-namespace \ --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ - --set nebariapp.enabled=true \ - --set nebariapp.hostname="pr-${{ github.event.pull_request.number }}.${{ steps.sandbox.outputs.domain }}" \ - --set nebariapp.auth.enabled=true \ + --set nebariapp.enabled=false \ --wait --timeout 5m - name: Wait for hub + proxy @@ -156,14 +145,6 @@ jobs: kubectl -n pr-preview rollout status deployment/hub --timeout=180s kubectl -n pr-preview rollout status deployment/proxy --timeout=180s - - name: Report NebariApp reconcile status - if: always() - env: - KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} - run: | - kubectl -n pr-preview get nebariapp -o wide || true - kubectl -n pr-preview describe nebariapp preview-nebari-data-science-pack || true - # Cleanup deletes the whole cluster next, so this is the only chance # to see why a pod/job didn't reach Ready if `helm --wait` timed out. - name: Debug pod/job status on deploy failure @@ -296,11 +277,9 @@ jobs: reaching JupyterHub — only members of this GitHub org get through. **Then JupyterHub login:** dummy authenticator, any username + any password. - This goes straight to JupyterHub's proxy and skips the operator-provisioned - OIDC/Keycloak route (unrelated realm/redirect URIs to Access's GitHub SSO). - The `NebariApp` resource is still deployed alongside it so the operator/ - Keycloak reconcile itself is exercised — see the "Report NebariApp reconcile - status" step in this run for its `Ready` condition. + This goes straight to JupyterHub's proxy; the operator-provisioned + NebariApp/OIDC route isn't deployed here (see workflow header comment + for why) — this preview doesn't exercise operator/Keycloak reconcile. Live only for this job's run (up to 90 min), or until the `deploy-preview` label is removed. Push a new commit or re-add the label to redeploy. From a43dfb707f0741d07e9931683a53eb5842dcb0a3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 3 Aug 2026 16:49:59 +0100 Subject: [PATCH 11/13] fix: set jupyterhub.custom.external-url to the tunnel hostname Without it, 02-jhub-apps.py falls back to bind_url=http://0.0.0.0:8000, which JupyterHub then bakes into browser-facing OAuth redirect URLs for the jhub-apps service (client_id=service-japps), breaking the login flow after dummy-auth. Confirmed via a real login attempt on the deployed preview. --- .github/workflows/k8s-preview.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index e780243..ab28466 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -136,6 +136,7 @@ jobs: --set jupyterhub.hub.image.name=nebari-data-science-pack-jupyterhub \ --set jupyterhub.hub.image.tag=preview \ --set nebariapp.enabled=false \ + --set jupyterhub.custom.external-url="pr-${{ github.event.pull_request.number }}.${{ env.PREVIEW_DOMAIN }}" \ --wait --timeout 5m - name: Wait for hub + proxy From a14661698c645e168b2b027e0380e7527006a490 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 5 Aug 2026 13:32:44 +0100 Subject: [PATCH 12/13] ci: show deploy/expiry timestamps in preview PR comments The preview URL is stable per-PR, so redeploys posted byte-identical comment text and looked like they never updated. Adding deployed-at and expires-at timestamps makes every redeploy visibly change the comment and tells reviewers exactly when the link goes dead. --- .../workflows/k8s-preview-tunnel-smoketest.yaml | 9 ++++++++- .github/workflows/k8s-preview.yaml | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index ac08747..c2e0c04 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -119,6 +119,12 @@ jobs: echo "Live for up to 15 minutes (this job's timeout)." >> "$GITHUB_STEP_SUMMARY" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + - name: Compute deployment timestamps + id: timestamps + run: | + echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "expires_at=$(date -u -d '+15 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + - name: Comment smoketest link on PR if: github.event.inputs.pr_number != '' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 @@ -130,8 +136,9 @@ jobs: **Tunnel smoketest** (static page, not the real stack): ${{ steps.cf_dns.outputs.url }} + Deployed: ${{ steps.timestamps.outputs.deployed_at }} · Expires: ${{ steps.timestamps.outputs.expires_at }} + Sign-in via Cloudflare Access (GitHub SSO), then should show "Tunnel smoketest OK". - Live for up to 15 minutes from this run. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index ab28466..d99e51a 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -263,6 +263,15 @@ jobs: echo "DNS_RECORD_ID=${record_id}" >> "$GITHUB_ENV" echo "url=https://${PREVIEW_HOSTNAME}" >> "$GITHUB_OUTPUT" + # The URL itself (pr-.github.fyi) is identical on every run, so + # without a timestamp the sticky comment would post byte-identical + # text each redeploy and look like it never updated. + - name: Compute deployment timestamps + id: timestamps + run: | + echo "deployed_at=$(date -u +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + echo "expires_at=$(date -u -d '+90 minutes' +'%Y-%m-%d %H:%M UTC')" >> "$GITHUB_OUTPUT" + - name: Comment preview link on PR uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: @@ -272,6 +281,8 @@ jobs: **K8s stack preview** for `${{ github.event.pull_request.head.ref }}`: ${{ steps.cf_dns.outputs.url }} + Deployed: ${{ steps.timestamps.outputs.deployed_at }} · Expires: ${{ steps.timestamps.outputs.expires_at }} + ${{ github.event.pull_request.head.repo.fork && '⚠️ **This PR is from a fork** — the code running in this preview is not from a trusted maintainer branch.' || '' }} You'll be asked to sign in via Cloudflare Access (GitHub SSO) before @@ -282,8 +293,8 @@ jobs: NebariApp/OIDC route isn't deployed here (see workflow header comment for why) — this preview doesn't exercise operator/Keycloak reconcile. - Live only for this job's run (up to 90 min), or until the `deploy-preview` - label is removed. Push a new commit or re-add the label to redeploy. + Live until the expiry time above, or until the `deploy-preview` label + is removed. Push a new commit or re-add the label to redeploy. - name: Run tunnel until the job times out run: /tmp/cloudflared tunnel --no-autoupdate run --token "${TUNNEL_TOKEN}" From 65651b453af49420ff3922c712ee8b620449f2d8 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 5 Aug 2026 15:14:41 +0100 Subject: [PATCH 13/13] fix: make tunnel creation idempotent, retries reuse the run_id A workflow retry keeps the same github.run_id (only run_attempt changes), so a re-run after an attempt already created the tunnel but didn't reach its cleanup step hit a 409 name conflict. Now falls back to looking up and reusing the existing tunnel by name instead of failing outright. Confirmed via a real run_attempt=2 failure. --- .../k8s-preview-tunnel-smoketest.yaml | 19 +++++++++++---- .github/workflows/k8s-preview.yaml | 23 +++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/k8s-preview-tunnel-smoketest.yaml b/.github/workflows/k8s-preview-tunnel-smoketest.yaml index c2e0c04..be92bb7 100644 --- a/.github/workflows/k8s-preview-tunnel-smoketest.yaml +++ b/.github/workflows/k8s-preview-tunnel-smoketest.yaml @@ -58,15 +58,24 @@ jobs: tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" - create_resp=$(curl -fsS -X POST \ + tunnel_name="smoketest-${{ github.run_id }}" + create_resp=$(curl -sS -X POST \ "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ -H "Authorization: Bearer ${CF_API_TOKEN}" \ -H "Content-Type: application/json" \ - -d "$(jq -n --arg name "smoketest-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + -d "$(jq -n --arg name "$tunnel_name" --arg secret "$tunnel_secret" \ '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") - tunnel_id=$(jq -r '.result.id' <<< "$create_resp") - if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then - echo "::error::Tunnel creation failed: $create_resp" + tunnel_id=$(jq -r '.result.id // empty' <<< "$create_resp") + + # Same retry-safety as the real workflow: a retry reuses run_id, + # so reuse the existing tunnel by name on a 409 name conflict. + if [ -z "$tunnel_id" ]; then + echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" + tunnel_id=$(curl -fsS "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel?name=${tunnel_name}&is_deleted=false" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id // empty') + fi + if [ -z "$tunnel_id" ]; then + echo "::error::Tunnel creation failed and no existing tunnel named ${tunnel_name} found: $create_resp" exit 1 fi echo "TUNNEL_ID=${tunnel_id}" >> "$GITHUB_ENV" diff --git a/.github/workflows/k8s-preview.yaml b/.github/workflows/k8s-preview.yaml index d99e51a..3347767 100644 --- a/.github/workflows/k8s-preview.yaml +++ b/.github/workflows/k8s-preview.yaml @@ -207,15 +207,28 @@ jobs: tunnel_secret=$(openssl rand -base64 32) echo "::add-mask::${tunnel_secret}" - create_resp=$(curl -fsS -X POST \ + tunnel_name="pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" + create_resp=$(curl -sS -X POST \ "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel" \ -H "Authorization: Bearer ${CF_API_TOKEN}" \ -H "Content-Type: application/json" \ - -d "$(jq -n --arg name "pr-${{ github.event.pull_request.number }}-${{ github.run_id }}" --arg secret "$tunnel_secret" \ + -d "$(jq -n --arg name "$tunnel_name" --arg secret "$tunnel_secret" \ '{name: $name, config_src: "cloudflare", tunnel_secret: $secret}')") - tunnel_id=$(jq -r '.result.id' <<< "$create_resp") - if [ -z "$tunnel_id" ] || [ "$tunnel_id" = "null" ]; then - echo "::error::Tunnel creation failed: $create_resp" + tunnel_id=$(jq -r '.result.id // empty' <<< "$create_resp") + + # A GitHub Actions retry reuses the same run_id (only run_attempt + # changes), so a re-run after the first attempt already created + # this tunnel (and didn't get to clean it up) hits a 409 name + # conflict here. Reuse the existing tunnel by name instead of + # failing — it doesn't need the original tunnel_secret, just a + # fresh --token from the /token endpoint below. + if [ -z "$tunnel_id" ]; then + echo "::warning::Tunnel create failed (likely a name conflict from a retry), looking up existing tunnel named ${tunnel_name}: $create_resp" + tunnel_id=$(curl -fsS "https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT_ID}/cfd_tunnel?name=${tunnel_name}&is_deleted=false" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" | jq -r '.result[0].id // empty') + fi + if [ -z "$tunnel_id" ]; then + echo "::error::Tunnel creation failed and no existing tunnel named ${tunnel_name} found: $create_resp" exit 1 fi echo "tunnel_id=${tunnel_id}" >> "$GITHUB_OUTPUT"