From 19ab2e0aff8b7f8259905e137dc62d56b7c7b4b4 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:45:47 -0700 Subject: [PATCH 1/2] ci: fix e2e-sandbox by using the nebari-apps ArgoCD project, migrate to sandbox v3 The e2e-sandbox job has been red on main for several commits, in two distinct phases. The originally-reported failure was Keycloak never being installed by the sandbox platform stack. That was NIC v0.13.0 removing the existing-cluster + file:// GitOps combination the action's k3d path depended on, reached via the action's `nic-version: latest` default. action-nebari-sandbox v2.3.0 fixed it upstream by pinning NIC to v0.12.0, so the platform now comes up. What replaced it is ours. NIC locked the `default` ArgoCD AppProject down to a deny-all (empty sourceRepos and destinations) and added `nebari-apps` as the home for consumer applications. Our Application still declared `project: default`, so ArgoCD rejected it with two InvalidSpecErrors - the destination namespace and the file:// repo are both outside the project - and it never synced. The wait loop saw sync=Unknown health=Unknown for all 90 polls. Switching to `nebari-apps` matches the action's documented consumer pattern and its own test scenario. Also migrate off the k3d path, which v2 pins to NIC v0.12.0 permanently: - Bump to @v3, which deploys via NIC's local (kind) provider. v3 drops the `profile` input, as platform is now the only profile. - Pin `nic-version: v0.13.0` rather than tracking `latest`, so a NIC release cannot turn this job red without a deliberate bump - the same reasoning already applied to golangci-lint and kubeconform here. - Load the image with `kind load docker-image` instead of `k3d image import`. kind 0.32.0 is preinstalled on ubuntu-latest. - Drop the explicit teardown step. v3 destroys the deployment in its own post step, including on failure and cancellation. - Correct the dev-mode auth comment: v3 resolves *.nebari.local from inside the cluster, so the untrusted self-signed gateway CA is the only remaining reason real auth cannot be exercised here. --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42a76a3..dfd4dda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,13 +116,19 @@ jobs: - name: Build frames image run: docker build --platform=linux/amd64 -t nebari-frames:e2e . - - uses: nebari-dev/action-nebari-sandbox@v2 + - uses: nebari-dev/action-nebari-sandbox@v3 id: sandbox with: - profile: platform + # Pinned rather than tracking `latest`, so a NIC release cannot turn + # this job red without a deliberate bump. v3 requires NIC >= v0.13.0 + # (the local/kind cluster + local repository providers). + nic-version: v0.13.0 - name: Import image into the sandbox cluster - run: k3d image import nebari-frames:e2e -c ${{ steps.sandbox.outputs.cluster-name }} + # v3 deploys via NIC's local provider, which is kind, not k3d. `kind` is + # preinstalled on ubuntu-latest; the kind cluster name is the action's + # cluster-name output (its kubectl context is `kind-`). + run: kind load docker-image nebari-frames:e2e --name ${{ steps.sandbox.outputs.cluster-name }} - name: Deploy frames via GitOps env: @@ -136,7 +142,11 @@ jobs: name: nebari-frames namespace: argocd spec: - project: default + # Consumer apps must use the `nebari-apps` project. NIC locks the + # `default` project down to an empty sourceRepos/destinations + # deny-all, so an Application in it is rejected with + # InvalidSpecError and never syncs. + project: nebari-apps source: repoURL: "file://${GITOPS_DIR}" targetRevision: HEAD @@ -147,12 +157,16 @@ jobs: image: repository: nebari-frames tag: e2e - # Auth runs in dev mode: the app's OIDC readiness check needs - # to reach the issuer (https://keycloak.nebari.local) from - # inside the pod, but the sandbox has no in-cluster DNS for - # *.nebari.local and its CA is self-signed, so /readyz stays - # 503 forever with real auth. Routing, TLS, and the gateway - # path are still fully exercised via the NebariApp. + # Auth runs in dev mode: the app's OIDC readiness check + # needs to reach the issuer (https://keycloak.nebari.local) + # from inside the pod. The sandbox now resolves + # *.nebari.local in-cluster, but its gateway CA is + # self-signed and the pod does not trust it, so /readyz + # stays 503 forever with real auth. Enabling real auth means + # mounting the sandbox CA into the pod (the action exposes + # it as ConfigMap `nebari-sandbox-ca` in `kube-public`). + # Routing, TLS, and the gateway path are still fully + # exercised via the NebariApp. nebariapp: enabled: true hostname: frames.nebari.local @@ -257,7 +271,3 @@ jobs: 200|302|303|307) echo "endpoint OK" ;; *) echo "::error::unexpected HTTP ${CODE}"; exit 1 ;; esac - - - name: Cleanup - if: always() - run: k3d cluster delete ${{ steps.sandbox.outputs.cluster-name }} || true From 73d87da4e80f0f24b04e0ad2139cbb81a70e6484 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:01:09 -0700 Subject: [PATCH 2/2] ci: pin MetalLB's address pool to the kind Docker network With the ArgoCD project fixed, the e2e job got all the way to the final gateway probe and failed there: curl to the gateway LoadBalancer IP timed out with no response at all. The gateway IP was 192.168.1.100, which is NIC's fallback MetalLB pool (192.168.1.100-192.168.1.110), not a derived one. kind puts its nodes on 172.18.0.0/16, so that address has no route from the runner and the connection never establishes. On the old k3d path the gateway IP was the node IP itself (172.18.0.2), directly routable, which is why this probe used to work. NIC does derive a pool from the kind Docker network, but caches it on the provider instance during Deploy, and InfraSettings only picks it up if Deploy populated that same instance first. Something in that chain does not hold, and the IPAddressPool silently gets the fallback - no warning is emitted, which rules out a derivation error and points at the derived value simply never being consulted. An explicitly configured address_pool takes precedence over both the cache and the fallback, so pin it. The pool is derived from the network kind will actually use rather than hardcoded, mirroring NIC's own formula (the .100-.110 range of the subnet's last /24 block), so it self-corrects if Docker picks a different subnet. The `kind` bridge network is created up front to fix that subnet, which kind then reuses. Supplying nic-config means the action skips its own config template, so this restates the platform defaults and pins cluster-name to match project_name, as the action requires for its outputs to resolve. --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfd4dda..6135eae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,59 @@ jobs: - name: Build frames image run: docker build --platform=linux/amd64 -t nebari-frames:e2e . + - name: Pin MetalLB's pool to the kind Docker network + id: metallb + run: | + # NIC is supposed to derive MetalLB's IPAddressPool from the kind + # Docker network, but the derived value is cached on the provider + # instance during Deploy and is not what the IPAddressPool ends up + # with: the pool gets NIC's unroutable 192.168.1.100-110 fallback + # instead, so the gateway LoadBalancer IP has no route from the + # runner and every request to it times out. An explicitly configured + # pool takes precedence over both, so pin it ourselves. + # + # Derive it from the network kind will actually use rather than + # hardcoding a range, so this self-corrects if Docker picks a + # different subnet. kind reuses a bridge network named `kind` when one + # already exists, so creating it up front fixes the subnet without + # fighting kind for it. + docker network create --driver bridge --subnet 172.18.0.0/16 kind 2>/dev/null \ + || echo "network 'kind' already exists; using its subnet as-is" + SUBNET=$(docker network inspect kind --format '{{ (index .IPAM.Config 0).Subnet }}') + # Mirrors NIC's own formula: the .100-.110 range of the subnet's last + # /24 block (172.18.0.0/16 -> 172.18.255.100-172.18.255.110). + POOL=$(python3 -c ' + import ipaddress, sys + net = ipaddress.ip_network(sys.argv[1]) + if net.prefixlen > 24: + raise SystemExit(f"kind network {net} is smaller than a /24") + last = ipaddress.ip_network(f"{net.broadcast_address}/24", strict=False) + print(f"{last[100]}-{last[110]}") + ' "${SUBNET}") + echo "kind network ${SUBNET} -> MetalLB pool ${POOL}" + echo "pool=${POOL}" >> "$GITHUB_OUTPUT" + + - name: Write the NIC config + run: | + # Supplying a config means the action skips its own template, so this + # has to restate the platform defaults it would otherwise set. + # `project_name` MUST equal the sandbox `cluster-name` below, and the + # providers must both be `local`, or the action's outputs do not + # resolve. + cat > /tmp/nic-config.yaml <= v0.13.0 # (the local/kind cluster + local repository providers). nic-version: v0.13.0 + nic-config: /tmp/nic-config.yaml + # Must match `project_name` in the config above. + cluster-name: nebari-test - name: Import image into the sandbox cluster # v3 deploys via NIC's local provider, which is kind, not k3d. `kind` is