From 7f7c0b63abd860eaeaa20d303a6270a2635e4229 Mon Sep 17 00:00:00 2001 From: Johnny Bouder Date: Thu, 20 Aug 2026 12:23:23 -0400 Subject: [PATCH 1/4] docs: write the pack documentation Fills the Astro + Starlight scaffold with real content, following the structure apps-pack uses. Getting Started getting-started install, why the chart renders its own Services, first deployment deployment the two NebariApps, why ignoreDifferences is needed, and what the broad rayClusterConfig ignore costs jupyter ray:// vs HTTP, version matching, the NetworkPolicy, and what does not survive a cluster roll local-development what dev/ builds, and the four things it cannot exercise Guides serve-applications declarative vs notebook deployment, building the image, proxy_location, checking health scaling replicas, GPUs end to end (resource, runtime class, auto-injected toleration, and the ray_actor_options step people miss), and the probe overrides behind issue #7 ca-bundle the injection, the four env vars and why GIT_SSL_CAINFO is separate, the ConfigMap rationale, the Argo CD footgun (#17), and the httpx gap troubleshooting the failures this pack actually produces Reference configuration every value, with the asymmetries called out architecture the mermaid diagram from the README, plus why serve-svc targets only the head and what that implies for worker readiness The through-line is that several of this pack's sharpest edges are invisible from kubectl: an unset nebariapp.hostname silently skips the serve NebariApp, `probe: {}` does not suppress a probe because Helm's deep merge keeps existing keys, and Argo CD's rayClusterConfig ignore drops the CA injection while reporting fully synced. Each is documented where someone would hit it, with the command that confirms it. Adds an editLink baseUrl so every page carries an "Edit page" link. --- docs/astro.config.mjs | 29 ++- docs/src/content/docs/architecture.md | 151 ++++++++++++++ docs/src/content/docs/ca-bundle.md | 171 ++++++++++++++++ docs/src/content/docs/configuration.md | 180 +++++++++++++++++ docs/src/content/docs/deployment.md | 197 +++++++++++++++++++ docs/src/content/docs/getting-started.md | 115 +++++++++++ docs/src/content/docs/index.md | 84 ++++++-- docs/src/content/docs/jupyter.md | 146 ++++++++++++++ docs/src/content/docs/local-development.md | 117 +++++++++++ docs/src/content/docs/scaling.md | 205 ++++++++++++++++++++ docs/src/content/docs/serve-applications.md | 181 +++++++++++++++++ docs/src/content/docs/troubleshooting.md | 178 +++++++++++++++++ 12 files changed, 1736 insertions(+), 18 deletions(-) create mode 100644 docs/src/content/docs/architecture.md create mode 100644 docs/src/content/docs/ca-bundle.md create mode 100644 docs/src/content/docs/configuration.md create mode 100644 docs/src/content/docs/deployment.md create mode 100644 docs/src/content/docs/getting-started.md create mode 100644 docs/src/content/docs/jupyter.md create mode 100644 docs/src/content/docs/local-development.md create mode 100644 docs/src/content/docs/scaling.md create mode 100644 docs/src/content/docs/serve-applications.md create mode 100644 docs/src/content/docs/troubleshooting.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index cbb600e..7c4cdea 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -26,10 +26,37 @@ export default defineConfig({ // GitHub social link) comes from the @nebari/starlight theme plugin. On the // portal the header logo returns users to the pack catalog. plugins: [nebari({ logoHref: 'https://packs.nebari.dev/' })], + editLink: { + // Starlight appends the source path (src/content/docs/.md) to this + // base, so it must point at the Astro project root inside the repo. + baseUrl: 'https://github.com/nebari-dev/rayserve-pack/edit/main/docs/', + }, sidebar: [ { label: 'Getting Started', - items: [{ label: 'Introduction', link: '/' }], + items: [ + { label: 'Introduction', link: '/' }, + { label: 'Getting started', link: '/getting-started/' }, + { label: 'Deploying on Nebari', link: '/deployment/' }, + { label: 'Connecting from Jupyter', link: '/jupyter/' }, + { label: 'Local development', link: '/local-development/' }, + ], + }, + { + label: 'Guides', + items: [ + { label: 'Deploying models', link: '/serve-applications/' }, + { label: 'Scaling and GPUs', link: '/scaling/' }, + { label: 'Organization CA bundle', link: '/ca-bundle/' }, + { label: 'Troubleshooting', link: '/troubleshooting/' }, + ], + }, + { + label: 'Reference', + items: [ + { label: 'Configuration', link: '/configuration/' }, + { label: 'Architecture', link: '/architecture/' }, + ], }, ], }), diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md new file mode 100644 index 0000000..64a47f8 --- /dev/null +++ b/docs/src/content/docs/architecture.md @@ -0,0 +1,151 @@ +--- +title: Architecture +description: How the KubeRay operator, RayService, Services, and NebariApps fit together. +--- + +```mermaid +flowchart TD + subgraph KO["KubeRay Operator"] + op["Manages RayService lifecycle"] + end + + subgraph RS["RayService CRD"] + subgraph RC["RayCluster"] + head["Head Pod\n:8265 dashboard\n:8000 serve\n:10001 client"] + workers["Worker Pod(s)\nRay Workers"] + end + end + + subgraph SVC["Kubernetes Services"] + headsvc["-head-svc\n:8265 :10001 :6379"] + servesvc["-serve-svc\n:8000"] + end + + subgraph NB["NebariApp (optional)"] + route["HTTPRoute + OIDC auth\nvia Envoy Gateway"] + end + + jupyter["Jupyter Notebook\n(in-cluster)"] + browser["Browser\n(external)"] + + KO --> RS + head --- workers + head --> headsvc + head --> servesvc + servesvc --> route + + jupyter -->|"ray:// :10001"| headsvc + jupyter -->|"HTTP :8000"| servesvc + browser -->|"HTTPS"| route + + style KO fill:#fef0db,stroke:#e8952c,color:#7c4a03 + style RS fill:#eeeef3,stroke:#4a4a6a,color:#1a1a2e + style RC fill:#e8faf8,stroke:#20aaa1,color:#0d5d57 + style SVC fill:#d4f5f2,stroke:#20aaa1,color:#0d5d57 + style NB fill:#f3e8fc,stroke:#c840e9,color:#6b21a8 +``` + +## The chain + +**KubeRay operator** watches `RayService` resources. It creates the underlying +`RayCluster`, deploys the Serve applications named in `serveConfigV2`, monitors their +health, and performs zero-downtime upgrades when the config changes. + +**RayService** is the chart's central object. It carries two things: the Serve config, and +the cluster config. The chart writes both from values — `serve.proxyLocation` and +`serveApplications` into `serveConfigV2`, and the head/worker specs into +`rayClusterConfig`. + +**The head pod** runs the GCS (Ray's metadata store), the dashboard, the Serve controller, +and — with `proxyLocation: EveryNode` — an HTTP proxy. It exposes four ports: `6379` GCS, +`8265` dashboard, `10001` Ray client, `8000` Serve HTTP. + +**Worker pods** run Ray workers, and Serve replicas land on them. One group, +`groupName: workers`, sized by `worker.replicas` between `minReplicas` and `maxReplicas`. + +## Why the chart renders its own Services + +RayService creates its own stable Services — but only after every Serve application reports +healthy. With the default empty `serveApplications` that condition never holds, so the +dashboard and Serve endpoint would be unreachable on a fresh install. + +The chart therefore renders both itself, selecting the head pod directly: + +```yaml +selector: + ray.io/node-type: head + app.kubernetes.io/name: kuberay +``` + +They exist from the moment the chart installs, regardless of Serve state. Both carry +`argocd.argoproj.io/compare-options: IgnoreExtraneous` so Argo CD tolerates the Services +KubeRay creates alongside them. + +Note that `serve-svc` targets **only the head pod**, even under `proxyLocation: EveryNode`. +The per-node proxies serve direct-to-pod traffic; the Service does not load-balance across +them. This is also why worker readiness has no effect on user-visible HTTP routing — the +reason the chart can safely simplify the worker probes. + +## Serve config + +```yaml +serveConfigV2: | + proxy_location: {{ serve.proxyLocation }} + http_options: + host: "0.0.0.0" + port: 8000 + applications: [...] +``` + +`host: "0.0.0.0"` is set here so the proxy binds all interfaces from the start — without it +Serve binds loopback and nothing outside the pod can reach it. That is why there is no +manual `serve start` step anywhere in this pack. + +## Two access paths, by design + +| | In-cluster | External | +|---|---|---| +| Client | notebooks, other pods | browsers, API clients | +| Route | ClusterIP Service | `NebariApp` → HTTPRoute → Envoy | +| Auth | none | OIDC at the gateway, when enabled | +| Protocols | `ray://` and HTTP | HTTPS | + +The split is not an oversight. The Ray client protocol cannot traverse an OIDC redirect, so +notebooks must reach the head service directly. Access control on that path is +NetworkPolicy, not identity — anything permitted to reach `:10001` can submit arbitrary +code to the Ray cluster, which is worth scoping deliberately. + +## The two NebariApps + +Serve and dashboard get separate resources with separate hostnames, because they are +separate audiences with different exposure appetites. The recommended posture keeps the +serve endpoint internal (`serve.enabled: false`) and exposes only the dashboard. + +Both inherit the same `auth` and `gateway` settings — there is no per-endpoint override. + +## Conditional injections + +Two features render nothing at all when unused, so the output is byte-identical to a plain +install: + +- **[CA bundle](/ca-bundle/)** — an initContainer, volumes, mounts, and four environment + variables on both pod specs, only when `orgCABundle.configMapName` is set. +- **[GPU toleration](/scaling/#gpus)** — an `nvidia.com/gpu` toleration, only when that + group's `resources` mention the GPU resource, and only when you have not defined one + yourself. + +Both live under `spec.rayClusterConfig`, which is what makes the Argo CD +`ignoreDifferences` rule on that path consequential rather than cosmetic. See +[Deploying on Nebari](/deployment/#why-ignoredifferences-is-there). + +## What state lives where + +| State | Where | Survives a cluster roll | +|---|---|---| +| Declarative Serve applications | `serveConfigV2` in the RayService | yes | +| Applications deployed via `serve.run()` | the running Ray cluster | no | +| Model code | the container image | yes | +| Anything written to a pod filesystem | the pod | no | + +The chart provisions no persistent volumes. Anything that must survive belongs in the image +or in external storage. diff --git a/docs/src/content/docs/ca-bundle.md b/docs/src/content/docs/ca-bundle.md new file mode 100644 index 0000000..a7046e5 --- /dev/null +++ b/docs/src/content/docs/ca-bundle.md @@ -0,0 +1,171 @@ +--- +title: Organization CA bundle +description: Running Ray behind a TLS-inspecting proxy, and the Argo CD interaction that silently defeats it. +--- + +Behind a TLS-inspecting proxy — Netskope, Zscaler, BlueCoat, an internal corporate CA — every +outbound HTTPS call from a Ray pod fails certificate verification. Model registries, dataset +URLs, the Hugging Face hub, S3: all `CERTIFICATE_VERIFY_FAILED`, because the proxy re-signs +certificates with a root the container's trust store has never heard of. + +Point the chart at a ConfigMap holding your organization's root CA and it fixes this for the +head and worker pods. + +## Enabling + +Create the ConfigMap out of band — through your GitOps layer, `kubectl`, or trust-manager: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: org-ca-bundle + namespace: rayserve +data: + ca.crt: | + -----BEGIN CERTIFICATE----- + ...your org CA... + -----END CERTIFICATE----- +``` + +Then: + +```yaml +orgCABundle: + configMapName: org-ca-bundle +``` + +The key must be `ca.crt`, and the ConfigMap must be in the release namespace. + +When `configMapName` is empty — the default — nothing is rendered and the resulting +RayService is byte-identical to the no-injection case. + +## What it does + +For both head and worker pods, the chart adds: + +1. **An initContainer** (`build-ca-bundle`, `alpine:3.20` by default) that concatenates the + base image's system trust store with your CA: + ```sh + cat /etc/ssl/certs/ca-certificates.crt /var/local/org-ca/ca.crt > /shared/combined-ca.crt + ``` +2. **A shared `emptyDir`** the main container mounts read-only at `/shared`. The Ray + container never sees the raw ConfigMap. +3. **Four environment variables** on the main container, all pointing at + `/shared/combined-ca.crt`. + +Concatenating matters. Replacing the system trust store rather than extending it would +break TLS to everything the proxy does not re-sign. + +## The four environment variables + +| Variable | Honored by | +|---|---| +| `SSL_CERT_FILE` | OpenSSL, Python `ssl`, most tooling | +| `REQUESTS_CA_BUNDLE` | `requests`, `urllib3` | +| `CURL_CA_BUNDLE` | curl | +| `GIT_SSL_CAINFO` | git | + +`GIT_SSL_CAINFO` is separate because git's libcurl ignores the other three and reads only +that one. Without it, `pip install git+https://...` and any other git-over-HTTPS call in a +worker fails verification even though plain `requests` and `pip` succeed — a confusing +partial failure. + +## The Argo CD interaction + +:::caution[The CA bundle can silently not apply under Argo CD] +The example `Application` in [Deploying on Nebari](/deployment/) sets +`RespectIgnoreDifferences=true` together with an `ignoreDifferences` rule on +`/spec/rayClusterConfig`. With server-side apply, that combination tells Argo CD to stop +managing **every field** under `rayClusterConfig` — which is exactly where this chart injects +the initContainer, volumes, volume mounts, and CA environment variables. + +The result: Argo CD reports a healthy, fully-synced Application, and the running pods never +receive the CA bundle. TLS calls keep failing with `CERTIFICATE_VERIFY_FAILED` while every +dashboard says green. + +See [issue #17](https://github.com/nebari-dev/nebari-rayserve-pack/issues/17). +::: + +The broad ignore exists only to suppress the autoscaler and runtime mutations KubeRay makes +to `rayClusterConfig`. To use `orgCABundle` under Argo CD, narrow it: replace +`/spec/rayClusterConfig` with targeted JSON pointers at the specific subpaths KubeRay +rewrites, or drop the rule and add narrower ones as drift appears. + +Then verify against the running pod rather than the sync status: + +```bash +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) \ + -- printenv SSL_CERT_FILE +# /shared/combined-ca.crt +``` + +Empty output means the injection did not reach the pod, whatever Argo CD reports. + +## Verifying + +```bash +POD=$(kubectl -n rayserve get pod -l ray.io/node-type=worker -o name) + +kubectl -n rayserve exec $POD -- printenv SSL_CERT_FILE REQUESTS_CA_BUNDLE CURL_CA_BUNDLE GIT_SSL_CAINFO +kubectl -n rayserve exec $POD -- ls -l /shared/combined-ca.crt +kubectl -n rayserve exec $POD -- python -c "import requests; print(requests.get('https://huggingface.co').status_code)" +``` + +A `200` from the last command against a host that traverses the proxy is the real test. + +If the initContainer failed, the pod never starts and the reason is in its logs: + +```bash +kubectl -n rayserve logs $POD -c build-ca-bundle +``` + +The usual causes are a missing ConfigMap or a key that is not `ca.crt`. + +## Why a ConfigMap and not a Secret + +A CA certificate is public material by design — the entire PKI trust model depends on root +CAs being widely distributed. Mozilla's bundle ships in every browser and OS, and corporate +inspecting-proxy roots are pushed to every device that traverses them. + +Kubernetes itself distributes the cluster's own CA through a ConfigMap +(`kube-root-ca.crt`, auto-projected into every namespace), and cert-manager's +[trust-manager](https://cert-manager.io/docs/trust/trust-manager/) distributes CA bundles as +ConfigMaps via its `Bundle` CR. This chart follows that precedent. + +Reserve Secrets for things that actually need confidentiality — private keys, OAuth client +secrets. + +## Known gap: httpx + +:::caution[httpx ignores `SSL_CERT_FILE`] +httpx clients built with the default `verify=True` hardcode `cafile=certifi.where()`, so +they never see the injected bundle. Application code making httpx calls through the proxy +must construct the context explicitly: + +```python +import ssl, httpx + +client = httpx.Client(verify=ssl.create_default_context()) +# or per-call: +httpx.get(url, verify=ssl.create_default_context()) +``` + +`ssl.create_default_context()` with no `cafile=` honors `SSL_CERT_FILE` and `SSL_CERT_DIR` +per the standard OpenSSL convention, so it picks up the bundle. +::: + +`requests`, `urllib3`, stdlib `urllib`, curl, git, and most non-Python TLS tooling honor the +environment variables automatically. httpx is the notable exception, and it is increasingly +common in modern Python libraries — worth grepping your dependencies for. + +## The initContainer image + +```yaml +orgCABundle: + initImage: "alpine:3.20" +``` + +It needs only `sh` and `cat`. Override it if your organization requires images from a +vetted registry — and note that in an air-gapped environment, an unreachable +`alpine:3.20` blocks pod startup entirely rather than just skipping the injection. diff --git a/docs/src/content/docs/configuration.md b/docs/src/content/docs/configuration.md new file mode 100644 index 0000000..635b389 --- /dev/null +++ b/docs/src/content/docs/configuration.md @@ -0,0 +1,180 @@ +--- +title: Configuration +description: Values reference for the Nebari Rayserve Pack Helm chart. +--- + +Everything is owned by this chart except `kuberay-operator.*`, which passes through to the +[KubeRay operator chart](https://github.com/ray-project/kuberay-helm) (1.3.0). + +## Image + +| Value | Default | Purpose | +|---|---|---| +| `image.repository` | `rayproject/ray` | Used for both head and workers. | +| `image.tag` | `2.43.0` | Ray version. Also written to the RayService's `rayVersion`. | + +For production, build a custom image with your model code — see +[Deploying models](/serve-applications/). Notebook environments must match this version; +see [Connecting from Jupyter](/jupyter/#versions-must-match). + +## `head` + +| Value | Default | Purpose | +|---|---|---| +| `head.resources.requests` | `cpu: 1`, `memory: 2Gi` | — | +| `head.resources.limits` | `cpu: 2`, `memory: 4Gi` | — | +| `head.runtimeClassName` | unset | e.g. `nvidia`. | +| `head.containerEnv` | `[]` | Extra environment variables. | +| `head.tolerations` | `[]` | Extra tolerations; an `nvidia.com/gpu` one is injected automatically when the GPU resource is requested. | +| `head.readinessProbe` | `{}` | Empty means KubeRay's built-in probe applies. | +| `head.livenessProbe` | `{}` | Same. | + +## `worker` + +| Value | Default | Purpose | +|---|---|---| +| `worker.replicas` | `1` | Worker pods. | +| `worker.minReplicas` | `1` | Autoscaler lower bound; defaults to `replicas`. | +| `worker.maxReplicas` | `1` | Autoscaler upper bound; defaults to `replicas`. | +| `worker.resources.requests` | `cpu: 1`, `memory: 2Gi` | — | +| `worker.resources.limits` | `cpu: 2`, `memory: 4Gi` | — | +| `worker.runtimeClassName` | unset | e.g. `nvidia`. | +| `worker.containerEnv` | `[]` | Extra environment variables. | +| `worker.tolerations` | `[]` | As above. | +| `worker.readinessProbe` | raylet healthz | Overrides KubeRay's Serve-dependent default. | +| `worker.livenessProbe` | raylet healthz | Same command; longer thresholds. | + +:::caution[`{}` does not suppress a probe] +Helm's deep merge keeps existing keys when overlaying with an empty map. Use `null` (`~`) to +fall back to KubeRay's default. Full rationale in [Scaling and GPUs](/scaling/#probes). +::: + +## `serve` and `serveApplications` + +| Value | Default | Purpose | +|---|---|---| +| `serve.proxyLocation` | `EveryNode` | `EveryNode`, `HeadOnly`, or `Disabled`. | +| `serveApplications` | `[]` | Applications, serialized into `serveConfigV2`. | + +Each entry needs `name`, `route_prefix`, and an `import_path` resolvable inside the image. +Anything else valid in +[Ray Serve's config schema](https://docs.ray.io/en/latest/serve/production-guide/config.html) +is passed through. See [Deploying models](/serve-applications/). + +## `nebariapp` + +| Value | Default | Purpose | +|---|---|---| +| `nebariapp.enabled` | `false` | Render `NebariApp` resources. | +| `nebariapp.hostname` | unset | Serve endpoint hostname. Required for the serve `NebariApp`. | +| `nebariapp.serve.enabled` | `false` | Expose the serve endpoint externally. | +| `nebariapp.dashboard.enabled` | `true` | Create a `NebariApp` for the dashboard. | +| `nebariapp.dashboard.hostname` | unset | **Required** when the dashboard is enabled — the render fails without it. | +| `nebariapp.dashboard.landingPage.*` | disabled | Landing-page tile; dashboard only. | +| `nebariapp.service.name` | `""` | Overrides the backend service for **both** resources. | +| `nebariapp.service.servePort` | `8000` | — | +| `nebariapp.service.dashboardPort` | `8265` | — | +| `nebariapp.gateway` | `public` | `public` or `internal`; applies to both. | + +:::caution[The two hostnames behave differently when missing] +`dashboard.hostname` uses `required`, so its absence fails the Helm render with a clear +message. `nebariapp.hostname` does not — the serve `NebariApp` is simply not rendered, with +no error. See [Deploying on Nebari](/deployment/). +::: + +:::note[`service.name` overrides both] +Set it and both the serve and dashboard `NebariApp`s point at the same service. Leave it +empty and each uses its correct default (`-serve-svc` and `-head-svc` respectively) — which +is almost always what you want. +::: + +### `nebariapp.auth` + +| Value | Default | Purpose | +|---|---|---| +| `auth.enabled` | `false` | OIDC at the gateway, for both resources. | +| `auth.provider` | `keycloak` | — | +| `auth.provisionClient` | `true` | Operator creates the Keycloak client. | +| `auth.redirectURI` | `/oauth2/callback` | Envoy Gateway rejects a bare `/`. | +| `auth.scopes` | `openid, profile, email` | — | + +Auth is all-or-nothing across both endpoints. + +## `orgCABundle` + +| Value | Default | Purpose | +|---|---|---| +| `orgCABundle.configMapName` | `""` | ConfigMap with key `ca.crt`. Empty disables injection entirely. | +| `orgCABundle.initImage` | `alpine:3.20` | Needs only `sh` and `cat`. | + +Read [Organization CA bundle](/ca-bundle/) before enabling this under Argo CD — the example +sync policy can silently drop the injection. + +## Subchart and overrides + +| Value | Default | Purpose | +|---|---|---| +| `kuberay-operator.enabled` | `true` | Install the operator with the chart. Set false when one already runs cluster-wide. | +| `nameOverride` | `""` | Changes the chart name in the fullname helper — and therefore both Service names. | +| `fullnameOverride` | `""` | Same, more directly. | + +:::caution[Name overrides change the Service names] +`-head-svc` and `-serve-svc` are both derived from the fullname helper, and the `NebariApp` +defaults follow it. Notebook connection strings are *not* — anything hardcoding a service +name has to be updated too. +::: + +## A GPU production values file + +```yaml +image: + repository: your-registry/your-ray-image + tag: "2.43.0-custom" + +head: + resources: + requests: { cpu: "2", memory: "8Gi" } + limits: { cpu: "4", memory: "16Gi" } + +worker: + replicas: 2 + minReplicas: 2 + maxReplicas: 8 + runtimeClassName: nvidia + resources: + requests: { cpu: "4", memory: "16Gi" } + limits: { cpu: "8", memory: "32Gi", nvidia.com/gpu: 1 } + +serveApplications: + - name: my-model + route_prefix: /predict + import_path: myapp.model:app + deployments: + - name: MyModel + num_replicas: 2 + ray_actor_options: + num_gpus: 1 + +nebariapp: + enabled: true + serve: + enabled: false + dashboard: + enabled: true + hostname: ray-dashboard.example.com + landingPage: + enabled: true + auth: + enabled: true + gateway: internal +``` + +## Inspecting + +```bash +helm template rayserve chart --set nebariapp.enabled=true \ + --set nebariapp.dashboard.hostname=ray-dashboard.example.com | less + +helm -n rayserve get values rayserve +helm -n rayserve get values rayserve --all +``` diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md new file mode 100644 index 0000000..adedd87 --- /dev/null +++ b/docs/src/content/docs/deployment.md @@ -0,0 +1,197 @@ +--- +title: Deploying on Nebari +description: The Argo CD Application, two NebariApps, and the ignoreDifferences rules KubeRay makes necessary. +--- + +## Two NebariApps, two audiences + +The chart can render two separate `NebariApp` resources: + +| | Serve endpoint | Dashboard | +|---|---|---| +| Value | `nebariapp.serve.enabled` | `nebariapp.dashboard.enabled` | +| Default | `false` | `true` | +| Hostname value | `nebariapp.hostname` | `nebariapp.dashboard.hostname` | +| Backend | `-serve-svc:8000` | `-head-svc:8265` | +| Audience | external API clients | operators and developers | + +Keeping the serve endpoint internal-only is the recommended posture. Notebooks reach it +over cluster DNS, so exposing it externally is only needed for clients outside the cluster +— and an unauthenticated inference endpoint on the public internet is rarely what anyone +means to build. + +:::caution[The serve `NebariApp` needs `nebariapp.hostname`] +Its template is guarded on both `serve.enabled` **and** a non-empty `hostname`. With +`serve.enabled: true` and no hostname, the resource simply is not rendered — no error, no +route. The dashboard `NebariApp` behaves differently: it uses `required`, so a missing +`dashboard.hostname` fails the render loudly. +::: + +## Argo CD + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: rayserve-pack + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "7" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/nebari-dev/nebari-rayserve-pack.git + targetRevision: main + path: chart + helm: + releaseName: rayserve + values: | + nebariapp: + enabled: true + serve: + enabled: false # keep the endpoint internal + dashboard: + enabled: true + hostname: ray-dashboard.example.com + auth: + enabled: true + provider: keycloak + provisionClient: true + redirectURI: /oauth2/callback + destination: + server: https://kubernetes.default.svc + namespace: rayserve + syncPolicy: + automated: + prune: true + selfHeal: true + managedNamespaceMetadata: + labels: + nebari.dev/managed: "true" + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - SkipDryRunOnMissingResource=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: { duration: 5s, factor: 2, maxDuration: 3m } + ignoreDifferences: + - group: "" + kind: Service + jsonPointers: + - /spec/selector + - /spec/clusterIP + - /spec/clusterIPs + - group: ray.io + kind: RayService + jsonPointers: + - /spec/rayClusterConfig + - /status +``` + +Pin `targetRevision` to a tag rather than `main` for anything you depend on. + +## Why `ignoreDifferences` is there + +The KubeRay controller mutates both `Service` and `RayService` objects at runtime — adding +selectors, rewriting autoscaler fields, writing status. Without these rules Argo CD sees +permanent drift, reports `OutOfSync` forever, and with `selfHeal: true` fights the +controller in a loop. + +The two Services this chart renders also carry +`argocd.argoproj.io/compare-options: IgnoreExtraneous`, as does the RayService, which +handles the resources KubeRay creates alongside them. + +:::caution[`/spec/rayClusterConfig` is a very broad ignore] +Combined with `RespectIgnoreDifferences=true` and server-side apply, it tells Argo CD to +stop managing **every field** under `rayClusterConfig` — which is the entire pod spec for +head and workers. + +Most of the time that is harmless. It is not harmless if you enable +[`orgCABundle`](/ca-bundle/), because the CA initContainer, volumes, volume mounts, and +environment variables all live under that path. Argo CD reports a healthy, fully-synced +Application while the running pods never receive any of it. See +[the Argo CD interaction](/ca-bundle/#the-argo-cd-interaction). +::: + +Two other sync options are doing real work: + +- **`SkipDryRunOnMissingResource=true`** lets the first sync proceed before the `RayService` + CRD exists, since the KubeRay operator that registers it is installed by the same chart. +- **`managedNamespaceMetadata`** applies `nebari.dev/managed: "true"`. Without it the + operator ignores both `NebariApp` resources — silently. + +## Authentication + +```yaml +nebariapp: + auth: + enabled: true + provider: keycloak + provisionClient: true + redirectURI: /oauth2/callback + scopes: [openid, profile, email] +``` + +Auth applies to **both** `NebariApp` resources — there is no per-endpoint switch. Envoy +enforces OIDC at the gateway, so neither Ray's dashboard nor the Serve HTTP proxy sees an +unauthenticated request. + +:::caution[`redirectURI` cannot be `/`] +Envoy Gateway rejects a bare `/` as an OIDC callback path. `/oauth2/callback` is the +default and there is no reason to change it — Ray serves nothing at that path. +::: + +Gateway-enforced auth means the Serve endpoint, if exposed, is behind a browser login flow. +That is fine for humans and wrong for programmatic clients: an API caller cannot complete +the redirect. If you need authenticated machine access to inference from outside the +cluster, terminate that separately rather than expecting the OIDC filter to accommodate it. + +## Gateway + +```yaml +nebariapp: + gateway: public # or: internal +``` + +Applies to both resources. `internal` is worth considering for the dashboard even when the +serve endpoint is exposed — it shows cluster internals, logs, and job state. + +## Landing-page tile + +Available on the dashboard `NebariApp` only: + +```yaml +nebariapp: + dashboard: + landingPage: + enabled: true + displayName: "Ray Dashboard" + description: "Monitor and manage Ray clusters and Serve deployments" + category: "Data Science" + priority: 20 + healthCheck: + enabled: true + path: /api/component_activities + intervalSeconds: 30 + timeoutSeconds: 5 +``` + +The health-check path is a real Ray dashboard API endpoint, not `/` — the dashboard's root +is a single-page app that returns 200 even when the backend is unhealthy. + +## Verifying + +```bash +kubectl -n rayserve get nebariapp +kubectl -n rayserve describe nebariapp rayserve-nebari-rayserve-dashboard +kubectl -n rayserve get httproute,securitypolicy +kubectl get namespace rayserve -o jsonpath='{.metadata.labels}' +``` + +A dashboard returning 500 through the gateway usually means the `NebariApp` points at a +service that does not exist — compare `spec.service.name` against `kubectl get svc`. See +[Troubleshooting](/troubleshooting/). diff --git a/docs/src/content/docs/getting-started.md b/docs/src/content/docs/getting-started.md new file mode 100644 index 0000000..b8f14b5 --- /dev/null +++ b/docs/src/content/docs/getting-started.md @@ -0,0 +1,115 @@ +--- +title: Getting started +description: Install the pack and reach the Ray dashboard and Serve endpoint. +--- + +## Prerequisites + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) and [Helm 3](https://helm.sh/docs/intro/install/) +- A Kubernetes cluster — [kind](https://kind.sigs.k8s.io/) is enough for a first look +- For the Nebari path only: nebari-operator, Envoy Gateway, cert-manager, and Keycloak + +## Install + +```bash +cd chart +helm dependency update . +helm install rayserve . --create-namespace -n rayserve --wait --timeout 5m +``` + +`--timeout 5m` is not padding. The Ray image is large, and the head pod has to come up and +start its GCS before workers can join. + +`nebariapp.enabled` defaults to `false`, so this installs standalone. For the Nebari path +see [Deploying on Nebari](/deployment/). + +## What gets deployed + +| Object | Kind | Purpose | +|---|---|---| +| `rayserve-nebari-rayserve` | RayService | Ray cluster and Serve config | +| `rayserve-nebari-rayserve-head-svc` | Service | `:8265` dashboard, `:10001` ray client, `:6379` GCS | +| `rayserve-nebari-rayserve-serve-svc` | Service | `:8000` Serve HTTP | +| `kuberay-operator` | Deployment | Reconciles the RayService | + +Names come from the fullname helper — `-` — so a release named `rayserve` +gives `rayserve-nebari-rayserve`. Long, but predictable, and the same helper feeds the +`NebariApp` service references. + +:::note[The two Services are the chart's, not KubeRay's] +RayService creates its own stable services only once every Serve application reports +healthy. With the default empty `serveApplications` that never happens, so the dashboard +would be unreachable. The chart renders both Services itself, selecting the head pod +directly, so they work from the first second. +::: + +## Reach it + +```bash +# Ray dashboard +kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 -n rayserve + +# Serve HTTP endpoint +kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -n rayserve +``` + +The dashboard at `http://localhost:8265` shows the cluster, its nodes, and the Serve +controller. With no applications deployed the Serve tab is empty — expected. + +## Verify + +```bash +kubectl -n rayserve get rayservice,raycluster,pods + +# The head pod's own view +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) -- ray status +``` + +`ray status` should list the head and one worker with their CPU and memory. A worker stuck +at `0/1 Ready` is worth reading about in [Scaling and GPUs](/scaling/#probes) — the chart +overrides KubeRay's default probes precisely to avoid that. + +## Deploy something + +The fastest check is from inside the cluster: + +```bash +kubectl -n rayserve exec -it $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) -- python -c " +from ray import serve + +@serve.deployment +class Hello: + async def __call__(self, request): + return 'Hello from Ray Serve!' + +serve.run(Hello.bind(), name='hello', route_prefix='/hello') +print('deployed') +" +``` + +Then, with the serve port forwarded: + +```bash +curl http://localhost:8000/hello +# Hello from Ray Serve! +``` + +For real work there are two proper paths — a notebook against `ray://`, or declarative +applications baked into an image. See [Connecting from Jupyter](/jupyter/) and +[Deploying models](/serve-applications/). + +## Uninstall + +```bash +helm uninstall rayserve -n rayserve +``` + +This removes the RayService, and KubeRay tears down the Ray cluster with it. Anything +deployed at runtime through `serve.run()` goes too — Serve state lives in the cluster, not +in Kubernetes. + +## Next + +- **[Connecting from Jupyter](/jupyter/)** — the version-matching requirement bites early +- **[Deploying models](/serve-applications/)** — how to make applications survive a restart +- **[Deploying on Nebari](/deployment/)** — external access with TLS and OIDC diff --git a/docs/src/content/docs/index.md b/docs/src/content/docs/index.md index d862637..fa74a9b 100644 --- a/docs/src/content/docs/index.md +++ b/docs/src/content/docs/index.md @@ -1,22 +1,72 @@ --- title: Introduction -description: "Scalable model serving with Ray Serve through the RayService CRD, with optional routing, TLS, and OIDC via the nebari-operator." +description: Ray Serve model serving on Kubernetes via the RayService CRD, with optional Nebari routing, TLS, and OIDC. --- The Nebari Rayserve Pack deploys [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) -on [Nebari](https://nebari.dev) for scalable model serving, with a `NebariApp` -custom resource for routing, TLS, and gateway authentication on a Nebari -cluster. - -:::note[Documentation in progress] -This site is the scaffolding for the pack's documentation. Content is being -written; until it lands here, the -[repository README](https://github.com/nebari-dev/rayserve-pack#readme) -is the reference for installing and configuring the pack. -::: - -## Contributing to these docs - -Pages live in `docs/src/content/docs/`. See the -[docs README](https://github.com/nebari-dev/rayserve-pack/blob/main/docs/README.md) -for how to run the site locally and add a page. +on Kubernetes using the +[RayService CRD](https://docs.ray.io/en/latest/serve/production-guide/kubernetes.html) — the +recommended way to run Ray Serve in production — with optional routing, TLS, and OIDC +authentication through the +[nebari-operator](https://github.com/nebari-dev/nebari-operator). + +Ray 2.43.0, KubeRay operator 1.3.0. + +``` + KubeRay operator + │ manages + ▼ + RayService ─────► RayCluster ──► head pod :8265 dashboard + │ :8000 serve + │ :10001 ray client + └── worker pod(s) + + head-svc :8265 :10001 :6379 ◄── notebooks (ray:// and HTTP) + serve-svc :8000 ◄── NebariApp ◄── browsers +``` + +## Two ways in + +| From | Path | Authenticated | +|---|---|---| +| Jupyter notebook, in-cluster | straight to the Kubernetes service | no | +| Browser or external client | Envoy Gateway via `NebariApp` | yes, when auth is enabled | + +Notebooks connect over `ray://` and plain HTTP to cluster DNS. That is the normal path for +model development, and it deliberately bypasses the gateway — the Ray client cannot follow +an OIDC redirect. See [Connecting from Jupyter](/jupyter/). + +## What gets deployed + +- **KubeRay operator**, managing the Ray cluster and Serve lifecycle. +- **A `RayService`**, with the Serve proxy pre-initialized on `0.0.0.0:8000` — no manual + `serve start` step. +- **Two stable Kubernetes Services**, `-head-svc` and `-serve-svc`, that exist from the + moment the chart installs. RayService creates its own stable services only after every + Serve application is healthy, which is never on a fresh cluster with no applications. +- **`NebariApp` resources**, optionally, for the serve endpoint and the dashboard — + separate hostnames, because they are different audiences. + +## In this guide + +- **[Getting started](/getting-started/)** — install standalone and reach both endpoints +- **[Deploying on Nebari](/deployment/)** — the Argo CD `Application`, and the + `ignoreDifferences` rules KubeRay makes necessary +- **[Connecting from Jupyter](/jupyter/)** — `ray://`, version matching, and the + NetworkPolicy +- **[Local development](/local-development/)** — the kind stack in `dev/` + +## Guides + +- **[Deploying models](/serve-applications/)** — declarative `serveApplications` versus + deploying from a notebook +- **[Scaling and GPUs](/scaling/)** — replicas, resources, runtime classes, the automatic + GPU toleration, and the probe defaults +- **[Organization CA bundle](/ca-bundle/)** — running behind a TLS-inspecting proxy, and + the Argo CD interaction that silently defeats it +- **[Troubleshooting](/troubleshooting/)** — the failures this pack actually produces + +## Reference + +- **[Configuration](/configuration/)** — every value +- **[Architecture](/architecture/)** — how the pieces fit diff --git a/docs/src/content/docs/jupyter.md b/docs/src/content/docs/jupyter.md new file mode 100644 index 0000000..baab887 --- /dev/null +++ b/docs/src/content/docs/jupyter.md @@ -0,0 +1,146 @@ +--- +title: Connecting from Jupyter +description: Reaching the Ray cluster from an in-cluster notebook — client, versions, and NetworkPolicy. +--- + +From a notebook in the same cluster — for example via +[nebari-data-science-pack](https://github.com/nebari-dev/nebari-data-science-pack) — connect +straight to the Kubernetes services. No gateway, no auth. + +```python +import ray +from ray import serve +import requests + +ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") + +@serve.deployment +class Hello: + async def __call__(self, request): + return "Hello from Ray Serve!" + +serve.run(Hello.bind(), name="hello", route_prefix="/hello") + +resp = requests.get( + "http://rayserve-nebari-rayserve-serve-svc.rayserve.svc.cluster.local:8000/hello" +) +print(resp.text) +# Hello from Ray Serve! +``` + +No manual Serve initialization — the RayService starts the proxy with `host: 0.0.0.0` on +port 8000 automatically. + +The two addresses do different jobs: `ray://…:10001` is the Ray client protocol for +submitting work, and `http://…:8000` is the Serve HTTP proxy for calling deployed models. + +## Versions must match + +:::caution[Ray and Python versions in the notebook must match the cluster] +The Ray client protocol is not version-tolerant. A mismatch fails at `ray.init()`, usually +with a message about the protocol version rather than anything mentioning versions you +recognise. +::: + +The chart deploys **Ray 2.43.0 on Python 3.9**. Confirm against the running cluster rather +than trusting the default: + +```bash +POD=$(kubectl get pod -n rayserve -l ray.io/node-type=head -o name) +kubectl exec -n rayserve $POD -- ray --version +kubectl exec -n rayserve $POD -- python --version +``` + +With [Nebi](https://github.com/nebari-dev/nebari-nebi-pack) managing environments: + +```toml +[workspace] +name = "ray-serve" +channels = ["conda-forge"] +platforms = ["linux-64"] + +[dependencies] +python = "3.9.*" +ray-serve = "2.43.*" +ipykernel = ">=6.0" +``` + +If you change `image.tag`, every notebook environment has to move with it. That coupling is +the main argument for pinning the image tag rather than tracking a moving one. + +## NetworkPolicy + +JupyterHub's default singleuser policy blocks egress to private IPs, which includes every +in-cluster Service. Add to your data-science-pack values: + +```yaml +jupyterhub: + singleuser: + networkPolicy: + egressAllowRules: + privateIPs: true +``` + +Existing servers do not pick this up — users must stop and start from the hub control +panel. + +The symptom is a hang rather than an error: `ray.init()` sits until it times out, and the +`requests.get()` does the same. + +:::caution[`privateIPs: true` is broad] +It permits egress to every private address the notebook can route to, not only Ray. If your +cluster needs tighter scoping, write a targeted rule for the `rayserve` namespace instead — +the ports are `10001` (client), `8000` (serve), and `6379` (GCS) if you use it. +::: + +## Service names + +Both follow `--{head,serve}-svc`: + +```bash +kubectl get svc -n rayserve +``` + +``` +rayserve-nebari-rayserve-head-svc ClusterIP 8265/TCP,10001/TCP,6379/TCP +rayserve-nebari-rayserve-serve-svc ClusterIP 8000/TCP +``` + +Copy them from that output rather than reconstructing them by hand — a `nameOverride` or a +different release name changes both. + +## What survives a restart + +Nothing deployed this way. `serve.run()` puts the deployment in the running Ray cluster's +state; if the RayService rolls the cluster — an image change, a config change, a node +failure — the application is gone. + +That is the right trade for development. For anything that must come back on its own, use +declarative `serveApplications` with the code baked into an image. See +[Deploying models](/serve-applications/). + +## Working with an existing deployment + +Connecting does not require deploying. From a notebook you can inspect and call what is +already running: + +```python +import ray +from ray import serve + +ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") +print(serve.status()) + +handle = serve.get_deployment_handle("MyModel", app_name="my-model") +print(handle.remote({"input": 1}).result()) +``` + +`serve.status()` is also the quickest way to see why an application is not serving — it +reports per-deployment state and the last error. + +## Resources + +A notebook connected over `ray://` submits tasks that run on the Ray **cluster**, using the +head and worker resources configured in the chart — not the notebook pod's. The default is +one worker with 2 CPU and 4Gi. Anything demanding needs the cluster scaled first; see +[Scaling and GPUs](/scaling/). diff --git a/docs/src/content/docs/local-development.md b/docs/src/content/docs/local-development.md new file mode 100644 index 0000000..dc6953a --- /dev/null +++ b/docs/src/content/docs/local-development.md @@ -0,0 +1,117 @@ +--- +title: Local development +description: The kind-based dev stack in dev/. +--- + +## Prerequisites + +docker, kind, helm, kubectl, git. + +## Two stacks + +```bash +cd dev + +make up # kind + MetalLB + Envoy Gateway + cert-manager + Keycloak + # + nebari-operator + Ray Serve with both NebariApps +make up-standalone # kind + Ray Serve only, no operator +make down # delete the cluster +``` + +`make up` finishes by printing both URLs: + +| | Default hostname | +|---|---| +| Ray Serve | `https://rayserve.nebari.local` | +| Ray dashboard | `https://ray-dashboard.nebari.local` | + +Override with `make up HOSTNAME=... DASH_HOSTNAME=...`, or `CLUSTER_NAME=` for the kind +cluster (default `rayserve-dev`). + +`make up-standalone` skips the platform entirely and reuses an existing cluster if one is +there. Reach it by port-forward: + +```bash +kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 +kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 +``` + +## What `make up` does + +1. Creates the kind cluster and installs MetalLB, deriving its IP pool from the `kind` + Docker network at runtime. +2. Clones nebari-operator into `.cache/` and runs its service-install and Keycloak setup + scripts (Envoy Gateway, cert-manager, Keycloak). +3. Installs the operator and labels the `default` namespace `nebari.dev/managed=true`. +4. `helm dependency update` then `helm upgrade --install` with both `NebariApp`s enabled. +5. Waits for `nebariapp/rayserve-nebari-rayserve` to reach `Ready`. +6. Runs the operator's `update-hosts.sh` to add both hostnames to `/etc/hosts` — **this uses + sudo**. + +The `--wait --timeout 5m` on the Helm install is doing real work: the Ray image is large, +and the head has to be up before workers can join. + +## Iterating on the chart + +```bash +make up # re-runs helm upgrade --install against the existing cluster +``` + +The cluster and platform stay in place, so the loop is a Helm upgrade rather than a +rebuild. For a faster check with no cluster at all: + +```bash +helm template rayserve chart --set nebariapp.enabled=true \ + --set nebariapp.hostname=rayserve.nebari.local \ + --set nebariapp.dashboard.hostname=ray-dashboard.nebari.local | less +``` + +Useful for confirming that a values change renders what you expect — particularly the +[CA bundle](/ca-bundle/) injection and the +[GPU toleration](/scaling/#gpus), both of which are conditional. + +## Watching it come up + +```bash +kubectl get pods -w +kubectl get rayservice,raycluster +kubectl logs -l ray.io/node-type=head -f +``` + +The dev stack installs into `default`, not a `rayserve` namespace — that is what the +Makefile labels for the operator. + +## Cleaning up + +```bash +make down +``` + +Deletes the kind cluster. The `/etc/hosts` entries and the `.cache/nebari-operator` clone +are left behind. + +## What local development cannot tell you + +- **The Argo CD `ignoreDifferences` behaviour.** The dev stack uses Helm, so the interaction + that can silently drop the [CA bundle](/ca-bundle/#the-argo-cd-interaction) never appears + here. +- **GPU scheduling.** No GPU nodes, no taints — the toleration logic renders but is never + exercised. +- **Real TLS.** Certificates are issued locally; a public ACME issuer and DNS validation + behave differently. +- **Version-matched notebooks.** There is no JupyterHub in the dev stack, so the Ray client + version coupling described in [Connecting from Jupyter](/jupyter/) goes untested. + +## Docs site + +```bash +cd docs +npm ci +npm run dev # hot reload at http://localhost:4321 +npm run build # static build into docs/dist/ +npm test # unit tests +``` + +Pages live in `docs/src/content/docs/`; the sidebar is in `docs/astro.config.mjs`. Merges to +`main` publish to [packs.nebari.dev/rayserve-pack/](https://packs.nebari.dev/rayserve-pack/), +and pull requests touching `docs/` get a preview URL posted as a comment. diff --git a/docs/src/content/docs/scaling.md b/docs/src/content/docs/scaling.md new file mode 100644 index 0000000..4806c6f --- /dev/null +++ b/docs/src/content/docs/scaling.md @@ -0,0 +1,205 @@ +--- +title: Scaling and GPUs +description: Replicas, resources, runtime classes, the automatic GPU toleration, and the probe defaults. +--- + +## Defaults + +```yaml +head: + resources: + requests: { cpu: "1", memory: "2Gi" } + limits: { cpu: "2", memory: "4Gi" } + +worker: + replicas: 1 + minReplicas: 1 + maxReplicas: 1 + resources: + requests: { cpu: "1", memory: "2Gi" } + limits: { cpu: "2", memory: "4Gi" } +``` + +Sized to fit a kind cluster. One worker, no autoscaling headroom, no GPU. + +## Adding workers + +```yaml +worker: + replicas: 3 + minReplicas: 3 + maxReplicas: 3 + resources: + requests: { cpu: "4", memory: "16Gi" } + limits: { cpu: "8", memory: "32Gi" } +``` + +`minReplicas` and `maxReplicas` both default to `replicas` when unset. Setting them equal +pins the group; setting `maxReplicas` higher lets the Ray autoscaler grow it in response to +pending tasks. + +The Ray autoscaler works within these bounds and asks Kubernetes for pods — it cannot add +nodes. On a cluster with a node autoscaler, a `maxReplicas` above what current nodes can +hold triggers node scale-up; without one, the extra pods stay `Pending`. + +## GPUs + +Three things have to line up. + +**1. Request the GPU resource:** + +```yaml +worker: + resources: + limits: + nvidia.com/gpu: 1 + cpu: "8" + memory: "32Gi" + requests: + cpu: "4" + memory: "16Gi" +``` + +**2. Set the runtime class, if your cluster uses one:** + +```yaml +worker: + runtimeClassName: nvidia +``` + +**3. Tolerate the taint — which the chart does for you.** When either `limits` or `requests` +mentions `nvidia.com/gpu`, the chart injects: + +```yaml +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule +``` + +so pods schedule onto nodes tainted `nvidia.com/gpu=...:NoSchedule` — the pattern +[nebari-infrastructure-core](https://github.com/nebari-dev/nebari-infrastructure-core) uses +for AWS GPU node groups. `operator: Exists` matches any taint value. + +The injection is skipped if you already define a toleration with key `nvidia.com/gpu`, so +your own is treated as a deliberate override. Any other tolerations you list are appended: + +```yaml +worker: + tolerations: + - key: dedicated + operator: Equal + value: ml + effect: NoSchedule +``` + +Both `head` and `worker` support this. A GPU head is unusual — the head coordinates rather +than computes — but it is available. + +**4. Claim the GPU in the deployment.** Kubernetes allocating a GPU to the pod is not the +same as Ray scheduling your replica onto it: + +```yaml +serveApplications: + - name: my-model + import_path: myapp.model:app + deployments: + - name: MyModel + ray_actor_options: + num_gpus: 1 +``` + +Skip this and the deployment runs on CPU inside a pod holding an idle GPU. + +Verify end to end: + +```bash +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=worker -o name) -- nvidia-smi +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) -- ray status +``` + +`ray status` should show `GPU` in the cluster resources. If `nvidia-smi` works but Ray +reports no GPU, the device plugin exposed it to the pod after Ray started — restart the +worker. + +## Probes + +The chart overrides KubeRay's default worker probes, and the reason is worth knowing. + +KubeRay's defaults chain a raylet health check with +`wget http://localhost:8000/-/healthz | grep success`. That second check needs both a +deployed Serve application **and** a local Serve HTTP proxy. On a fresh cluster there are +no applications — `serveApplications` is empty by default — so the check fails and the +worker pod sits at `0/1 Ready` forever +([issue #7](https://github.com/nebari-dev/nebari-rayserve-pack/issues/7)). + +The chart's defaults check the raylet alone: + +```yaml +worker: + readinessProbe: + exec: + command: [bash, -c, "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success"] + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 1 + livenessProbe: + # same command; initialDelaySeconds 30, failureThreshold 120 +``` + +A Ray node is ready when its raylet is healthy. Serve application health is the Serve +controller's business, and since the chart's `serve-svc` targets only the head pod, worker +readiness has no effect on user-visible HTTP routing anyway. + +The liveness probe's `failureThreshold: 120` at `periodSeconds: 5` gives a worker ten +minutes of unhealthy raylet before restart — deliberately tolerant, because a worker busy +with a long task should not be killed for a slow health response. + +:::caution[`{}` does not suppress a probe — use `null`] +Helm's deep merge keeps existing keys when overlaying with an empty map, so +`readinessProbe: {}` leaves the chart's probe in place. To fall back to KubeRay's defaults, +set it to `null` (`~` in YAML): + +```yaml +worker: + readinessProbe: ~ + livenessProbe: ~ +``` +::: + +`head.readinessProbe` and `head.livenessProbe` default to `{}`, which means the head keeps +KubeRay's built-in probes. Override them the same way if you need explicit control. + +## Environment variables + +```yaml +head: + containerEnv: + - name: RAY_DEDUP_LOGS + value: "0" +worker: + containerEnv: + - name: HF_HOME + value: /tmp/hf +``` + +These are concatenated with the CA bundle variables when +[`orgCABundle`](/ca-bundle/) is enabled, so both coexist. + +## Sizing the head + +The head runs the GCS, the dashboard, the Serve controller, and — with the default +`proxyLocation: EveryNode` — an HTTP proxy. It does not run your model replicas unless you +place them there. + +Scale it for coordination load: more workers and more deployments mean more GCS traffic. A +head that starts OOM-killing takes the whole cluster with it, so it is worth headroom. + +## What is not here + +- **Per-deployment autoscaling** — Ray Serve's own `autoscaling_config` goes in a + `serveApplications` deployment entry, not in the chart's values. +- **Multiple worker groups** — the chart renders one `workerGroupSpecs` entry. Heterogeneous + pools (CPU plus GPU) need a chart change or a second release. +- **Node autoscaling** — that is your cluster autoscaler's job. diff --git a/docs/src/content/docs/serve-applications.md b/docs/src/content/docs/serve-applications.md new file mode 100644 index 0000000..8a290c7 --- /dev/null +++ b/docs/src/content/docs/serve-applications.md @@ -0,0 +1,181 @@ +--- +title: Deploying models +description: Declarative serveApplications versus deploying from a notebook. +--- + +Two ways to get a model serving, with different durability. + +| | From a notebook | Declarative | +|---|---|---| +| How | `serve.run()` over `ray://` | `serveApplications` in values | +| Code lives in | the notebook session | a container image | +| Survives a cluster roll | no | yes | +| Zero-downtime upgrade | no | yes, via RayService | +| Good for | development, experiments | production | + +## Declarative applications + +```yaml +image: + repository: your-registry/your-ray-image + tag: "2.43.0-custom" + +serveApplications: + - name: my-model + route_prefix: /predict + import_path: myapp.model:app + deployments: + - name: MyModel + num_replicas: 2 +``` + +This becomes the `applications` list in the RayService's `serveConfigV2`. The RayService +controller takes it from there: deploying, health-monitoring, and performing zero-downtime +upgrades when the config changes. + +`import_path` is `module:attribute` — a Python module importable inside the Ray image, and +an attribute holding a bound Serve application. It is **not** a path in the chart or on +your laptop. + +:::caution[The code must already be in the image] +There is no mechanism here for shipping source into the cluster. `import_path` resolves +against the running container's Python path, so a module the image does not contain fails +to import and the application never becomes healthy. +::: + +## Building the image + +```dockerfile +FROM rayproject/ray:2.43.0 + +COPY myapp /home/ray/myapp +RUN pip install --no-cache-dir -r /home/ray/myapp/requirements.txt +ENV PYTHONPATH=/home/ray +``` + +with `myapp/model.py`: + +```python +from ray import serve + +@serve.deployment +class MyModel: + def __init__(self): + self.model = load_model() + + async def __call__(self, request): + payload = await request.json() + return self.model.predict(payload["input"]) + +app = MyModel.bind() +``` + +Base the image on the **same Ray version** the chart deploys — `image.tag`, default +`2.43.0`. Ray does not tolerate a version skew between the image and `rayVersion`, which +the chart sets from the same value. + +## Serve config options + +Anything valid in +[Ray Serve's config schema](https://docs.ray.io/en/latest/serve/production-guide/config.html) +can go in a `serveApplications` entry, since the list is serialized straight into +`serveConfigV2`: + +```yaml +serveApplications: + - name: my-model + route_prefix: /predict + import_path: myapp.model:app + runtime_env: + pip: ["torch==2.1.0"] + deployments: + - name: MyModel + num_replicas: 2 + max_ongoing_requests: 10 + ray_actor_options: + num_cpus: 1 + num_gpus: 1 +``` + +`runtime_env.pip` installs at deployment time. Convenient for iteration; slow and +network-dependent on every replica start, so bake dependencies into the image for +production. + +`ray_actor_options.num_gpus` is how a deployment claims a GPU. The pod also has to be +scheduled somewhere with one — see [Scaling and GPUs](/scaling/). + +## Multiple applications + +```yaml +serveApplications: + - name: classifier + route_prefix: /classify + import_path: myapp.classifier:app + - name: embedder + route_prefix: /embed + import_path: myapp.embedder:app +``` + +They share the cluster and the HTTP proxy, routed by prefix. Route prefixes must not +overlap. + +## The HTTP proxy + +```yaml +serve: + proxyLocation: EveryNode +``` + +| Value | Effect | +|---|---| +| `EveryNode` (default) | An HTTP proxy on every Ray pod. Each pod is its own scheduling unit, can ingress traffic, and pod-local probes can hit `localhost:8000`. | +| `HeadOnly` | One proxy on the head pod. Saves resources; single ingress point. Useful on very large clusters. | +| `Disabled` | No HTTP at all — programmatic Serve handles only. | + +`EveryNode` is the right default with KubeRay. Note that the chart's `serve-svc` targets +**only the head pod** regardless, so the extra proxies matter for in-cluster direct-to-pod +traffic, not for the Service. + +## Updating + +Change `serveApplications` (or the image tag) and upgrade. The RayService controller +performs a zero-downtime rollout: it brings up a new cluster or new replicas, waits for +health, then shifts traffic. + +```bash +helm upgrade rayserve chart -n rayserve -f my-values.yaml +kubectl -n rayserve get rayservice -o yaml | grep -A20 status +``` + +Under Argo CD, `/status` and `/spec/rayClusterConfig` are in `ignoreDifferences`, so watch +the RayService status rather than the Application's sync state. See +[Deploying on Nebari](/deployment/#why-ignoredifferences-is-there). + +## Checking application health + +```bash +kubectl -n rayserve get rayservice rayserve-nebari-rayserve -o jsonpath='{.status}' | jq +``` + +Or from the dashboard's Serve tab, which shows per-deployment replica counts and the last +error. From Python: + +```python +from ray import serve +print(serve.status()) +``` + +An application stuck in `DEPLOY_FAILED` is nearly always an import error — the module is +not in the image, or a dependency is missing. The Serve controller logs carry the +traceback: + +```bash +kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) | grep -i "deploy" +``` + +## Runtime deployment via the REST API + +The Ray dashboard exposes a REST API for deploying applications without a chart change. +Useful for experiments; the same durability caveat as the notebook path applies — the +config lives in the cluster, and the chart's `serveConfigV2` reasserts itself on the next +RayService reconcile. diff --git a/docs/src/content/docs/troubleshooting.md b/docs/src/content/docs/troubleshooting.md new file mode 100644 index 0000000..ee5e335 --- /dev/null +++ b/docs/src/content/docs/troubleshooting.md @@ -0,0 +1,178 @@ +--- +title: Troubleshooting +description: The failures this pack actually produces, and how to tell them apart. +--- + +## First look + +```bash +kubectl -n rayserve get rayservice,raycluster,pods,svc +kubectl -n rayserve describe rayservice rayserve-nebari-rayserve +kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) +``` + +## Worker stuck at 0/1 Ready + +The chart's probe overrides exist to prevent this, so seeing it means the overrides were +replaced or suppressed. + +KubeRay's default worker probes chain a raylet check with +`wget http://localhost:8000/-/healthz`, which requires a deployed Serve application and a +local HTTP proxy. On a cluster with no applications the check never passes and the pod never +becomes ready ([issue #7](https://github.com/nebari-dev/nebari-rayserve-pack/issues/7)). + +```bash +kubectl -n rayserve get pod -l ray.io/node-type=worker -o jsonpath='{.items[0].spec.containers[0].readinessProbe}' | jq +``` + +It should be the raylet-only check on port `52365`. If it mentions `:8000`, KubeRay's +default is in force — most likely because someone set `readinessProbe: {}` expecting that to +suppress the chart's. It does not; Helm's deep merge keeps existing keys. Use `null` (`~`). +See [Scaling and GPUs](/scaling/#probes). + +## Ray dashboard returns 500 through the gateway + +The `NebariApp` is pointing at a service that does not exist. + +```bash +kubectl -n rayserve get svc +kubectl -n rayserve get nebariapp rayserve-nebari-rayserve-dashboard -o jsonpath='{.spec.service}' +``` + +The stable services are `-nebari-rayserve-head-svc` and +`-nebari-rayserve-serve-svc`. A `nameOverride`, `fullnameOverride`, or an explicit +`nebariapp.service.name` can put these out of step. + +## The serve NebariApp was never created + +Its template requires **both** `nebariapp.serve.enabled` (not `false`) and a non-empty +`nebariapp.hostname`. With the hostname missing, the resource is silently not rendered. + +```bash +kubectl -n rayserve get nebariapp +helm -n rayserve get values rayserve | grep -A5 nebariapp +``` + +The dashboard `NebariApp` behaves differently — it uses `required`, so a missing +`dashboard.hostname` fails the Helm render with a message rather than doing nothing. + +## NebariApp never reaches Ready + +Check the namespace label first: + +```bash +kubectl get namespace rayserve --show-labels | grep nebari.dev/managed +``` + +Missing means the operator is ignoring the resource entirely, with no event to say so: + +```bash +kubectl label namespace rayserve nebari.dev/managed=true +``` + +Under Argo CD, `managedNamespaceMetadata` does this — see +[Deploying on Nebari](/deployment/). + +## Version mismatch connecting from Jupyter + +`ray.init()` fails, usually with a protocol-version message. + +```bash +POD=$(kubectl get pod -n rayserve -l ray.io/node-type=head -o name) +kubectl exec -n rayserve $POD -- ray --version +kubectl exec -n rayserve $POD -- python --version +``` + +Match both in the notebook environment. See +[Connecting from Jupyter](/jupyter/#versions-must-match). + +## JupyterHub notebooks cannot reach Ray + +Connections hang rather than erroring. JupyterHub's default singleuser NetworkPolicy blocks +egress to private IPs: + +```yaml +jupyterhub: + singleuser: + networkPolicy: + egressAllowRules: + privateIPs: true +``` + +Users must restart their server afterwards. + +## Serve application will not deploy + +```bash +kubectl -n rayserve get rayservice rayserve-nebari-rayserve -o jsonpath='{.status}' | jq +``` + +`DEPLOY_FAILED` is nearly always an import error — the module named in `import_path` is not +in the image, or one of its dependencies is missing. The traceback is in the head pod's +logs: + +```bash +kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) | grep -i -A20 "deploy" +``` + +Confirm the module actually imports inside the image: + +```bash +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) \ + -- python -c "import myapp.model; print(myapp.model.app)" +``` + +## CERTIFICATE_VERIFY_FAILED on outbound HTTPS + +A TLS-inspecting proxy. Enable [`orgCABundle`](/ca-bundle/) — and if the cluster is managed +by Argo CD with the example sync policy, read +[the Argo CD interaction](/ca-bundle/#the-argo-cd-interaction) first: the injection can be +silently dropped while everything reports healthy. + +```bash +kubectl -n rayserve exec $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) \ + -- printenv SSL_CERT_FILE +``` + +If that prints the path and httpx calls still fail, it is the +[httpx gap](/ca-bundle/#known-gap-httpx). + +## Argo CD permanently OutOfSync + +The KubeRay controller mutates `Service` and `RayService` at runtime. Without the +`ignoreDifferences` rules — and with `selfHeal: true` — Argo CD fights the controller in a +loop. The full rule set is in [Deploying on Nebari](/deployment/). + +## Pods Pending + +```bash +kubectl -n rayserve describe pod | tail -20 +``` + +`Insufficient cpu`/`memory` means the requests exceed what nodes can offer — the defaults +ask for 1 CPU and 2Gi per pod. For a GPU worker, check whether the node is tainted and +whether the toleration was injected: + +```bash +kubectl -n rayserve get pod -o jsonpath='{.spec.tolerations}' | jq +``` + +The chart injects an `nvidia.com/gpu` toleration only when `resources` mention +`nvidia.com/gpu`. See [Scaling and GPUs](/scaling/#gpus). + +## Head pod OOMKilled + +The head runs the GCS, dashboard, Serve controller, and an HTTP proxy. Coordination load +grows with worker count and deployment count, and a dead head takes the cluster with it. +Raise `head.resources.limits.memory`. + +## Gathering state for an issue + +```bash +kubectl -n rayserve get all +kubectl -n rayserve describe rayservice rayserve-nebari-rayserve +kubectl -n rayserve get rayservice -o yaml +kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) --tail=200 +kubectl -n rayserve get events --sort-by=.lastTimestamp | tail -30 +helm -n rayserve get values rayserve +``` From 64a372bd257528bb1639871d4406e119c4247dc6 Mon Sep 17 00:00:00 2001 From: Johnny Bouder Date: Mon, 24 Aug 2026 08:23:17 -0400 Subject: [PATCH 2/4] docs: correct resource names, scaling story, and dev stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accuracy review findings against the chart and live `helm template` renders. Chart rename (#24) never reached the docs. The fullname helper yields `rayserve-nebari-rayserve-pack`, so all 22 hardcoded names across six pages were wrong — port-forwards, ray.init() addresses, kubectl targets, and NebariApp names. Same bug in dev/Makefile. There is no autoscaler. The chart does not set enableInTreeAutoscaling, so minReplicas/maxReplicas are clamps, not a range something grows within. Rewrote the scaling prose and the configuration table, and dropped the maxReplicas: 8 headroom from the reference values file since nothing would consume it. minReplicas/maxReplicas do not default to replicas. values.yaml pins both to 1, so the template's `default` never fires; KubeRay then clamps desired replicas into [min, max] and `--set worker.replicas=3` yields one worker. Documented as a caution. make up did not deploy both NebariApps. It never set nebariapp.serve.enabled=true, so with the chart default of false only the dashboard rendered — and the `kubectl wait` on the serve NebariApp targeted a resource that was never created, under its pre-rename name. Set the flag and fixed both waits, which makes the page's description true rather than documenting a broken command. Also: - gateway applies to both NebariApps, so the dashboard cannot be internal while the serve endpoint is public; the page claimed otherwise - IgnoreExtraneous skips a resource that is live but not in the desired state; it does not make Argo CD tolerate resources KubeRay creates - route_prefix is optional in Serve's config schema, defaulting to / - image.tag is copied verbatim into rayVersion, so `2.43.0-custom` lands there as a version that does not exist; examples now tag `2.43.0` - notebooks reach Serve over serve-svc, not head-svc (index diagram) - helm template needs `helm dependency update chart` first - canonicalized repo URLs past their rename redirects Verified: npm test 10 passed; build 12 pages; check-links.sh LINKS_OK at BASE=/ and BASE=/rayserve-pack/; every corrected example re-rendered through helm template. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RHXnjMGsnSqmgxnx99Pqj6 --- dev/Makefile | 8 +++--- docs/src/content/docs/architecture.md | 5 ++-- docs/src/content/docs/ca-bundle.md | 2 +- docs/src/content/docs/configuration.md | 23 +++++++++++----- docs/src/content/docs/deployment.md | 17 +++++++----- docs/src/content/docs/getting-started.md | 12 ++++----- docs/src/content/docs/index.md | 5 ++-- docs/src/content/docs/jupyter.md | 14 +++++----- docs/src/content/docs/local-development.md | 16 ++++++++--- docs/src/content/docs/scaling.md | 30 ++++++++++++++------- docs/src/content/docs/serve-applications.md | 13 ++++++--- docs/src/content/docs/troubleshooting.md | 14 +++++----- 12 files changed, 102 insertions(+), 57 deletions(-) diff --git a/dev/Makefile b/dev/Makefile index 24702dd..99a225a 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -118,11 +118,13 @@ up: cluster helm dependency update $(CHART_DIR) helm upgrade --install rayserve $(CHART_DIR) \ --set nebariapp.enabled=true \ + --set nebariapp.serve.enabled=true \ --set nebariapp.hostname=$(HOSTNAME) \ --set nebariapp.dashboard.enabled=true \ --set nebariapp.dashboard.hostname=$(DASH_HOSTNAME) \ --wait --timeout 5m - kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=180s + kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=180s + kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack-dashboard --timeout=180s CLUSTER_NAME=$(CLUSTER_NAME) $(OPERATOR_REPO)/dev/scripts/networking/update-hosts.sh @echo "" @echo "Ray Serve deployed with NebariApp." @@ -144,8 +146,8 @@ up-standalone: @echo "Ray Serve deployed (standalone, no NebariApp)." @echo "" @echo "Access via port-forward:" - @echo " kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000" - @echo " kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265" + @echo " kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000" + @echo " kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265" # -------------------------------------------------------------------------- # update-hosts - update /etc/hosts with all NebariApp hostnames diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index 64a47f8..2721c78 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -78,8 +78,9 @@ selector: ``` They exist from the moment the chart installs, regardless of Serve state. Both carry -`argocd.argoproj.io/compare-options: IgnoreExtraneous` so Argo CD tolerates the Services -KubeRay creates alongside them. +`argocd.argoproj.io/compare-options: IgnoreExtraneous`, which tells Argo CD to skip them +during comparison when they are live in the cluster but absent from the desired state — so +KubeRay adopting and rewriting them does not surface as drift or trigger a prune. Note that `serve-svc` targets **only the head pod**, even under `proxyLocation: EveryNode`. The per-node proxies serve direct-to-pod traffic; the Service does not load-balance across diff --git a/docs/src/content/docs/ca-bundle.md b/docs/src/content/docs/ca-bundle.md index a7046e5..773aca6 100644 --- a/docs/src/content/docs/ca-bundle.md +++ b/docs/src/content/docs/ca-bundle.md @@ -84,7 +84,7 @@ The result: Argo CD reports a healthy, fully-synced Application, and the running receive the CA bundle. TLS calls keep failing with `CERTIFICATE_VERIFY_FAILED` while every dashboard says green. -See [issue #17](https://github.com/nebari-dev/nebari-rayserve-pack/issues/17). +See [issue #17](https://github.com/nebari-dev/rayserve-pack/issues/17). ::: The broad ignore exists only to suppress the autoscaler and runtime mutations KubeRay makes diff --git a/docs/src/content/docs/configuration.md b/docs/src/content/docs/configuration.md index 635b389..e1f637a 100644 --- a/docs/src/content/docs/configuration.md +++ b/docs/src/content/docs/configuration.md @@ -11,7 +11,7 @@ Everything is owned by this chart except `kuberay-operator.*`, which passes thro | Value | Default | Purpose | |---|---|---| | `image.repository` | `rayproject/ray` | Used for both head and workers. | -| `image.tag` | `2.43.0` | Ray version. Also written to the RayService's `rayVersion`. | +| `image.tag` | `2.43.0` | Ray version. Written verbatim into the RayService's `rayVersion`, so tag custom images with the plain Ray version. | For production, build a custom image with your model code — see [Deploying models](/serve-applications/). Notebook environments must match this version; @@ -34,8 +34,8 @@ see [Connecting from Jupyter](/jupyter/#versions-must-match). | Value | Default | Purpose | |---|---|---| | `worker.replicas` | `1` | Worker pods. | -| `worker.minReplicas` | `1` | Autoscaler lower bound; defaults to `replicas`. | -| `worker.maxReplicas` | `1` | Autoscaler upper bound; defaults to `replicas`. | +| `worker.minReplicas` | `1` | Lower clamp on `replicas`. Pinned to `1` in `values.yaml` — it does not follow `replicas`. | +| `worker.maxReplicas` | `1` | Upper clamp on `replicas`. Same caveat; leave it below `replicas` and you get `maxReplicas` workers. | | `worker.resources.requests` | `cpu: 1`, `memory: 2Gi` | — | | `worker.resources.limits` | `cpu: 2`, `memory: 4Gi` | — | | `worker.runtimeClassName` | unset | e.g. `nvidia`. | @@ -49,6 +49,13 @@ Helm's deep merge keeps existing keys when overlaying with an empty map. Use `nu fall back to KubeRay's default. Full rationale in [Scaling and GPUs](/scaling/#probes). ::: +:::caution[Always set `replicas`, `minReplicas`, and `maxReplicas` together] +The chart does not enable `enableInTreeAutoscaling`, so there is no Ray autoscaler and the +group size is exactly `replicas` — clamped into `[minReplicas, maxReplicas]` by KubeRay. +Since `values.yaml` pins both bounds to `1`, raising `replicas` alone changes nothing. See +[Scaling and GPUs](/scaling/#adding-workers). +::: + ## `serve` and `serveApplications` | Value | Default | Purpose | @@ -56,8 +63,9 @@ fall back to KubeRay's default. Full rationale in [Scaling and GPUs](/scaling/#p | `serve.proxyLocation` | `EveryNode` | `EveryNode`, `HeadOnly`, or `Disabled`. | | `serveApplications` | `[]` | Applications, serialized into `serveConfigV2`. | -Each entry needs `name`, `route_prefix`, and an `import_path` resolvable inside the image. -Anything else valid in +Each entry needs a `name` and an `import_path` resolvable inside the image. `route_prefix` +is optional — it defaults to `/`, which means you must set it explicitly once you have more +than one application. Anything else valid in [Ray Serve's config schema](https://docs.ray.io/en/latest/serve/production-guide/config.html) is passed through. See [Deploying models](/serve-applications/). @@ -129,7 +137,7 @@ name has to be updated too. ```yaml image: repository: your-registry/your-ray-image - tag: "2.43.0-custom" + tag: "2.43.0" head: resources: @@ -139,7 +147,7 @@ head: worker: replicas: 2 minReplicas: 2 - maxReplicas: 8 + maxReplicas: 2 runtimeClassName: nvidia resources: requests: { cpu: "4", memory: "16Gi" } @@ -172,6 +180,7 @@ nebariapp: ## Inspecting ```bash +helm dependency update chart # once — templating fails without the kuberay-operator subchart helm template rayserve chart --set nebariapp.enabled=true \ --set nebariapp.dashboard.hostname=ray-dashboard.example.com | less diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index adedd87..596673d 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -42,7 +42,7 @@ metadata: spec: project: default source: - repoURL: https://github.com/nebari-dev/nebari-rayserve-pack.git + repoURL: https://github.com/nebari-dev/rayserve-pack.git targetRevision: main path: chart helm: @@ -102,8 +102,10 @@ permanent drift, reports `OutOfSync` forever, and with `selfHeal: true` fights t controller in a loop. The two Services this chart renders also carry -`argocd.argoproj.io/compare-options: IgnoreExtraneous`, as does the RayService, which -handles the resources KubeRay creates alongside them. +`argocd.argoproj.io/compare-options: IgnoreExtraneous`, as does the RayService. That option +tells Argo CD to skip a resource during comparison when it is live in the cluster but not in +the desired state — so once KubeRay adopts and rewrites these objects they neither show as +drift nor get pruned. :::caution[`/spec/rayClusterConfig` is a very broad ignore] Combined with `RespectIgnoreDifferences=true` and server-side apply, it tells Argo CD to @@ -157,8 +159,11 @@ nebariapp: gateway: public # or: internal ``` -Applies to both resources. `internal` is worth considering for the dashboard even when the -serve endpoint is exposed — it shows cluster internals, logs, and job state. +One value, applied to both resources — there is no per-endpoint override, so exposing the +serve endpoint publicly necessarily exposes the dashboard on the same gateway. Since the +dashboard shows cluster internals, logs, and job state, that pairing is usually the wrong +trade: prefer `internal` with the serve endpoint kept off the gateway entirely +(`serve.enabled: false`). Splitting them across two gateways needs two releases. ## Landing-page tile @@ -187,7 +192,7 @@ is a single-page app that returns 200 even when the backend is unhealthy. ```bash kubectl -n rayserve get nebariapp -kubectl -n rayserve describe nebariapp rayserve-nebari-rayserve-dashboard +kubectl -n rayserve describe nebariapp rayserve-nebari-rayserve-pack-dashboard kubectl -n rayserve get httproute,securitypolicy kubectl get namespace rayserve -o jsonpath='{.metadata.labels}' ``` diff --git a/docs/src/content/docs/getting-started.md b/docs/src/content/docs/getting-started.md index b8f14b5..cc9c028 100644 --- a/docs/src/content/docs/getting-started.md +++ b/docs/src/content/docs/getting-started.md @@ -27,13 +27,13 @@ see [Deploying on Nebari](/deployment/). | Object | Kind | Purpose | |---|---|---| -| `rayserve-nebari-rayserve` | RayService | Ray cluster and Serve config | -| `rayserve-nebari-rayserve-head-svc` | Service | `:8265` dashboard, `:10001` ray client, `:6379` GCS | -| `rayserve-nebari-rayserve-serve-svc` | Service | `:8000` Serve HTTP | +| `rayserve-nebari-rayserve-pack` | RayService | Ray cluster and Serve config | +| `rayserve-nebari-rayserve-pack-head-svc` | Service | `:8265` dashboard, `:10001` ray client, `:6379` GCS | +| `rayserve-nebari-rayserve-pack-serve-svc` | Service | `:8000` Serve HTTP | | `kuberay-operator` | Deployment | Reconciles the RayService | Names come from the fullname helper — `-` — so a release named `rayserve` -gives `rayserve-nebari-rayserve`. Long, but predictable, and the same helper feeds the +gives `rayserve-nebari-rayserve-pack`. Long, but predictable, and the same helper feeds the `NebariApp` service references. :::note[The two Services are the chart's, not KubeRay's] @@ -47,10 +47,10 @@ directly, so they work from the first second. ```bash # Ray dashboard -kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 -n rayserve # Serve HTTP endpoint -kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000 -n rayserve ``` The dashboard at `http://localhost:8265` shows the cluster, its nodes, and the Serve diff --git a/docs/src/content/docs/index.md b/docs/src/content/docs/index.md index fa74a9b..447c111 100644 --- a/docs/src/content/docs/index.md +++ b/docs/src/content/docs/index.md @@ -21,8 +21,9 @@ Ray 2.43.0, KubeRay operator 1.3.0. │ :10001 ray client └── worker pod(s) - head-svc :8265 :10001 :6379 ◄── notebooks (ray:// and HTTP) - serve-svc :8000 ◄── NebariApp ◄── browsers + head-svc :8265 :10001 :6379 ◄── notebooks (ray://), dashboard + serve-svc :8000 ◄── notebooks (HTTP) + ◄── NebariApp ◄── browsers ``` ## Two ways in diff --git a/docs/src/content/docs/jupyter.md b/docs/src/content/docs/jupyter.md index baab887..4eaed29 100644 --- a/docs/src/content/docs/jupyter.md +++ b/docs/src/content/docs/jupyter.md @@ -4,7 +4,7 @@ description: Reaching the Ray cluster from an in-cluster notebook — client, ve --- From a notebook in the same cluster — for example via -[nebari-data-science-pack](https://github.com/nebari-dev/nebari-data-science-pack) — connect +[nebari-data-science-pack](https://github.com/nebari-dev/data-science-pack) — connect straight to the Kubernetes services. No gateway, no auth. ```python @@ -12,7 +12,7 @@ import ray from ray import serve import requests -ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") +ray.init("ray://rayserve-nebari-rayserve-pack-head-svc.rayserve.svc.cluster.local:10001") @serve.deployment class Hello: @@ -22,7 +22,7 @@ class Hello: serve.run(Hello.bind(), name="hello", route_prefix="/hello") resp = requests.get( - "http://rayserve-nebari-rayserve-serve-svc.rayserve.svc.cluster.local:8000/hello" + "http://rayserve-nebari-rayserve-pack-serve-svc.rayserve.svc.cluster.local:8000/hello" ) print(resp.text) # Hello from Ray Serve! @@ -51,7 +51,7 @@ kubectl exec -n rayserve $POD -- ray --version kubectl exec -n rayserve $POD -- python --version ``` -With [Nebi](https://github.com/nebari-dev/nebari-nebi-pack) managing environments: +With [Nebi](https://github.com/nebari-dev/nebi-pack) managing environments: ```toml [workspace] @@ -102,8 +102,8 @@ kubectl get svc -n rayserve ``` ``` -rayserve-nebari-rayserve-head-svc ClusterIP 8265/TCP,10001/TCP,6379/TCP -rayserve-nebari-rayserve-serve-svc ClusterIP 8000/TCP +rayserve-nebari-rayserve-pack-head-svc ClusterIP 8265/TCP,10001/TCP,6379/TCP +rayserve-nebari-rayserve-pack-serve-svc ClusterIP 8000/TCP ``` Copy them from that output rather than reconstructing them by hand — a `nameOverride` or a @@ -128,7 +128,7 @@ already running: import ray from ray import serve -ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") +ray.init("ray://rayserve-nebari-rayserve-pack-head-svc.rayserve.svc.cluster.local:10001") print(serve.status()) handle = serve.get_deployment_handle("MyModel", app_name="my-model") diff --git a/docs/src/content/docs/local-development.md b/docs/src/content/docs/local-development.md index dc6953a..a30ab92 100644 --- a/docs/src/content/docs/local-development.md +++ b/docs/src/content/docs/local-development.md @@ -28,12 +28,19 @@ make down # delete the cluster Override with `make up HOSTNAME=... DASH_HOSTNAME=...`, or `CLUSTER_NAME=` for the kind cluster (default `rayserve-dev`). +:::note[`make up` sets `nebariapp.serve.enabled=true`] +Deliberately unlike the production posture, which keeps the serve endpoint off the gateway. +The dev stack turns it on so both `NebariApp` resources actually render and get exercised — +with the chart default of `false`, the serve one is silently skipped. See +[Deploying on Nebari](/deployment/). +::: + `make up-standalone` skips the platform entirely and reuses an existing cluster if one is there. Reach it by port-forward: ```bash -kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 +kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000 +kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 ``` ## What `make up` does @@ -44,7 +51,8 @@ kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 scripts (Envoy Gateway, cert-manager, Keycloak). 3. Installs the operator and labels the `default` namespace `nebari.dev/managed=true`. 4. `helm dependency update` then `helm upgrade --install` with both `NebariApp`s enabled. -5. Waits for `nebariapp/rayserve-nebari-rayserve` to reach `Ready`. +5. Waits for `nebariapp/rayserve-nebari-rayserve-pack` and + `nebariapp/rayserve-nebari-rayserve-pack-dashboard` to reach `Ready`. 6. Runs the operator's `update-hosts.sh` to add both hostnames to `/etc/hosts` — **this uses sudo**. @@ -61,7 +69,9 @@ The cluster and platform stay in place, so the loop is a Helm upgrade rather tha rebuild. For a faster check with no cluster at all: ```bash +helm dependency update chart # once — templating fails without the kuberay-operator subchart helm template rayserve chart --set nebariapp.enabled=true \ + --set nebariapp.serve.enabled=true \ --set nebariapp.hostname=rayserve.nebari.local \ --set nebariapp.dashboard.hostname=ray-dashboard.nebari.local | less ``` diff --git a/docs/src/content/docs/scaling.md b/docs/src/content/docs/scaling.md index 4806c6f..331fe38 100644 --- a/docs/src/content/docs/scaling.md +++ b/docs/src/content/docs/scaling.md @@ -34,17 +34,25 @@ worker: limits: { cpu: "8", memory: "32Gi" } ``` -`minReplicas` and `maxReplicas` both default to `replicas` when unset. Setting them equal -pins the group; setting `maxReplicas` higher lets the Ray autoscaler grow it in response to -pending tasks. +:::caution[Set all three, every time] +`values.yaml` pins `minReplicas` and `maxReplicas` to `1`. They do **not** follow `replicas` +— the chart's `default` only applies to a value that is absent, and these never are. Raise +`replicas` to `3` on its own and you still get one worker, because KubeRay clamps desired +replicas into the `[minReplicas, maxReplicas]` range. +::: + +There is no autoscaler behind these bounds. The chart does not set +`enableInTreeAutoscaling`, so KubeRay runs no Ray autoscaler sidecar and the group size is +exactly `replicas`. Growing the pool means changing `replicas` and running `helm upgrade`; +`minReplicas` and `maxReplicas` are clamps, not a range something moves within. -The Ray autoscaler works within these bounds and asks Kubernetes for pods — it cannot add -nodes. On a cluster with a node autoscaler, a `maxReplicas` above what current nodes can -hold triggers node scale-up; without one, the extra pods stay `Pending`. +Whether the new pods actually land is a separate question. On a cluster with a node +autoscaler, asking for more than current nodes can hold triggers node scale-up; without one, +the extra pods stay `Pending`. ## GPUs -Three things have to line up. +Four things have to line up — and the fourth is the one people miss. **1. Request the GPU resource:** @@ -131,7 +139,7 @@ KubeRay's defaults chain a raylet health check with deployed Serve application **and** a local Serve HTTP proxy. On a fresh cluster there are no applications — `serveApplications` is empty by default — so the check fails and the worker pod sits at `0/1 Ready` forever -([issue #7](https://github.com/nebari-dev/nebari-rayserve-pack/issues/7)). +([issue #7](https://github.com/nebari-dev/rayserve-pack/issues/7)). The chart's defaults check the raylet alone: @@ -198,8 +206,12 @@ head that starts OOM-killing takes the whole cluster with it, so it is worth hea ## What is not here +- **Ray cluster autoscaling** — the chart does not set `enableInTreeAutoscaling`, so the + worker group never grows on its own. - **Per-deployment autoscaling** — Ray Serve's own `autoscaling_config` goes in a - `serveApplications` deployment entry, not in the chart's values. + `serveApplications` deployment entry, not in the chart's values. It scales replicas within + the resources the cluster already has, which is all it can do without the cluster + autoscaler above. - **Multiple worker groups** — the chart renders one `workerGroupSpecs` entry. Heterogeneous pools (CPU plus GPU) need a chart change or a second release. - **Node autoscaling** — that is your cluster autoscaler's job. diff --git a/docs/src/content/docs/serve-applications.md b/docs/src/content/docs/serve-applications.md index 8a290c7..258c05a 100644 --- a/docs/src/content/docs/serve-applications.md +++ b/docs/src/content/docs/serve-applications.md @@ -18,7 +18,7 @@ Two ways to get a model serving, with different durability. ```yaml image: repository: your-registry/your-ray-image - tag: "2.43.0-custom" + tag: "2.43.0" serveApplications: - name: my-model @@ -71,8 +71,13 @@ app = MyModel.bind() ``` Base the image on the **same Ray version** the chart deploys — `image.tag`, default -`2.43.0`. Ray does not tolerate a version skew between the image and `rayVersion`, which -the chart sets from the same value. +`2.43.0`. Ray does not tolerate a version skew between the image and `rayVersion`, which the +chart sets from the same value. + +That coupling also constrains how you tag. `image.tag` is copied verbatim into the +RayService's `rayVersion`, which KubeRay parses as a Ray version, so tag a custom image +`2.43.0` and distinguish it by repository — not `2.43.0-custom`, which lands in `rayVersion` +as a version that does not exist. ## Serve config options @@ -154,7 +159,7 @@ the RayService status rather than the Application's sync state. See ## Checking application health ```bash -kubectl -n rayserve get rayservice rayserve-nebari-rayserve -o jsonpath='{.status}' | jq +kubectl -n rayserve get rayservice rayserve-nebari-rayserve-pack -o jsonpath='{.status}' | jq ``` Or from the dashboard's Serve tab, which shows per-deployment replica counts and the last diff --git a/docs/src/content/docs/troubleshooting.md b/docs/src/content/docs/troubleshooting.md index ee5e335..0cec1f4 100644 --- a/docs/src/content/docs/troubleshooting.md +++ b/docs/src/content/docs/troubleshooting.md @@ -7,7 +7,7 @@ description: The failures this pack actually produces, and how to tell them apar ```bash kubectl -n rayserve get rayservice,raycluster,pods,svc -kubectl -n rayserve describe rayservice rayserve-nebari-rayserve +kubectl -n rayserve describe rayservice rayserve-nebari-rayserve-pack kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) ``` @@ -19,7 +19,7 @@ replaced or suppressed. KubeRay's default worker probes chain a raylet check with `wget http://localhost:8000/-/healthz`, which requires a deployed Serve application and a local HTTP proxy. On a cluster with no applications the check never passes and the pod never -becomes ready ([issue #7](https://github.com/nebari-dev/nebari-rayserve-pack/issues/7)). +becomes ready ([issue #7](https://github.com/nebari-dev/rayserve-pack/issues/7)). ```bash kubectl -n rayserve get pod -l ray.io/node-type=worker -o jsonpath='{.items[0].spec.containers[0].readinessProbe}' | jq @@ -36,11 +36,11 @@ The `NebariApp` is pointing at a service that does not exist. ```bash kubectl -n rayserve get svc -kubectl -n rayserve get nebariapp rayserve-nebari-rayserve-dashboard -o jsonpath='{.spec.service}' +kubectl -n rayserve get nebariapp rayserve-nebari-rayserve-pack-dashboard -o jsonpath='{.spec.service}' ``` -The stable services are `-nebari-rayserve-head-svc` and -`-nebari-rayserve-serve-svc`. A `nameOverride`, `fullnameOverride`, or an explicit +The stable services are `-nebari-rayserve-pack-head-svc` and +`-nebari-rayserve-pack-serve-svc`. A `nameOverride`, `fullnameOverride`, or an explicit `nebariapp.service.name` can put these out of step. ## The serve NebariApp was never created @@ -104,7 +104,7 @@ Users must restart their server afterwards. ## Serve application will not deploy ```bash -kubectl -n rayserve get rayservice rayserve-nebari-rayserve -o jsonpath='{.status}' | jq +kubectl -n rayserve get rayservice rayserve-nebari-rayserve-pack -o jsonpath='{.status}' | jq ``` `DEPLOY_FAILED` is nearly always an import error — the module named in `import_path` is not @@ -170,7 +170,7 @@ Raise `head.resources.limits.memory`. ```bash kubectl -n rayserve get all -kubectl -n rayserve describe rayservice rayserve-nebari-rayserve +kubectl -n rayserve describe rayservice rayserve-nebari-rayserve-pack kubectl -n rayserve get rayservice -o yaml kubectl -n rayserve logs $(kubectl -n rayserve get pod -l ray.io/node-type=head -o name) --tail=200 kubectl -n rayserve get events --sort-by=.lastTimestamp | tail -30 From 99b5efe80998c497650f5bf3991afde65ac1c0bf Mon Sep 17 00:00:00 2001 From: Johnny Bouder Date: Mon, 24 Aug 2026 08:24:29 -0400 Subject: [PATCH 3/4] fix(ci,readme): finish the chart rename that broke integration tests Same root cause as the previous commit, outside docs/. The rename in #24 left pre-rename resource names in the CI workflows and the README. test-integration.yaml has failed on main on every run since 94e8fba (2026-07-22): both `kubectl wait --for=condition=Ready nebariapp/...` steps target names the chart no longer produces, so the job dies before it reaches the HTTPRoute and auth assertions. test.yaml port-forwards a head-svc under the old name for the same reason. Also correcting two README facts the docs review surfaced: - nebariapp.serve.enabled documented as `true`; values.yaml sets `false`, and the recommendation elsewhere in the README agrees with `false` - image.tag `2.43.0-custom` lands verbatim in the RayService's rayVersion and canonicalizing repo URLs past their rename redirects. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RHXnjMGsnSqmgxnx99Pqj6 --- .github/workflows/test-integration.yaml | 6 +++--- .github/workflows/test.yaml | 2 +- README.md | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 8ba30af..a4b0925 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -144,10 +144,10 @@ jobs: kubectl wait --for=condition=ready pod -l ray.io/node-type=head --timeout=300s - name: Verify NebariApp (serve) - run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve --timeout=300s + run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack --timeout=300s - name: Verify NebariApp (dashboard) - run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-dashboard --timeout=300s + run: kubectl wait --for=condition=Ready nebariapp/rayserve-nebari-rayserve-pack-dashboard --timeout=300s - name: Verify HTTPRoutes created run: | @@ -161,7 +161,7 @@ jobs: - name: Health check Ray Dashboard via port-forward run: | - kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & + kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 & sleep 5 curl -sf http://localhost:8265/api/version kill %1 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bc84c94..640019a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -78,7 +78,7 @@ jobs: - name: Health check Ray Dashboard run: | - kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 & + kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 & sleep 5 curl -sf http://localhost:8265/api/version kill %1 diff --git a/README.md b/README.md index 6933bb1..785a93f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Nebari Ray Serve Software Pack -A [Nebari Software Pack](https://github.com/nebari-dev/nebari-software-pack-template) that deploys [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) on Kubernetes using the [RayService CRD](https://docs.ray.io/en/latest/serve/production-guide/kubernetes.html), with optional routing, TLS, and OIDC authentication via the [nebari-operator](https://github.com/nebari-dev/nebari-operator). +A [Nebari Software Pack](https://github.com/nebari-dev/software-pack-template) that deploys [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) on Kubernetes using the [RayService CRD](https://docs.ray.io/en/latest/serve/production-guide/kubernetes.html), with optional routing, TLS, and OIDC authentication via the [nebari-operator](https://github.com/nebari-dev/nebari-operator). ## Overview @@ -40,10 +40,10 @@ Access via port-forward: ```bash # Ray Dashboard -kubectl port-forward svc/rayserve-nebari-rayserve-head-svc 8265:8265 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-head-svc 8265:8265 -n rayserve # Ray Serve endpoint -kubectl port-forward svc/rayserve-nebari-rayserve-serve-svc 8000:8000 -n rayserve +kubectl port-forward svc/rayserve-nebari-rayserve-pack-serve-svc 8000:8000 -n rayserve ``` ### On a Nebari cluster (via ArgoCD) @@ -63,7 +63,7 @@ metadata: spec: project: default source: - repoURL: https://github.com/nebari-dev/nebari-rayserve-pack.git + repoURL: https://github.com/nebari-dev/rayserve-pack.git targetRevision: main path: chart helm: @@ -127,7 +127,7 @@ spec: ## Connecting from Jupyter -From a notebook running in the same cluster (e.g., via the [nebari-data-science-pack](https://github.com/nebari-dev/nebari-data-science-pack)): +From a notebook running in the same cluster (e.g., via the [nebari-data-science-pack](https://github.com/nebari-dev/data-science-pack)): ```python import ray @@ -135,7 +135,7 @@ from ray import serve import requests # Connect to the Ray cluster -ray.init("ray://rayserve-nebari-rayserve-head-svc.rayserve.svc.cluster.local:10001") +ray.init("ray://rayserve-nebari-rayserve-pack-head-svc.rayserve.svc.cluster.local:10001") # Deploy a model @serve.deployment @@ -146,14 +146,14 @@ class Hello: serve.run(Hello.bind(), name="hello", route_prefix="/hello") # Run inference -resp = requests.get("http://rayserve-nebari-rayserve-serve-svc.rayserve.svc.cluster.local:8000/hello") +resp = requests.get("http://rayserve-nebari-rayserve-pack-serve-svc.rayserve.svc.cluster.local:8000/hello") print(resp.text) # Hello from Ray Serve! ``` No manual Serve initialization is needed — the RayService CRD starts the Serve proxy with `host: 0.0.0.0` automatically. -**Note:** The Ray and Python versions in your Jupyter environment must match the Ray cluster. This chart deploys Ray 2.43.0 with Python 3.9 by default. If using [Nebi](https://github.com/nebari-dev/nebari-nebi-pack) for environment management, create a workspace with: +**Note:** The Ray and Python versions in your Jupyter environment must match the Ray cluster. This chart deploys Ray 2.43.0 with Python 3.9 by default. If using [Nebi](https://github.com/nebari-dev/nebi-pack) for environment management, create a workspace with: ```toml [workspace] @@ -174,7 +174,7 @@ For production, bake your model code into a custom Docker image and declare appl ```yaml image: repository: your-registry/your-ray-image - tag: "2.43.0-custom" + tag: "2.43.0" serveApplications: - name: my-model @@ -196,7 +196,7 @@ Key values in `chart/values.yaml`: | Value | Default | Description | |-------|---------|-------------| | `nebariapp.enabled` | `false` | Create NebariApp resources for routing/TLS/auth | -| `nebariapp.serve.enabled` | `true` | Expose the serve endpoint externally (set `false` to keep internal-only) | +| `nebariapp.serve.enabled` | `false` | Expose the serve endpoint externally (set `false` to keep internal-only) | | `nebariapp.hostname` | - | Hostname for the Ray Serve endpoint (required when serve.enabled) | | `nebariapp.dashboard.enabled` | `true` | Create a separate NebariApp for the Ray Dashboard | | `nebariapp.dashboard.hostname` | - | Hostname for the Ray Dashboard (required when dashboard enabled) | @@ -258,7 +258,7 @@ orgCABundle: > **⚠️ ArgoCD footgun — the CA bundle silently won't apply.** The ArgoCD `Application` shown under [On a Nebari cluster (via ArgoCD)](#on-a-nebari-cluster-via-argocd) sets `RespectIgnoreDifferences=true` together with an `ignoreDifferences` rule on `/spec/rayClusterConfig`. With server-side apply, that combination tells ArgoCD to **stop managing every field under `rayClusterConfig`** — which is exactly where this chart injects the initContainer, volumes, volumeMounts, and CA env vars for the head and worker pods. The result is a silent failure: ArgoCD reports a healthy, fully-synced `Application`, but the running RayService never gets the CA bundle, and TLS calls keep failing with `CERTIFICATE_VERIFY_FAILED`. > -> If you enable `orgCABundle` on a cluster managed by ArgoCD with the example sync policy, you must **narrow the ignore rule** so the CA fields are still reconciled. The broad `/spec/rayClusterConfig` ignore exists only to suppress the autoscaler/runtime mutations KubeRay makes; replace it with targeted pointers (or drop it and ignore only the specific subpaths KubeRay rewrites). After changing it, confirm the head and worker pods actually carry `SSL_CERT_FILE` (`kubectl exec ... -- printenv SSL_CERT_FILE`) rather than trusting the ArgoCD sync status. See [#17](https://github.com/nebari-dev/nebari-rayserve-pack/issues/17) for details. +> If you enable `orgCABundle` on a cluster managed by ArgoCD with the example sync policy, you must **narrow the ignore rule** so the CA fields are still reconciled. The broad `/spec/rayClusterConfig` ignore exists only to suppress the autoscaler/runtime mutations KubeRay makes; replace it with targeted pointers (or drop it and ignore only the specific subpaths KubeRay rewrites). After changing it, confirm the head and worker pods actually carry `SSL_CERT_FILE` (`kubectl exec ... -- printenv SSL_CERT_FILE`) rather than trusting the ArgoCD sync status. See [#17](https://github.com/nebari-dev/rayserve-pack/issues/17) for details. **Coverage caveat — httpx default `verify=True`:** httpx hardcodes its SSL context to `cafile=certifi.where()`, which means it **ignores** `SSL_CERT_FILE`. Application code making httpx calls that need to traverse a TLS-inspecting proxy must construct an explicit context: From 175518773591ee0cb68fcdf71ad337875b2d30d1 Mon Sep 17 00:00:00 2001 From: Johnny Bouder Date: Mon, 24 Aug 2026 08:38:33 -0400 Subject: [PATCH 4/4] fix(ci): hold Ray sizing constant across the auth upgrade With the rename fixed, nebariapp-integration ran past the NebariApp waits for the first time and exposed a second breakage in a step that had never been reached. The upgrade step repeats the install step's --set flags except the sizing overrides, so it silently reverts head requests to 1 CPU / 2Gi and worker replicas to 1. That is a rayClusterConfig change: KubeRay rolls the cluster into pods a 2-CPU runner cannot schedule, generation 2 is never observed, and `--wait --timeout 5m` fails with RayService generation is 2, but latest observed generation is 1 Auth is meant to be the only delta, so carry the sizing through. Verified by rendering both steps' flags: the RayService is now byte-identical between install and upgrade, so nothing rolls, while auth still renders on both NebariApps for the SecurityPolicy assertion. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RHXnjMGsnSqmgxnx99Pqj6 --- .github/workflows/test-integration.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index a4b0925..331462f 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -171,6 +171,11 @@ jobs: # ============================================================ - name: Upgrade with auth enabled run: | + # Auth is the only intended delta. The sizing overrides from the + # install step must be repeated: dropping them doubles the head's + # requests and adds a worker, which rolls the Ray cluster into pods + # a 2-CPU runner cannot schedule, and --wait then times out on + # "generation is 2, but latest observed generation is 1". helm upgrade rayserve chart/ \ --set kuberay-operator.enabled=false \ --set nebariapp.enabled=true \ @@ -179,6 +184,11 @@ jobs: --set nebariapp.dashboard.enabled=true \ --set nebariapp.dashboard.hostname=ray-dashboard.nebari.local \ --set nebariapp.auth.enabled=true \ + --set head.resources.requests.cpu=500m \ + --set head.resources.requests.memory=512Mi \ + --set head.resources.limits.cpu=1 \ + --set head.resources.limits.memory=1Gi \ + --set worker.replicas=0 \ --wait --timeout 5m - name: Verify SecurityPolicies created