diff --git a/.env.example b/.env.example index b2e8827..8b99fc9 100644 --- a/.env.example +++ b/.env.example @@ -29,9 +29,23 @@ BETTER_AUTH_SECRET=your_better_auth_secret_here # Better Auth URL (base URL for auth endpoints) # Development: http://localhost:3000 -# Production: https://your-domain.com +# Production: https://your-domain.com (must match the Databricks OAuth redirect URI) BETTER_AUTH_URL=http://localhost:3000 -NEXT_PUBLIC_BETTER_AUTH_URL=http://localhost:3000 + +# NEXT_PUBLIC_BETTER_AUTH_URL — client-side auth base URL. +# +# ⚠️ DO NOT set this for Vercel preview deployments. +# Preview URLs are dynamic (e.g. https://my-app--team.vercel.app) and +# this variable is baked in at build time. If it is set to a different origin +# than the page serving the request, the browser blocks auth API calls with a +# CORS error. Leave it unset: the auth client automatically uses +# window.location.origin, which is always correct for any deployment URL. +# +# Only set this if you have a stable, canonical URL (e.g. a custom domain +# attached to every deployment) AND that URL is also registered as a +# Databricks OAuth redirect URI. +# +# NEXT_PUBLIC_BETTER_AUTH_URL=https://your-domain.com # Encryption key for OAuth token storage (AES-256-GCM) # Must be a 32-byte hex-encoded string (64 characters) @@ -54,6 +68,21 @@ DATABRICKS_APP_URL=https://your-code-editor-app.databricksapps.com # Databricks notebook editor app URL (Marimo) DATABRICKS_NOTEBOOK_APP_URL=https://your-notebook-editor-app.databricksapps.com +# ============================================================================= +# Agent Panel Configuration (managed-memory OpenAI agent) +# ============================================================================= + +# Toggle the slide-out Agent panel in the SSO-SPN org view. +# Set to "true" to show the panel; anything else hides it. +NEXT_PUBLIC_AGENT_ENABLED=false + +# Deployed Databricks App URL for the managed-memory agent (Genie + memory). +# The Vercel-native reverse proxy at /api/agent-proxy forwards to this app, +# minting the current user's (or guest's) mapped SPN token as a bearer. No Go +# proxy / Cloud Run is required for the agent. Build/deploy the app from the +# vendor/app-templates submodule via scripts/assemble_agent.sh. +DATABRICKS_AGENT_APP_URL=https://your-agent-app.databricksapps.com + # ============================================================================= # Embedded Dashboard Configuration # ============================================================================= @@ -85,7 +114,9 @@ SPN_AUTH_DATABRICKS_ACCOUNTS_URL=https://accounts.cloud.databricks.com # Databricks Workspace URL for SPN tokens SPN_AUTH_DATABRICKS_WORKSPACE_URL=https://your-workspace.cloud.databricks.com -# Okta OAuth Configuration for SPN Auth +# Okta OAuth Configuration for SPN Auth (optional — leave unset if not using Okta) +# When all three are set, an Okta provider is registered for SPN identity mapping. +# When any are absent, the provider is skipped (no build error). SPN_AUTH_OKTA_CLIENT_ID=your_okta_client_id SPN_AUTH_OKTA_CLIENT_SECRET=your_okta_client_secret SPN_AUTH_OKTA_ISSUER=https://your-tenant.okta.com/oauth2/default diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..ab99de0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,33 @@ +## What & why + + + +## Bootstrap / runbook changes + + + +GitHub Actions is currently disabled on this repo, so the invariant guard does **not** run +automatically. Run it locally and paste the result: + +```bash +bash scripts/check-runbook-invariants.sh +``` + +- [ ] Guard passes (or this PR does not touch the runbook/bootstrap) +- [ ] pnpm is still installed via `npm install -g pnpm@` — **not** `corepack enable`/`prepare` (ENV-0, #69) +- [ ] Phase 0 of `BOOTSTRAP.md` still has runnable commands, not prose-only guidance +- [ ] Corporate-network logic still lives only in `scripts/lib/corp-network.sh`, sourced by both consumers +- [ ] `README.md` and `BOOTSTRAP.md` still agree on the pnpm install method + +> Why these exist: the corepack guard was deleted once and then silently reintroduced by a +> docs-sync commit, breaking setup for everyone on a network that blocks public npm. See +> `AGENTS.md` → "Bootstrap invariants". + +## Testing + + + +## Risk / rollback + + diff --git a/.github/workflows/phase8-origin-guard.yml b/.github/workflows/phase8-origin-guard.yml new file mode 100644 index 0000000..f50e6ba --- /dev/null +++ b/.github/workflows/phase8-origin-guard.yml @@ -0,0 +1,21 @@ +name: Phase 8 origin guard + +on: + pull_request: + branches: + - genie-agent + push: + branches: + - genie-agent + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Run hermetic Phase 8 origin tests + run: bash tests/test_phase8_origin.sh diff --git a/.github/workflows/pypi-proxy-guard.yml b/.github/workflows/pypi-proxy-guard.yml new file mode 100644 index 0000000..ff0a80d --- /dev/null +++ b/.github/workflows/pypi-proxy-guard.yml @@ -0,0 +1,21 @@ +name: PyPI proxy guard + +on: + pull_request: + branches: + - genie-agent + push: + branches: + - genie-agent + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Run hermetic proxy guard tests + run: bash tests/test_pypi_proxy_guard.sh diff --git a/.github/workflows/runbook-invariants.yml b/.github/workflows/runbook-invariants.yml new file mode 100644 index 0000000..532aa53 --- /dev/null +++ b/.github/workflows/runbook-invariants.yml @@ -0,0 +1,63 @@ +name: runbook invariants + +# Guards the ENV-0 regression: the runbook must not require corepack (blocked wherever +# public npm is blocked), and the corporate-network bridges must stay reachable from +# BOOTSTRAP.md rather than living only in bootstrap.sh. + +on: + pull_request: + paths: + - BOOTSTRAP.md + - README.md + - package.json + - scripts/** + - .github/workflows/runbook-invariants.yml + push: + branches: [genie-agent] + workflow_dispatch: + +jobs: + invariants: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install ripgrep + run: sudo apt-get update && sudo apt-get install -y ripgrep + + - name: Check runbook invariants + run: bash scripts/check-runbook-invariants.sh + + - name: Shell syntax check + run: | + bash -n scripts/bootstrap.sh + bash -n scripts/lib/corp-network.sh + bash -n scripts/check-runbook-invariants.sh + + # BOOTSTRAP.md Phase 0a asks users to source the library from their own shell, and + # zsh is the macOS default. Verify it is genuinely portable, not just bash-valid. + - name: Install zsh + run: sudo apt-get install -y zsh + + - name: Library must be sourceable from zsh + run: | + zsh -n scripts/lib/corp-network.sh + zsh -c "source scripts/lib/corp-network.sh; ok ok; note note; warn warn; fail fail" + + # Behavioural counterpart to the static checks above. Adapted from the fresh-HOME test on + # fix/corepack-mkdir-home-bin (#67) and reframed to assert the outcome rather than the + # install mechanism. Runs on macOS because that is what the runbook targets. + # NOTE: GitHub runners have unrestricted npm access, so this covers fresh-HOME and the + # version pin — NOT the blocked-registry case, which needs a corporate-network VM. + phase1a-fresh-home: + runs-on: macos-15 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install ripgrep + run: brew install ripgrep + - name: Fresh-HOME Phase 1a regression + run: bash scripts/test-bootstrap-phase1a.sh diff --git a/.gitignore b/.gitignore index 5d8be11..c5e23b6 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,14 @@ go/databricks-proxy go/proxy # Local skills and personal tooling .local/ + +# Assembled agent (ephemeral, from vendor submodule + agent/ overlay) +agent-build/ + +# Python build artifacts +__pycache__/ +*.pyc +.env* + +# Bootstrap runner local state (secrets, resume markers) +.firefly-bootstrap/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8fc3f1f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/app-templates"] + path = vendor/app-templates + url = https://github.com/databricks/app-templates.git diff --git a/.vercelignore b/.vercelignore new file mode 100644 index 0000000..2cb9a09 --- /dev/null +++ b/.vercelignore @@ -0,0 +1,5 @@ +# Agent build inputs/outputs are not part of the Next.js frontend build. +# Leading slash anchors to repo root so we DON'T exclude src/components/agent/. +/agent/ +/agent-build/ +/vendor/ diff --git a/AGENTS.md b/AGENTS.md index c5b8d1b..063ee22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,4 +16,66 @@ - Custom API attribution/user-agent logic appears in `databricks-apps/code-editor/app.py` via explicit `User-Agent` headers. - Pipeline Studio encodes Databricks SQL AI functions (`ai_query`, `ai_extract`), Delta Change Data Feed, and three-level UC table names in `src/lib/pipeline-to-sql.ts`. - Unity Catalog Volumes are first-class in the product via `src/lib/databricks-volumes-api.ts`, including `catalog.schema.volume` parsing and `/Volumes/...` path generation. -- The repo exposes Databricks pipelines via REST API routes, but there is no obvious repository-local IaC or CI/CD automation checked in. +- The repo exposes Databricks pipelines via REST API routes. The only checked-in CI is `.github/workflows/runbook-invariants.yml`; GitHub Actions is currently **disabled at the repository level**, so it does not run — the invariants below are enforced by convention and by running the guard script manually. +- The managed-memory Agent panel embeds a Databricks App (Genie + memory) via a Vercel-native reverse proxy at `src/app/api/agent-proxy/[[...path]]/route.ts` (mints the user/guest SPN token, forwards HTTP + SSE to `DATABRICKS_AGENT_APP_URL`) — distinct from the Go proxy used by the code/notebook editors. UI: `src/components/agent/agent-panel.tsx` (gated by `NEXT_PUBLIC_AGENT_ENABLED`). +- The agent app source is the `vendor/app-templates` git submodule (`agent-openai-agents-sdk` + `e2e-chatbot-app-next`) plus the local `agent/` overlay, merged by `scripts/assemble_agent.sh` into the gitignored `agent-build/`. Keep `vendor/**` pristine; put deltas in `agent/`. + +## Bootstrap invariants — do not regress (ENV-0, #69) + +Read this before editing `BOOTSTRAP.md`, `README.md`, or `scripts/bootstrap.sh`. +Verify any change with `bash scripts/check-runbook-invariants.sh`. + +- **Never require `corepack enable` or `corepack prepare`.** corepack fetches its package + manager from `registry.npmjs.org` and ignores the configured npm registry, so it fails + wherever public npm is blocked or blackholed. Install pnpm with + `npm install -g pnpm@`, which honors the user's own registry. The version + **must** be pinned: pnpm's `latest` dist-tag has shipped a 12.x alpha that ignores + `onlyBuiltDependencies` (`ERR_PNPM_IGNORED_BUILDS`). +- **Phase 0 of `BOOTSTRAP.md` must contain runnable commands, not prose.** Corporate-network + handling lives in `scripts/lib/corp-network.sh` and is sourced by *both* `bootstrap.sh` and + the Phase 0a block in `BOOTSTRAP.md`. Describing a bridge without giving the reader a + command to run it is the exact defect that caused #69 — the runbook is followed + top-to-bottom by agents (that is what the README's "Open in Cursor" badge launches), so an + unexecuted prose bridge means the install runs unbridged. +- **Never duplicate the bridge logic.** One implementation in `scripts/lib/corp-network.sh`; + both consumers source it. Two copies drift, and the drift is invisible until someone on a + restricted network runs the doc path. +- **`README.md` and `BOOTSTRAP.md` must agree** on how pnpm is installed. They contradicted + each other for months (README said "not corepack" while the runbook required corepack). +- **Never probe a registry with `npm ping`.** Corporate mirrors proxy package routes but not + service endpoints — `npm-proxy.dev.databricks.com` returns HTTP 404 for `/-/ping` while + serving `npm view pnpm@10.34.5 version` fine. Probe the real operation instead. +- **Never work around a blocked registry** with `--registry https://registry.npmjs.org`, by + disabling TLS verification, or by editing `/etc/hosts`. Bridge the user's own configured + mirror; never hardcode a mirror URL. +- **Every helper `BOOTSTRAP.md` calls must be defined in a `scripts/lib/*.sh` that the + runbook itself sources.** Not in `bootstrap.sh`, not left to the reader. The runbook + called `store_secret` / `read_secret` for ten days with no definition anywhere in the + file, while `bootstrap.sh`'s version took `(VARNAME, _, KEY)` and every runbook call site + used `$(read_secret KEY)` — so even copying it across did not work. A headless agent + invented its own, used bash-only `${!key}`, and got `bad substitution` under zsh. +- **Runnable blocks must parse under bash *and* zsh.** zsh is the macOS default and the + reader pastes these into their own shell. No `${!var}`, no `declare -F` as a function + test, no `mapfile`/`readarray` (macOS also ships bash 3.2). +- **A code block must do what it looks like it does.** `command -v X || { # install X }` + and `{ : ; }` shipped as the Phase 1b/1e installers: they read as installs and are + no-ops, which is invisible on any machine that already has the tool. +- **Never parse structured output with `grep`.** The `sync.exclude` list opens with a + comment naming `pyproject.toml` and `vendor-wheels/`, so `grep -q pyproject.toml` reports + it excluded when it is not — `bootstrap.sh` failed a correct config that way, and an + agent's `-\s` regex read the same list as empty and passed. Use + `check_sync_exclude_rules`. Same rule for CLI JSON: use the shared parsers. +- **Never derive an id from a `create` response you then act on.** `neonctl projects + create` succeeds server-side before any parse of its output can fail, so a parse bug + orphans a real project. Create, then look the id up by name; fail closed if it is empty. + +Historical note: this regressed once already. `e91322d` deleted the "do not use corepack" +guard and its rationale, then `99f2cc2` — a docs-sync commit — reintroduced corepack as the +first executable command in the runbook. Deleting the reason is what made reintroducing the +bug look reasonable. Keep the rationale attached to the rule. + +The same `99f2cc2` is also the origin of three of the defects listed above: it back-ported +`bootstrap.sh`'s *idempotency* fix for Neon while dropping the *parsing* fix from the same +block, replaced self-contained `keyring` one-liners with calls to functions it did not +carry across, and added the two comment-only installers. Blanket "sync the docs to the +script" commits are how this class of bug arrives. Edit the shared library instead. diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md new file mode 100644 index 0000000..7700c3c --- /dev/null +++ b/BOOTSTRAP.md @@ -0,0 +1,1404 @@ +# BOOTSTRAP.md — Firefly Genie-Agent: End-to-End Setup Runbook + +Harness-agnostic, interactive-auth bootstrap for a fresh deployment of the +Firefly frontend + Databricks managed-memory agent app. An AI agent should +work through this file top-to-bottom, completing every +**[ASK — REQUIRED, BLOCKING]** item in Phase 0 before running any later +phase, then executing each command exactly as written. + +--- + +## Phase 0 — Collect inputs + +> **STOP. This is a blocking step.** Do not run any command from Phase 1 +> onward — including read-only smoke tests, `whoami`, or profile probes — +> until the user has explicitly answered **every** `[ASK — REQUIRED, BLOCKING]` row below. +> +> Ask all `[ASK — REQUIRED, BLOCKING]` values in a single up-front prompt. The "Default" column +> is a *fallback offered to the user*, NOT permission to proceed silently. +> You MUST surface each value and get confirmation, even if you can infer it +> from the environment (e.g. an existing CLI profile, `whoami`, or env var). +> Detection ≠ consent: present what you detected as the suggested answer, +> but still require the user to confirm or override it. + +**Secret storage:** secrets persist in a gitignored, `chmod 600` file at +`$REPO_DIR/.firefly-bootstrap/state.env` (sourced by later phases via +`store_secret`/`read_secret`). This intentionally does **not** use macOS Keychain +(`keyring`) — the target machine may not have Python `keyring`/Keychain wired up. +The file is `0600` and gitignored; never print its contents. + +**Input persistence + resume:** non-secret answers persist to +`~/.firefly-bootstrap/inputs.env`, so a re-run offers to reuse them without + +Working through the phases **by hand or headlessly**, persist the answers yourself once +Phase 0a has sourced the helpers: + +```bash +firefly_store_inputs # every [ASK] key below, from the current shell +firefly_store_inputs UC_CATALOG UC_SCHEMA # ...or just the ones you changed +``` + +Do **not** reach for `for k in ...; do firefly_store_input "$k" "${!k}"; done`. `${!k}` +is bash-only indirect expansion and raises `bad substitution` under zsh, which is the +default shell on macOS. `firefly_store_inputs` does the same job in either shell. +re-prompting. The runner tracks completed phases (`COMPLETED_PHASES`) and supports +**skip-forward resume**: on a re-run, an already-completed phase prompts +`Re-execute Phase N? [y/N]` and pressing Enter **skips** it and advances; a +not-yet-run phase prompts `Execute Phase N? [Y/n]` (Enter proceeds, `n` stops). + +### 0a. Corporate-network setup — **run this before any Phase 1 command** + +This is a real step, not a description. It detects a TLS-intercepting proxy, and bridges +your existing `pip` index into `uv` (`UV_DEFAULT_INDEX`) and your `npm` registry into +corepack (`COREPACK_NPM_REGISTRY`), so those tools use your sanctioned mirrors instead of +blocked public registries. Off-proxy every bridge is a no-op, so it is always safe to run. + +```bash +# Run from the repo root. Safe to re-run; no-ops when public registries are reachable. +source scripts/lib/corp-network.sh +source scripts/lib/runbook.sh # store_secret / read_secret, CLI installers, checks +firefly_bridge_corp_network + +# Confirm what got set (empty output just means nothing needed bridging): +env | grep -E 'UV_DEFAULT_INDEX|UV_SYSTEM_CERTS|PIP_INDEX_URL|COREPACK_NPM_REGISTRY|NODE_EXTRA_CA_CERTS|SSL_CERT_FILE|CURL_CA_BUNDLE|REQUESTS_CA_BUNDLE' +``` + +**Keep this shell.** Both `source` lines define functions that later phases call +(`store_secret`, `read_secret`, `firefly_install_*`, the Phase 3e/4 checks). If you +open a new terminal, re-run both lines from the repo root before continuing. + +If it reports an intercepting proxy and you have verified the SHA-256 against your +organization's known root CA, trust it for this session: + +```bash +FIREFLY_TRUST_PROXY_CA=1 firefly_bridge_corp_network # or: TLS_PEM_PATH= firefly_bridge_corp_network +``` + +> Every value is read from **your own** config — no mirror is hardcoded. `scripts/bootstrap.sh` +> sources this same file, so the runbook and the automated runner cannot drift apart. +> Do **not** work around a blocked registry with `--registry https://registry.npmjs.org` +> or by disabling TLS verification. + +### Required inputs — confirm each before proceeding to Phase 1 + +- [ ] **[ASK — REQUIRED, BLOCKING]** `DATABRICKS_HOST` — workspace URL (e.g. `https://dbc-xxxx.cloud.databricks.com`) +- [ ] **[ASK — REQUIRED, BLOCKING]** `DB_PROFILE` — name for the local Databricks CLI profile +- [ ] **[ASK — REQUIRED, BLOCKING]** `UC_CATALOG` — Unity Catalog catalog to use (must allow MANAGE) +- [ ] **[ASK — REQUIRED, BLOCKING]** `UC_SCHEMA` — schema within that catalog +- [ ] **[ASK — REQUIRED, BLOCKING]** `SEED_SAMPLE_DATA` — if `$UC_CATALOG.$UC_SCHEMA` has **no tables**, copy `samples.wanderbricks` into it so Genie has something to answer from (16 tables, ~815k rows). `no` leaves the schema untouched +- [ ] **[ASK — REQUIRED, BLOCKING]** `GENIE_SPACE_IDS` — existing Genie space id(s) to point the agent at, comma-separated. `None` (the default) means bootstrap may create one for you +- [ ] **[ASK — REQUIRED, BLOCKING]** `CREATE_GENIE_SPACE` — when `GENIE_SPACE_IDS=None`, create a Genie space over the data in `$UC_CATALOG.$UC_SCHEMA`. Ignored when you supplied space ids +- [ ] **[ASK — REQUIRED, BLOCKING]** `GRANT_GUEST_SPACE_ACCESS` — **ask this on every path**, default `yes`. Grants the guest SP `CAN_RUN` on whichever space the app ends up using — one you supplied or one this bootstrap creates — and `SELECT` on the tables it references. Answer `no` and guest users will reach the app but be unable to ask data questions, which is the point of the guest flow; Phase 6c says so plainly when it happens. This answer is honoured exactly: an earlier version silently overrode `no` for a bootstrap-created space, which made this ask a lie +- [ ] **[ASK — REQUIRED, BLOCKING]** `AGENT_APP_NAME` — Databricks App name (dev target; bundle hardcodes this) +- [ ] **[ASK — REQUIRED, BLOCKING]** `DATABRICKS_ACCOUNT_ID` — account ID (a **UUID**, e.g. `32aad83d-ef89-4e74-9969-77784815fd46`) from `accounts.cloud.databricks.com` (Account Console → top-right menu). NB: the account ID is a UUID; the *workspace* ID is the numeric one. +- [ ] **[ASK — REQUIRED, BLOCKING]** `LAKEBASE_NAME` — name for the new Lakebase instance. A **request**, not a guarantee: if `$AGENT_APP_NAME` already exists, its own Lakebase binding wins and Phase 3a reconciles this value to whatever was actually bound +- [ ] **[ASK — REQUIRED, BLOCKING]** `NEON_PROJECT_NAME` — name for the new Neon project +- [ ] **[ASK — REQUIRED, BLOCKING]** `VERCEL_TEAM` — team slug (e.g. `acme-corp` from `vercel.com//...` in the dashboard) +- [ ] **[ASK — REQUIRED, BLOCKING]** `VERCEL_PROJECT` — new Vercel project name +- [ ] **[ASK — REQUIRED, BLOCKING]** `REPO_DIR` — local directory to clone into (created if missing; must be new/empty, **not** your home dir — default `$HOME/firefly`) + +> **`DATABRICKS_HOST` is auto-sanitized to `scheme://host`.** Pasting the full browser +> URL (e.g. `…/?autoLogin=true&o=…&email=…`) is fine — everything after the host is +> stripped. A query/path on the host otherwise pollutes the `DATABRICKS_HOST` env var +> (which overrides the CLI profile) and breaks SDK host-metadata resolution. + +| Variable | Default | How to get it | +|---|---|---| +| `DATABRICKS_HOST` | — | Workspace URL from the browser address bar | +| `DB_PROFILE` | `firefly-deploy` | Any name for the profile in `~/.databrickscfg` | +| `UC_CATALOG` | `workspace` | Writable catalog with MANAGE permission | +| `UC_SCHEMA` | `default` | Schema within that catalog | +| `SEED_SAMPLE_DATA` | `yes` | Only acts when the schema is empty; never overwrites an existing table | +| `GENIE_SPACE_IDS` | `None` | From a space's URL: `…/genie/rooms/`, or `databricks genie list-spaces` | +| `CREATE_GENIE_SPACE` | `yes` | Titled `Firefly Genie Agent — .`; reused, not duplicated, on a re-run | +| `GRANT_GUEST_SPACE_ACCESS` | `yes` | Asked on every path; honoured exactly | +| `AGENT_APP_NAME` | `firefly-openai-managed-mem-v2` | Dev target; bundle hardcodes this | +| `DATABRICKS_ACCOUNT_ID` | — | Account **UUID** from `accounts.cloud.databricks.com` (not the numeric workspace ID) | +| `LAKEBASE_NAME` | `firefly-lb` | Name for the new Lakebase instance. Reconciled in Phase 3a — an existing app's binding overrides it | +| `NEON_PROJECT_NAME` | `firefly-genie` | Name for the new Neon project | +| `VERCEL_TEAM` | — | Team slug from `vercel.com//...` in the dashboard | +| `VERCEL_PROJECT` | `firefly-genie` | New Vercel project name | +| `REPO_DIR` | `$HOME/firefly` | New/empty dir to clone into — **not** `$PWD`/home (a non-empty non-git dir is refused) | + +--- + +## Phase 1 — Auth + tooling (interactive, no tokens stored in files) + +> Install pnpm **first** (later CLIs and the frontend build need it). All CLI installs +> use official releases into `$HOME/bin` / `$HOME/.local/bin` — **no Homebrew required** +> (the target may not have it). Those dirs are added to `PATH` in Phase 0 so every phase +> (including skip-forward resumes) finds the CLIs. Auth steps **skip if already logged in**. + +### 1a. pnpm — pinned install via npm (needed for Drizzle migrations + frontend build) + +```bash +# Confirm your npm registry answers before installing. On a blocked network this prints +# the exact remedy instead of an opaque ECONNREFUSED/ETIMEDOUT/503 later on. +firefly_preflight_npm_registry # from Phase 0a + +# Pin the version: pnpm's "latest" dist-tag has shipped a 12.x alpha that ignores +# onlyBuiltDependencies (→ ERR_PNPM_IGNORED_BUILDS). +corepack disable >/dev/null 2>&1 || true # an enabled corepack shim shadows this install +npm install -g pnpm@10.34.5 +pnpm --version # must print 10.34.5 + +# If pnpm still prints "Update available! 10.34.5 -> 12.0.0-alpha.16", IGNORE IT. +# That alpha is precisely what the pin avoids. Phase 0a exports +# NPM_CONFIG_UPDATE_NOTIFIER=false to suppress the banner; it can reappear in a +# shell that skipped Phase 0a. +``` + +> **Do not use `corepack enable` / `corepack prepare` here (ENV-0).** corepack fetches its +> package manager from `registry.npmjs.org` and ignores your npm registry setting, so it +> fails wherever public npm is blocked or blackholed. `npm install -g` uses the registry +> you already have configured, which is why it needs no bridge. This constraint is enforced +> by `scripts/check-runbook-invariants.sh`. + +### 1b. Databricks CLI OAuth + +```bash +firefly_install_databricks_cli # no-op if present; installs the official release to $HOME/bin +databricks auth login --host "$DATABRICKS_HOST" --profile "$DB_PROFILE" +# Opens browser → U2M OAuth → ~/.databrickscfg. (databricks has no "already authed" guard; +# re-running just refreshes.) Smoke-test: databricks workspace list / --profile "$DB_PROFILE" + +# Does this workspace enforce an IP allowlist? Asked HERE — the first point where +# the CLI exists and is authenticated — and not at Phase 9, because Phases 1-8 all +# succeed with one enabled: the app deploys, the frontend deploys, guest login +# works, and then every Databricks data call from Vercel 403s. Meeting that after +# everything looks fine is how it gets misread as an application bug. +# One shared implementation (scripts/lib/runbook.sh), used here and again at +# Phase 9. There used to be two copies and they reached OPPOSITE conclusions on the +# same workspace: this one reported the pricing-tier error, Phase 9 swallowed it and +# printed "ok: no enabled IP allowlist". +ACL_STATUS="$(firefly_ip_allowlist_status "$DB_PROFILE")" +case "$ACL_STATUS" in + none) echo "no enabled IP allowlist on this workspace" ;; + enabled:*) echo "ENABLED IP allowlist(s): ${ACL_STATUS#enabled:}" + echo " Vercel's egress must be allowed or every data call 403s." ;; + unavailable:*) + # A determinate, reassuring answer that merely LOOKS like an error. + echo "no IP allowlist is possible on this workspace's tier" + echo " (${ACL_STATUS#unavailable:})" + echo " The feature does not exist here, so the data-plane risk this" + echo " check exists for does not apply. Not a failure." ;; + unknown:*) echo "could NOT determine the IP allowlist: ${ACL_STATUS#unknown:}" + echo " Treat that as unknown, never as safe." ;; +esac +# If any are enabled, Vercel's egress must be allowed or the data plane refuses it. +# That is a network decision this runbook cannot make — see "Enterprise network +# controls". Everything else still works. +``` + +### 1c. Neon CLI OAuth (skip if already authed) + +```bash +command -v neonctl || npm install -g neonctl +if ! neonctl me &>/dev/null; then neonctl auth; fi # only opens browser if needed +neonctl me # smoke-test / show identity +# `Projects Limit: 0` in that output is NOT "you cannot create projects" — it is +# an unset quota field. Project create and reuse both work with it at 0; Phase 7 +# proves it. Do not go looking for a plan upgrade. +``` + +### 1d. Vercel CLI OAuth (skip if already authed) + +> **`npm install -g vercel` prints nothing while it works, and can take minutes.** It is not +> stalled. Observed at ~40s on one run and ~2 minutes on another, so do not treat any +> particular number as the timeout to plan around — the first version of this note said "~40s" +> and the next run took three times that, which is exactly how a reassurance becomes the thing +> that misleads. Behind a corp proxy it is slower again. +> +> This matters for the same reason Phase 3a's slow step does: anything that backgrounds a +> command after N seconds returns its own exit 0 while npm is still working, so the install is +> indistinguishable from a hang. Wait on the process rather than on a clock, and do not +> re-run it concurrently — a second `npm install -g` against the same prefix can leave the +> CLI half-written. + +```bash +# Pin to a Tart-tested floor — do not chase npm `latest` (CLI deploy semantics move). +# Override with VERCEL_CLI_VERSION=x.y.z to bump deliberately. +VERCEL_CLI_VERSION="${VERCEL_CLI_VERSION:-56.3.1}" +if command -v vercel &>/dev/null; then + VERCEL_CURRENT=$(vercel --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + # Keep if current >= pin; otherwise install the pin. + if ! printf '%s\n%s\n' "$VERCEL_CLI_VERSION" "$VERCEL_CURRENT" | sort -C -V; then + npm install -g "vercel@${VERCEL_CLI_VERSION}" + fi +else + npm install -g "vercel@${VERCEL_CLI_VERSION}" # install BEFORE login +fi +# Suppresses the CLI's background update/telemetry check, which reaches for PUBLIC +# npm and fails behind a corp proxy. +# +# It does NOT silence "Error: Failed to fetch dist-tags from npm". That was marked +# UNVERIFIED here until a proxied run tested it: every vercel command still printed +# that Error line first and then succeeded. Set it anyway — it does cut outbound +# chatter — but expect the dist-tags line regardless. The note below is the part +# that matters, because that line reads like an auth or install failure and is not. +export VERCEL_TELEMETRY_DISABLED=1 + +if ! vercel whoami &>/dev/null; then vercel login; fi +vercel whoami +``` + +> **`Error: Failed to get package info: Error: Failed to fetch dist-tags from npm` +> is harmless — expect it on nearly every `vercel` command.** It is the CLI's own +> background update check reaching public npm, which a corporate proxy blocks. It +> is printed to stderr, prefixed `Error:`, and appears before output that then +> succeeds — so it reads like an auth or install failure and has stopped readers +> mid-phase. The command's real result is whatever follows. Judge `vercel` by its +> exit code and its output, never by this line. + +### 1e. GitHub CLI (for the submodule; optional otherwise) + +```bash +firefly_install_gh # no-op if present; installs the official release to $HOME/bin +if ! gh auth status &>/dev/null; then gh auth login; fi # browser or PAT +``` + +### 1f. uv (Python package manager; installs to $HOME/.local/bin) + +```bash +# The astral.sh installer verifies its download with `sha256sum`, which stock +# macOS does not have (it ships `shasum`). Without this it prints "skipping +# sha256 checksum verification" and installs an UNVERIFIED binary. The shim makes +# the installer's own check work; it does not bypass anything. +firefly_ensure_sha256sum + +command -v uv || curl -LsSf https://astral.sh/uv/install.sh | sh +uv --version +``` + +--- + +## Phase 2 — Clone and assemble + +```bash +# Idempotent clone: reuse an existing repo at $REPO_DIR if it's already a git checkout; +# clone if the dir is empty/absent; FAIL if it exists, is non-empty, and is not a git repo +# (so we never clobber e.g. your home dir — see the REPO_DIR guidance in Phase 0). +if [[ -d "$REPO_DIR/.git" ]]; then + echo "Repo already present at $REPO_DIR — reusing (skip clone)." +elif [[ -e "$REPO_DIR" && -n "$(ls -A "$REPO_DIR" 2>/dev/null)" ]]; then + echo "ERROR: $REPO_DIR is non-empty and not a git repo — pick a new REPO_DIR." >&2; exit 1 +else + # Branch-TOLERANT clone, because this runbook outlives its own branch. + # + # `--branch genie-agent` is correct today and breaks the day the branch merges and is + # deleted: git fails with "Remote branch genie-agent not found in upstream origin" at + # the first step a new reader takes. Dropping the branch instead breaks TODAY, because + # the default branch does not carry BOOTSTRAP.md yet. Neither fixed choice is right in + # both states, so resolve it: prefer the bootstrap branch while it exists, and fall + # back to the default branch once it has absorbed this runbook. + FIREFLY_REPO="${FIREFLY_REPO:-https://github.com/databrickslabs/firefly.git}" + FIREFLY_BRANCH="${FIREFLY_BRANCH:-genie-agent}" + if git ls-remote --exit-code --heads "$FIREFLY_REPO" "$FIREFLY_BRANCH" >/dev/null 2>&1; then + git clone --branch "$FIREFLY_BRANCH" "$FIREFLY_REPO" "$REPO_DIR" + else + echo "branch '$FIREFLY_BRANCH' is gone — cloning the default branch, which should" + echo "now carry this runbook." + git clone "$FIREFLY_REPO" "$REPO_DIR" + fi +fi +cd "$REPO_DIR" + +# Assert the runbook actually arrived. Without this the first symptom of cloning the +# wrong branch is a missing file several phases later, which reads as a broken repo. +if [[ ! -f BOOTSTRAP.md ]]; then + echo "ERROR: BOOTSTRAP.md is not in this checkout ($(git rev-parse --abbrev-ref HEAD))." >&2 + echo " Set FIREFLY_BRANCH to the branch that carries it and re-run Phase 2." >&2 + exit 1 +fi + +# NOTE: no GitHub fork push. Phase 8 deploys with the `vercel deploy` CLI (uploads the local +# build; no Vercel Git integration), so a user-owned GitHub repo is unnecessary. To enable +# push-to-deploy later, connect a repo from the Vercel dashboard (Project → Settings → Git). + +# Submodule must be initialised before assemble_agent.sh runs. +git submodule update --init + +# Assemble once here — do NOT run assemble_agent.sh again after quickstart (Phase 3a), +# as it wipes agent-build/ including quickstart's .env and vendored wheels. +bash scripts/assemble_agent.sh +``` + +--- + +## Phase 3 — Provision Databricks resources + +### 3a. Run quickstart (creates MLflow experiment + Lakebase) + +```bash +cd "$REPO_DIR/agent-build" + +# Everything below this point is uv-driven. Confirm the package index answers first; +# otherwise the first failure is a bare uv stack trace naming pypi.org, eight phases +# from the actual cause. Requires the Phase 0a `source` lines. +firefly_preflight_pypi_index || return 2>/dev/null || exit 1 + +# Lakebase create-vs-reuse (idempotent): --lakebase-create-new fails on re-run +# ("project slug already exists") AND disables quickstart's own .env-reuse path. +# quickstart names resources deterministically, so if the project's primary endpoint +# already exists, REUSE it; otherwise create new. +LB_ENDPOINT="projects/${LAKEBASE_NAME}/branches/${LAKEBASE_NAME}-branch/endpoints/primary" +if databricks api get "/api/2.0/postgres/${LB_ENDPOINT}" --profile "$DB_PROFILE" &>/dev/null; then + LB_ARG=(--lakebase-autoscaling-endpoint "$LB_ENDPOINT") # reuse existing +else + LB_ARG=(--lakebase-create-new "$LAKEBASE_NAME") # create new +fi + +# Say which way this will go BEFORE the run, because quickstart's own output is +# misleading in BOTH directions: +# * app exists -> ITS Lakebase binding wins and --lakebase-create-new is +# silently ignored, while later summaries still print the name +# that was asked for. +# * app absent -> quickstart prints "Could not fetch app details: App with +# name '...' does not exist or is deleted" and suggests +# `databricks bundle deployment bind`. On a first run nothing +# is wrong: Phase 4 creates the app, and no bind is needed. +firefly_warn_existing_app_wins "$AGENT_APP_NAME" "$DB_PROFILE" + +# Pass --app-name so quickstart does NOT interactively prompt to bind an app. +uv run --python 3.12 python scripts/quickstart.py \ + --profile "$DB_PROFILE" "${LB_ARG[@]}" --app-name "$AGENT_APP_NAME" +# --python 3.12 is required; omitting it picks the latest Python and fails on PyO3. +# quickstart writes agent-build/.env with PGHOST/PGUSER/PGDATABASE/LAKEBASE_* +# and patches agent-build/databricks.yml with the new experiment ID and Lakebase refs. + +# An existing --app-name wins over --lakebase-create-new: quickstart reuses the +# app's own Lakebase binding and never creates the requested project, while every +# later summary would still print $LAKEBASE_NAME. Reconcile so the name you are +# told is the name that exists. +firefly_reconcile_lakebase . + +# Confirm it actually finished before leaving this phase (see the warning below). +assert_bundle_quickstart_ran databricks.yml || return 2>/dev/null || exit 1 +``` + +> ### This step is slow, and a partial run looks like a successful one +> +> Provisioning Lakebase takes **several minutes**, on top of a first-run `uv` sync that +> downloads ~150 packages. Two failure modes look identical to success: +> +> * **Automated harnesses that time-slice long commands.** A wrapper that backgrounds a +> command after N seconds returns *its own* exit 0 while `quickstart.py` is still +> running. On 2026-07-25 a headless agent read that 0 at exactly 30.0 s, moved to +> Phase 4, and deployed an unpatched bundle. **A zero exit code from a wrapper is not +> evidence that quickstart finished.** +> * **Stopping at the first quiet moment.** The last line before the long pause is +> `Creating new Lakebase: `. That is the *start* of provisioning, not the end. +> +> The `assert_bundle_quickstart_ran` line above is the actual completion test: it passes +> only once quickstart has rewritten `experiment_id` in `agent-build/databricks.yml`. If +> it fails, quickstart has not finished — wait for it, or re-run this phase. Do not +> continue to Phase 4; the deploy will fail with a 404 naming the placeholder id. + +### 3b. Verify bundle variables (catalog/schema only) + +`DATABRICKS_HOST` and `DATABRICKS_WORKSPACE_ID` are injected at runtime by +`quickstart.py` — **do not edit them manually**. (`GENIE_ONE_URL` no longer +exists: the attribution link it fed was dead for guest users, who have no +workspace access, and named the wrong backend once the agent defaults to a Genie +space.) The bundle also declares +`catalog` and `schema` variables that default to `workspace` and `default`. + +`catalog` and `schema` are applied at **deploy time via `--var`** (Phase 4) — **do not +edit `databricks.yml` manually**. `DATABRICKS_MEMORY_STORE` resolves from them to +`$UC_CATALOG.$UC_SCHEMA.firefly_managed_memory`. Nothing to run here; the values you +entered in Phase 0 are passed as `--var catalog=$UC_CATALOG --var schema=$UC_SCHEMA` +on every `bundle deploy`/`bundle run`. + +### 3c. Create the UC wheels volume + +```bash +# The schema is assumed to exist, and on a fresh catalog it does not — the +# catalog can hold nothing but `information_schema`, and the volume create then +# fails with a message about the volume rather than the missing schema. Create it +# first; this is a no-op when it is already there. +databricks schemas create "$UC_SCHEMA" "$UC_CATALOG" --profile "$DB_PROFILE" 2>/dev/null \ + || echo "schema ${UC_CATALOG}.${UC_SCHEMA} already exists — continuing." + +# Idempotent: create only if the volume doesn't already exist (create errors on re-run). +if databricks volumes read "${UC_CATALOG}.${UC_SCHEMA}.firefly_wheels" --profile "$DB_PROFILE" &>/dev/null; then + echo "UC volume ${UC_CATALOG}.${UC_SCHEMA}.firefly_wheels already exists — skipping." +else + databricks volumes create "$UC_CATALOG" "$UC_SCHEMA" firefly_wheels MANAGED --profile "$DB_PROFILE" +fi +``` + +### 3d. Vendor Python wheels (build-time offline install) + +```bash +cd "$REPO_DIR/agent-build" +bash scripts/vendor_wheels.sh +# Downloads ~144 cp311 wheels ≤10 MB into vendor-wheels/. +# Wheels >10 MB (pyarrow, scipy, numpy, pandas, mlflow) install from +# pypi-proxy.cloud.databricks.com at build time — that host must be reachable +# from within the Apps build environment. +``` + +### 3e. Confirm sync.exclude rules in agent/databricks.yml + +```bash +check_sync_exclude_rules "$REPO_DIR/agent/databricks.yml" +``` + +Three rules — all three must hold simultaneously: + +| Path | Must be in sync.exclude? | Why | +|---|---|---| +| `pyproject.toml` | **No** — upload it | Apps build needs it to find the dep list | +| `uv.lock` | **Yes** — exclude it | Forces plain `uv sync` (not `--locked`), so `UV_FIND_LINKS` re-resolves with local wheels | +| `vendor-wheels/**` | **No** — upload it | Local wheels must be present for the build to use them | + +> Run the command rather than eyeballing the table or grepping. The `exclude:` list +> opens with comment lines that mention `pyproject.toml` and `vendor-wheels/`, so a +> plain `grep` reports both as excluded when they are not — and a naive `-\s` scan +> reads the list as empty and passes anything. Both mistakes have been made here. + +--- + +## Phase 3f — Seed the data and create the Genie space (BEFORE the app deploys) + +This runs before Phase 4 on purpose, and the ordering is the point. + +The app used to deploy first and be *corrected* afterwards: Phase 4 shipped it with +workspace-wide Genie because no space existed yet, then Phase 6c created a space and +redeployed. Any failure between those two points left the app answering from workspace-wide +Genie — which **guest users cannot query at all**, since they have no workspace access — +and nothing said so, because the deploy really had passed the variable it was given. + +Creating the space first removes that window entirely: the app is born scoped to its space +and never needs the redeploy. Nothing here depends on the app existing. Creating a space +needs a warehouse and tables; only *granting* a service principal `CAN_RUN` on the space +needs the app, and that happens in Phase 6 once the SP exists. + +```bash +cd "$REPO_DIR" + +# The warehouse is resolved HERE rather than in Phase 6, because seeding and space +# creation both need it and both now precede the deploy. Phase 6 recovers it from +# inputs.env rather than resolving it a second time. +WAREHOUSE_ID=$(databricks warehouses list -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; ws=json.load(sys.stdin); \ + print(ws[0]['id'] if ws else '')") +firefly_store_input WAREHOUSE_ID "$WAREHOUSE_ID" +firefly_require WAREHOUSE_ID || return 2>/dev/null || exit 1 + +# Seed + create the space. NO --agent-sp / --guest-sp / --grant-guest here: those service +# principals do not exist yet. Phase 6 and 6b grant on the space once they do, which is +# why this call and that one are separate rather than one step that half-works. +eval "$(bash scripts/genie-data-setup.sh \ + --phase 3f \ + --catalog "$UC_CATALOG" --schema "$UC_SCHEMA" --profile "$DB_PROFILE" \ + --warehouse-id "$WAREHOUSE_ID" \ + --seed "${SEED_SAMPLE_DATA:-yes}" \ + --space-ids "${GENIE_SPACE_IDS:-None}" \ + --defer-grants \ + --create-space "${CREATE_GENIE_SPACE:-yes}")" + +echo "seed=$SEED_STATUS tables=$SEED_TABLE_COUNT mode=$GENIE_MCP_MODE space=$GENIE_SPACE_ID" +store_secret GENIE_SPACE_ID "$GENIE_SPACE_ID" +firefly_store_input GENIE_MCP_MODE "$GENIE_MCP_MODE" +``` + +`GENIE_MCP_MODE` comes back `space` when a usable space was resolved or created, and `one` +only when none could be — no tables to build one over, or you declined at Phase 0. Phase 4 +passes whichever it is straight through, so the app's first deploy is already correct. + +> **`one` is an opt-out, not a soft landing.** If this phase reports `mode=one`, guest users +> will reach the app and be unable to ask data questions at all. That is a legitimate +> outcome when there is genuinely no space to build, and a silent one is what this +> reordering exists to prevent — so it is stated here, at the point it is decided. + +--- + +## Phase 4 — Deploy the agent app + +```bash +cd "$REPO_DIR/agent-build" + +# Refuse to deploy a bundle whose resource bindings quickstart never rewrote. The +# committed experiment id is a placeholder for the authoring workspace; deploying it +# returns "Node ID does not exist (404)", which names the id and nothing else. +assert_bundle_quickstart_ran databricks.yml || return 2>/dev/null || exit 1 + +# Deploy bundle (do NOT re-run assemble_agent.sh here — it wipes quickstart's .env) +# Apply catalog/schema via --var (Phase 3b) — no databricks.yml edits. +# The Genie vars ship with the FIRST deploy. Phase 3f already created the space, so there +# is no window in which the app answers from workspace-wide Genie and no redeploy to +# correct it. Recover them from a fresh shell rather than assuming Phase 3f's shell +# survived; they move together, because genie_mcp.py refuses to boot on space-mode with an +# empty id. +: "${GENIE_SPACE_ID:=$(read_secret GENIE_SPACE_ID 2>/dev/null || true)}" +: "${GENIE_MCP_MODE:=$(firefly_read_input GENIE_MCP_MODE 2>/dev/null || echo one)}" +BUNDLE_VARS=(--var "catalog=$UC_CATALOG" --var "schema=$UC_SCHEMA" + --var "genie_mcp_mode=$GENIE_MCP_MODE" + --var "genie_space_id=${GENIE_SPACE_ID:-none}") +if [ "$GENIE_MCP_MODE" = "space" ] && [ -z "${GENIE_SPACE_ID:-}" ]; then + echo "ERROR: mode=space with no GENIE_SPACE_ID — Phase 3f did not leave one." >&2 + echo " Re-run Phase 3f, or set GENIE_MCP_MODE=one to deploy workspace-wide" >&2 + echo " (guest users cannot query workspace-wide Genie)." >&2 + return 2>/dev/null || exit 1 +fi +databricks bundle deploy --profile "$DB_PROFILE" -t dev "${BUNDLE_VARS[@]}" +databricks bundle run agent_openai_agents_sdk --profile "$DB_PROFILE" -t dev "${BUNDLE_VARS[@]}" + +# Watch until app_status.state = RUNNING (deployment state leads by ~44s) +databricks apps get "$AGENT_APP_NAME" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); \ + print(d['app_status']['state'], d.get('active_deployment',{}).get('status',{}).get('state',''))" +# Expected: RUNNING SUCCEEDED + +# Do NOT trust the deploy's exit code. Databricks CLI v1.9.0 can panic +# (nil pointer in ResourceApp.OverrideChangeDesc) and still exit 0, so the +# runbook reads a crashed deploy as success and every later phase then fails +# opaquely on JSON-parsing a CLI error string. Assert the app actually exists. +if ! databricks apps get "$AGENT_APP_NAME" --profile "$DB_PROFILE" >/dev/null 2>&1; then + echo "✗ Phase 4 did not create $AGENT_APP_NAME, whatever the deploy exit code said." >&2 + echo " Re-read the deploy output for a panic or an env-var rejection." >&2 + echo " A stale bundle state can also cause this: if the app was deleted but" >&2 + echo " /Workspace/Users//.bundle/firefly_openai_managed_mem survives, the" >&2 + echo " CLI diffs against an app that is gone. Delete that path and redeploy." >&2 + return 2>/dev/null || exit 1 +fi +# bootstrap.sh Phase 4 fails closed if agent-build or guest-manager uv.lock +# stamps pypi-proxy.dev.databricks.com (unsanctioned index; implicated in the Apps +# install timeouts). Live pip/uv config only WARNS — set FIREFLY_STRICT_PYPI_PROXY=1 +# to make that fatal too. +# Bootstrap also refuses to bridge pip → uv when the index is that .dev host. +# Manual check / fix: +# grep -R pypi-proxy.dev --include='uv.lock' . +# # point pip/uv at pypi-proxy.cloud.databricks.com, then: +# rm -f uv.lock && uv lock +# # then re-run vendor_wheels.sh if needed +``` + +--- + +## Phase 5 — Set up UC managed memory (required for the headline feature) + +> **Workspace prerequisite: the "Managed Memory for Agents" preview.** Without it +> this phase fails with `NotImplemented: The Managed Memory for Agents preview is +> not enabled for this workspace` — and there is nothing you can do about it from +> here; it is enabled per-workspace by Databricks. This was the single +> most-reported gap in E2E runs (9 of 12) because the runbook charged ahead and +> failed opaquely. The preflight below tells you up front, and lets the rest of +> the bootstrap continue: everything except cross-session memory still works. + +```bash +# Get the app service principal's client ID from the deployed app +SP_CLIENT_ID=$(databricks apps get "$AGENT_APP_NAME" -o json \ + --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; \ + d=json.load(sys.stdin); \ + print(d['service_principal_client_id'])") + +# Attempt the real thing, then classify the failure. An earlier version probed +# /api/2.0/memory-stores and treated a clean response as "preview on" — but that +# path returns `Error: Not Found`, which matched none of its patterns, so it +# reported the preview as ENABLED on a workspace where setup then failed with +# NotImplemented. A preflight that cannot detect the state it exists to detect is +# worse than none: it adds a confident wrong answer. The operation itself is the +# only reliable signal, so run it and read what comes back. +cd "$REPO_DIR/agent-build" +MEM_LOG=$(mktemp) +if uv run --python 3.12 python scripts/setup_memory_store.py "$SP_CLIENT_ID" \ + --memory-store "$UC_CATALOG.$UC_SCHEMA.firefly_managed_memory" \ + --profile "$DB_PROFILE" >"$MEM_LOG" 2>&1; then + echo "✓ UC managed memory store configured" +elif grep -qiE 'not enabled|NotImplemented|preview' "$MEM_LOG"; then + echo "⚠ SKIPPING Phase 5 — the 'Managed Memory for Agents' preview is not enabled" + echo " on this workspace. It is enabled per-workspace by Databricks; there is" + echo " nothing to do from here. Bootstrap continues and the agent still runs —" + echo " it just has no cross-session memory. Re-run this phase once it is on." +else + echo "✗ Phase 5 failed for a reason other than the preview:" >&2 + cat "$MEM_LOG" >&2 +fi +rm -f "$MEM_LOG" +# The UC memory store is a distinct securable — not Lakebase, not auto-created. +# setup_memory_store.py calls the REST API directly (no CLI equivalent). +``` + +--- + +## Phase 6 — Grant agent SP access to your data + +The agent answers Genie queries as its service principal. Grant it: + +```bash +# Restore FIRST — before the very first command that reads one of these variables. +# +# The restore used to sit further down, ahead of the warehouse PATCH, which left THIS +# call (the first in the phase) still reading an empty $SP_CLIENT_ID in a fresh shell. +# The API answered "UpdatePermissions Missing required field: principal", which reads as +# a request-schema bug rather than an unset variable. Placement is the whole point of +# this helper, so it goes above the first consumer, not above the one that happened to +# be noticed first. +firefly_restore_phase6_context "$DB_PROFILE" +firefly_require SP_CLIENT_ID UC_CATALOG || return 2>/dev/null || exit 1 + +# 1. Unity Catalog — USE CATALOG +databricks api patch "/api/2.1/unity-catalog/permissions/catalog/$UC_CATALOG" \ + --profile "$DB_PROFILE" \ + --json "{\"changes\":[{\"principal\":\"$SP_CLIENT_ID\",\"add\":[\"USE CATALOG\"]}]}" + +# 2. SQL warehouse CAN_USE (required for Genie to run queries, and by the +# GRANTs below — resolve it before them). +# Phases 6, 6b and 6c all read WAREHOUSE_ID, SP_CLIENT_ID and GUEST_SP_CLIENT_ID from +# the shell, so a new terminal — or an agent running the phases as separate commands — +# arrives with them empty, and every downstream error then names something else. An +# empty warehouse id turns the PATCH below into `/permissions/warehouses/`, which the +# API rejects as "No API found for 'PATCH /permissions/warehouses/'": a missing route, +# apparently, rather than a missing variable. Restore first, and assert before using. +firefly_restore_phase6_context "$DB_PROFILE" +WAREHOUSE_ID=$(databricks warehouses list -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; ws=json.load(sys.stdin); \ + print(ws[0]['id'] if ws else '')") +# Persist it so 6b and 6c can recover it instead of assuming this shell survived. +firefly_store_input WAREHOUSE_ID "$WAREHOUSE_ID" +firefly_require WAREHOUSE_ID SP_CLIENT_ID || return 2>/dev/null || exit 1 +databricks api patch \ + "/api/2.0/permissions/warehouses/$WAREHOUSE_ID" \ + --profile "$DB_PROFILE" \ + --json "{\"access_control_list\":[{\"service_principal_name\":\"$SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}" + +# 3. USE SCHEMA + SELECT on the data Genie answers over. +# +# These used to be commented-out SQL telling you to "open a warehouse session" +# and paste them yourself — which does not work: the principal has to be +# backquoted in SQL, and backquotes inside a double-quoted `--json "..."` +# argument are command substitution. Nobody could run them as written, so the +# grants were silently skipped. firefly_sql executes them directly. +firefly_sql "$WAREHOUSE_ID" \ + "GRANT USE SCHEMA ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$SP_CLIENT_ID\`" +firefly_sql "$WAREHOUSE_ID" \ + "GRANT SELECT ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$SP_CLIENT_ID\`" +``` + +--- + +## Phase 6b — Create guest service principal with M2M credentials + +The guest login flow (`/api/guest/spns`) requires a Databricks Service Principal +with a known **client ID** and **client secret** (M2M OAuth credentials). The app SP +from Phase 5 does not expose a secret — create a dedicated guest SP. + +The Firefly frontend proxies the agent panel with the guest SP's M2M token. Without +**`CAN_USE` on the agent app**, the Databricks Apps front door rejects that token and +the embedded panel redirects to Databricks OAuth instead of loading. + +```bash +# 1. Create the service principal at workspace level — IDEMPOTENT: SCIM displayName is +# NOT unique, so a plain `create` on re-run makes a DUPLICATE SP (new id + secret) and +# orphans the old one. Reuse an existing firefly-guest-sp if present. +GUEST_SP_RESP=$(databricks service-principals list \ + --filter 'displayName eq "firefly-guest-sp"' -o json --profile "$DB_PROFILE" 2>/dev/null \ + | python3 -c " +import sys, json +raw = sys.stdin.read().strip() +try: l = json.loads(raw) if raw else [] +except ValueError: l = [] # SAFE-EMPTY: empty means 'no match', and the next + # step CREATES the SP. No claim is made about state. +m = [s for s in (l or []) if s.get('displayName') == 'firefly-guest-sp'] +print(json.dumps(m[0]) if m else '')") +if [[ -z "$GUEST_SP_RESP" ]]; then + GUEST_SP_RESP=$(databricks service-principals create \ + --display-name "firefly-guest-sp" -o json --profile "$DB_PROFILE") +fi + +# Note: the CLI returns SCIM camelCase — use applicationId, not application_id +GUEST_SP_CLIENT_ID=$(echo "$GUEST_SP_RESP" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['applicationId'])") +GUEST_SP_NUM_ID=$(echo "$GUEST_SP_RESP" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + +# 2. Generate an OAuth M2M secret at workspace level (no account console needed) +GUEST_SP_SECRET=$(databricks service-principal-secrets-proxy create \ + "$GUEST_SP_NUM_ID" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['secret'])") + +# 3. Store both values in $REPO_DIR/.firefly-bootstrap/state.env (0600, gitignored) — never print them +store_secret GUEST_SP_CLIENT_ID "$GUEST_SP_CLIENT_ID" +# An empty principal makes the warehouse PATCH answer "Principal: +# ServicePrincipalName() does not exist", which reads as a SCIM problem rather than an +# unset variable. Assert here, where the name is still the obvious cause. +firefly_restore_phase6_context "$DB_PROFILE" +firefly_require GUEST_SP_CLIENT_ID WAREHOUSE_ID || return 2>/dev/null || exit 1 +# Minted in an earlier phase, so a fresh shell arrives without it: recover before +# storing, or this line is a save that saves nothing. +: "${GUEST_SP_SECRET:=$(read_secret GUEST_SP_SECRET 2>/dev/null || true)}" +store_secret GUEST_SP_SECRET "$GUEST_SP_SECRET" + +# 4. Grant the guest SP data access. Executed, not described: see the note in +# Phase 6 — backquoted principals cannot be pasted into `--json "..."`. +firefly_sql "$WAREHOUSE_ID" \ + "GRANT USE CATALOG ON CATALOG \`$UC_CATALOG\` TO \`$GUEST_SP_CLIENT_ID\`" +firefly_sql "$WAREHOUSE_ID" \ + "GRANT USE SCHEMA ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$GUEST_SP_CLIENT_ID\`" +firefly_sql "$WAREHOUSE_ID" \ + "GRANT SELECT ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$GUEST_SP_CLIENT_ID\`" + +# 5. SQL warehouse CAN_USE (required for Genie to run queries as the guest SP) +# Re-use WAREHOUSE_ID from Phase 6 if still in shell; otherwise list warehouses again. +databricks api patch \ + "/api/2.0/permissions/warehouses/$WAREHOUSE_ID" \ + --profile "$DB_PROFILE" \ + --json "{\"access_control_list\":[{\"service_principal_name\":\"$GUEST_SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}" + +# 6. Agent app CAN_USE (required — without this the app rejects the guest M2M token) +databricks api patch \ + "/api/2.0/permissions/apps/$AGENT_APP_NAME" \ + --profile "$DB_PROFILE" \ + --json "{\"access_control_list\":[{\"service_principal_name\":\"$GUEST_SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}" +``` + +--- + +## Phase 6c — Grant the service principals on the Genie space + +Phase 3f already seeded the data and created the space, before the app was ever deployed. +What is left is the half that genuinely needs the service principals to exist: granting the +agent SP and the guest SP `CAN_RUN` on that space, and `SELECT` on the tables it references. + +Splitting it this way is deliberate. The two halves have different prerequisites — creating +a space needs a warehouse and tables, granting on it needs an app — and running them as one +step is what forced the app to deploy in the wrong mode and be corrected afterwards. + +```bash +cd "$REPO_DIR" + +# Both SPs and the space id come from earlier phases, so recover them rather than assuming +# one shell has spanned all of Phase 3f through 6b. +firefly_restore_phase6_context "$DB_PROFILE" +: "${GENIE_SPACE_ID:=$(read_secret GENIE_SPACE_ID 2>/dev/null || true)}" +: "${GENIE_MCP_MODE:=$(firefly_read_input GENIE_MCP_MODE 2>/dev/null || echo one)}" + +if [ "$GENIE_MCP_MODE" != "space" ] || [ -z "${GENIE_SPACE_ID:-}" ]; then + # Nothing to grant on. Say what that costs rather than skipping in silence, which is the + # failure this reordering was written to remove. + warn "no Genie space to grant on (mode=${GENIE_MCP_MODE:-unset})." + warn " Guest users will reach the app and be unable to ask data questions." + warn " Re-run Phase 3f if you expected a space to exist." +else + firefly_require SP_CLIENT_ID WAREHOUSE_ID || return 2>/dev/null || exit 1 + + # Grants only: --seed no and --create-space no, because Phase 3f did both. Passing the + # space id explicitly means this cannot accidentally create a second one. + eval "$(bash scripts/genie-data-setup.sh \ + --catalog "$UC_CATALOG" --schema "$UC_SCHEMA" --profile "$DB_PROFILE" \ + --warehouse-id "$WAREHOUSE_ID" \ + --phase 6c \ + --seed skip --create-space no \ + --space-ids "$GENIE_SPACE_ID" \ + --grant-guest "${GRANT_GUEST_SPACE_ACCESS:-yes}" \ + --guest-sp "${GUEST_SP_CLIENT_ID:-}" \ + --agent-sp "$SP_CLIENT_ID")" + + ok "granted on Genie space $GENIE_SPACE_ID (agent SP, and guest SP when asked for)" +fi +``` + +There is **no redeploy here**. The app received `genie_mcp_mode` and `genie_space_id` on its +first deploy in Phase 4, so it has been scoped to this space since it started. Grants change +what the space can read, not which space the app talks to. + +> **If you are resuming an older run** whose app was deployed before Phase 3f existed, it +> may still be in `one` mode. Check with `databricks apps get "$AGENT_APP_NAME" -o json` and +> re-run Phase 4 if `GENIE_MCP_MODE` is not `space`; Phase 4 is idempotent. + +--- + +## Phase 7 — Neon database + +```bash +# Credentials from neonctl auth (Phase 1b) — no API key needed. + +# Get org ID (if the account belongs to an org; skip --org-id if personal account) +ORG_ID=$(neonctl orgs list --output json 2>/dev/null \ + | python3 -c " +import sys, json +raw = sys.stdin.read().strip() +try: orgs = json.loads(raw) if raw else [] +except ValueError: orgs = [] # SAFE-EMPTY: empty means 'personal account', and + # --org-id is simply omitted. No claim is made. +print(orgs[0]['id'] if orgs else '')" \ + || echo "") + +# Create project — IDEMPOTENT: Neon project names are NOT unique (id-based); a plain +# `create` on re-run makes a SECOND project, orphans the first, and can hit the quota. +# Reuse an existing project with the same name if present. +ORG_FLAG=(); [[ -n "$ORG_ID" ]] && ORG_FLAG=(--org-id "$ORG_ID") +PROJECT_ID=$(firefly_neon_project_id "${ORG_FLAG[@]}") +if [[ -z "$PROJECT_ID" ]]; then + # Resolve the id by re-listing, NOT by parsing the create response. `create` + # succeeds server-side before any parse of its output can fail, so a parse bug + # there silently orphans a real project — which is how two projects named + # firefly-genie appeared. One lookup path, and it self-heals. + neonctl projects create --name "$NEON_PROJECT_NAME" "${ORG_FLAG[@]}" --output json >/dev/null + PROJECT_ID=$(firefly_neon_project_id "${ORG_FLAG[@]}") +fi +# An empty id makes the next call ambiguous ("Multiple projects found") and stores +# an empty DATABASE_URL, which only surfaces later in drizzle-kit. Stop here instead. +[[ -n "$PROJECT_ID" ]] || { echo "ERROR: no Neon project id for '$NEON_PROJECT_NAME'" >&2; return 2>/dev/null || exit 1; } + +# Get pooled connection string and store in state.env (0600, gitignored) +DB_URL=$(neonctl connection-string --project-id "$PROJECT_ID" --pooled) +store_secret DATABASE_URL "$DB_URL" +``` + +> The Neon API requires `org_id` in the project create body if the account is +> org-scoped. `neonctl orgs list` handles detection; personal accounts skip it. + +### Run Drizzle migrations + +```bash +cd "$REPO_DIR" +pnpm install # pnpm 10.34.5 pinned in Phase 1a; installs node_modules +DB_URL=$(read_secret DATABASE_URL) # from .firefly-bootstrap/state.env +DATABASE_URL="$DB_URL" node_modules/.bin/drizzle-kit push +``` + +--- + +## Phase 8 — Vercel frontend + +### 8a. Create + link project (no Git integration) + force Next.js preset + +```bash +cd "$REPO_DIR" +# Pre-create the project (idempotent) so `vercel link` only ATTACHES. Creating a NEW +# project via link also tries to wire up Git auto-deploy — detecting the repo remotes, +# prompting "which remote?", and calling Git connect, which needs a Vercel↔GitHub Login +# Connection many accounts lack (→ HTTP 400). We deploy via the CLI and need NO Git +# integration. (Enable push-to-deploy later from the dashboard: Project → Settings → Git.) +vercel project add "$VERCEL_PROJECT" --scope "$VERCEL_TEAM" 2>/dev/null || true +vercel link --project "$VERCEL_PROJECT" --scope "$VERCEL_TEAM" --yes --non-interactive + +# Force the Next.js framework preset. A project created with framework:null builds +# `next build` but Vercel serves the output as STATIC → every route 404s despite a +# "Ready" deployment. PATCH the preset via the API (flips all routes 200). +# Token sources in order: explicit env (CI / non-interactive / a token-based setup), +# then the CLI's store on macOS, then its XDG location. A single hardcoded path is a +# silent 404 or a hard stop for anyone whose Vercel auth does not live there — and +# ~/Library/.../auth.json only exists after an interactive `vercel login`. +# Sets V_TOKEN / V_ORG / V_PROJ (scripts/lib/runbook.sh). Phase 8e calls the same +# helper, so a fresh shell there cannot produce a different answer. +firefly_vercel_context "$REPO_DIR" +curl -s -X PATCH "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" \ + -H "Authorization: Bearer $V_TOKEN" -H "Content-Type: application/json" \ + -d '{"framework":"nextjs"}' -o /dev/null +``` + +### 8a-2. Resolve the origin Vercel serves this project on + +```bash +# NEVER build this host from $VERCEL_PROJECT. `.vercel.app` is globally unique +# across all Vercel accounts, and when the name is taken Vercel assigns a RANDOM suffix +# (`demo` -> `demo-zeta-seven-61.vercel.app`) — the host cannot be guessed (#19). +# +# Read it here, BEFORE the first deploy: the domain is allocated at project-creation +# time, so it is already available. That is also what makes a RE-RUN correct — parsing +# `vercel deploy` output only yields the production domain on a project's FIRST deploy; +# on any later run a bare deploy is a preview with a per-deployment host. +curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ/domains?teamId=$V_ORG" > /tmp/domains.json +curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" > /tmp/project.json + +# targets.production.alias[0] once a production deploy exists (it can disagree with +# /domains, so it wins); otherwise the single verified .vercel.app from /domains. +# If neither yields exactly one host, STOP — do not fall back to a guess. +APP_ORIGIN=$(python3 - /tmp/domains.json /tmp/project.json <<'PY' +import json, sys +domains = json.load(open(sys.argv[1])); project = json.load(open(sys.argv[2])) +alias = [a for a in (((project.get("targets") or {}).get("production") or {}).get("alias") or []) if a] +if alias: + print("https://" + alias[0]); raise SystemExit +hosts = [d["name"] for d in (domains.get("domains") or []) + if d.get("verified") and not d.get("gitBranch") and not d.get("redirect") + and str(d.get("name", "")).endswith(".vercel.app")] +if len(hosts) == 1: + print("https://" + hosts[0]) +PY +) +[[ -n "$APP_ORIGIN" ]] || { echo "No verified .vercel.app domain — refusing to guess (#19)"; exit 1; } +store_secret APP_ORIGIN "$APP_ORIGIN" +``` + +### 8b. Set environment variables + +#### Tier 1 — required for guest login path (Phase 9 verification) + +```bash +AGENT_APP_URL=$(databricks apps get "$AGENT_APP_NAME" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('url',''))") +BETTER_AUTH_SECRET=$(openssl rand -base64 32) +ENCRYPTION_KEY=$(openssl rand -hex 32) +GUEST_API_SECRET=$(openssl rand -hex 64) +DB_URL=$(read_secret DATABASE_URL) # from state.env + +# Persist the minted secrets NOW, not at the end of Phase 8. They are generated +# here and were only written to state.env in 8d; a shell that died in between +# left Vercel holding a value the local side no longer knew, and `vercel env +# pull` returns an 11-character redacted placeholder rather than the secret. The +# only recovery was a remint. Writing them at mint time removes that window. +store_secret BETTER_AUTH_SECRET "$BETTER_AUTH_SECRET" +store_secret ENCRYPTION_KEY "$ENCRYPTION_KEY" +store_secret GUEST_API_SECRET "$GUEST_API_SECRET" + +# Clear stale JWKS. Phase 7 REUSES a same-named Neon project, but BETTER_AUTH_SECRET above is +# freshly minted every run. Better Auth's jwt plugin stores a JWKS encrypted under that secret; +# a leftover jwks row from an earlier run (different secret) fails to decrypt, so every +# GET /api/auth/get-session 500s and guest logins silently bounce to the /sso-spn-login dead +# end. Deleting it makes Better Auth regenerate under the current secret. No-op on a fresh DB. +cd "$REPO_DIR" && DATABASE_URL="$DB_URL" node --input-type=module -e \ + 'import {neon} from "@neondatabase/serverless"; const sql=neon(process.env.DATABASE_URL); await sql.query("DELETE FROM jwks"); console.log("jwks cleared");' + +# Use --value (no stdin) + --force (idempotent overwrite) + --non-interactive. A plain +# `vercel env add … <<< value` for PREVIEW scope stalls on a "? Git branch?" prompt. +# An empty AGENT_APP_URL means Phase 4 never created the app. Setting it anyway +# succeeds, and the frontend then deploys pointing at nothing — the guest panel +# loads and simply cannot reach the agent, which looks like a frontend bug. +if [ -z "${AGENT_APP_URL:-}" ]; then + echo "✗ AGENT_APP_URL is empty — Phase 4 did not produce a running app." >&2 + echo " Fix Phase 4 before deploying the frontend, or it will point at nothing." >&2 + return 2>/dev/null || exit 1 +fi + +for SCOPE in preview production; do + add() { vercel env add "$1" "$SCOPE" --value "$2" --force --non-interactive --scope "$VERCEL_TEAM"; } + add DATABRICKS_AGENT_APP_URL "$AGENT_APP_URL" + add DATABASE_URL "$DB_URL" + add BETTER_AUTH_SECRET "$BETTER_AUTH_SECRET" + add ENCRYPTION_KEY "$ENCRYPTION_KEY" + add NEXT_PUBLIC_AGENT_ENABLED "true" + add GUEST_API_SECRET "$GUEST_API_SECRET" + add SPN_AUTH_DATABRICKS_ACCOUNTS_URL "https://accounts.cloud.databricks.com" + add SPN_AUTH_DATABRICKS_WORKSPACE_URL "$DATABRICKS_HOST" + # Guest Catalog Explorer allowlist (#20): only lists catalogs matching an allowed prefix + # (app default "firefly"). Set it to the catalog chosen in Phase 0 so guests can BROWSE + # the data provisioned there (the app's memory store lives in $UC_CATALOG too). + add GUEST_ALLOWED_CATALOG_PREFIXES "$UC_CATALOG" + # Production is the serving target and its origin is already known from 8a-2, so set + # it now — one deploy, no second pass. Preview is deliberately left unset: preview URLs + # are per-deployment, so pointing preview auth at the production origin is wrong. + [[ "$SCOPE" == "production" ]] && add BETTER_AUTH_URL "$APP_ORIGIN" +done +``` + +> **DO NOT set `NEXT_PUBLIC_BETTER_AUTH_URL`** — it is baked at build time and causes +> CORS failures on preview deployments. The auth client falls back to `window.location.origin`. +> +> **Omit `SPN_AUTH_OKTA_*` entirely** — the plugin is conditional; absent vars are skipped. +> +> **`GUEST_ALLOWED_CATALOG_PREFIXES` (security note, #20):** the guest Catalog Explorer +> only lists catalogs whose name starts with one of these (comma-separated, case-insensitive; +> default `firefly`). Setting it to include your `UC_CATALOG` lets guests **browse** that +> catalog's tree — still scoped by the guest SP's UC grants, and browse-only (it does not +> affect Genie, which queries UC directly). If `UC_CATALOG` is the shared `workspace` +> catalog, guests can browse the whole `workspace` tree — fine for a demo; narrow it (or use +> a dedicated `firefly_*` catalog) if that's too broad. + +#### Tier 2 — required only for admin Databricks OAuth login (not needed for guest path) + +```bash +# These vars power the "Login with Databricks" button for workspace admins. +# For a guest-only verification (Phase 9), set placeholder values to satisfy +# the build; the auth routes will 404 at runtime if a user tries admin login, +# but the guest flow is unaffected. +# +# auth-dynamic.ts passes these to genericOAuth as a plain config object, so +# placeholder values do not crash the Next.js build — they only fail at runtime +# when the admin login route is actually invoked. +# +# To enable admin login: replace placeholders with real values from a Databricks +# OAuth app registered at accounts.cloud.databricks.com → App connections. + +for SCOPE in preview production; do + add() { vercel env add "$1" "$SCOPE" --value "$2" --force --non-interactive --scope "$VERCEL_TEAM"; } + add DATABRICKS_U2M_CLIENT_ID "${DATABRICKS_U2M_CLIENT_ID:-placeholder}" + add DATABRICKS_U2M_CLIENT_SECRET "${DATABRICKS_U2M_CLIENT_SECRET:-placeholder}" + add DATABRICKS_ACCOUNT_ID "$DATABRICKS_ACCOUNT_ID" + add SPN_AUTH_DATABRICKS_ACCOUNT_ID "$DATABRICKS_ACCOUNT_ID" +done +``` + +### 8c. Disable Vercel preview protection (needed for guest API calls) + +```bash +# Vercel SSO protection is on by default for preview deployments. +# Without this, /api/guest/* returns 401 "Protected deployment". +vercel project protection disable "$VERCEL_PROJECT" --sso --scope "$VERCEL_TEAM" +``` + +### 8d. Deploy — single pass + +```bash +# Always --prod. A bare `vercel deploy` is production only on a project's FIRST deploy; +# on a re-run it produces a preview with a per-deployment host, which is how a +# discover-from-stdout flow silently sets BETTER_AUTH_URL to a dead origin (#19). +# Never `vercel alias set` to $VERCEL_PROJECT.vercel.app — that name may belong to +# another Vercel account, and the alias call hard-fails with "already in use". +vercel deploy --prod --scope "$VERCEL_TEAM" + +# The CLI's own success text points the wrong way here, and it is not wrong so much as +# misread-by-design: after --prod it prints a PER-DEPLOYMENT host labelled "Production" +# (firefly-genie-h5ef9wq49-.vercel.app) and offers "Promote to production" as the next +# step. Both suggest the deploy is not live yet at the origin you want. It is: the serving +# origin is the one Phase 8a-2 read from the domains API, and 8e re-checks it. Do NOT follow +# the promote hint or copy that host anywhere -- a run reported being sent toward a second, +# unnecessary deploy by it. + +# APP_ORIGIN comes from Phase 8b, so a fresh shell arrives without it and this used to +# store PREVIEW_URL="" — after which Phase 9's guest-login curls returned HTTP 000 with +# nothing to explain why. Recover it from state.env before trusting the shell. +: "${APP_ORIGIN:=$(read_secret APP_ORIGIN 2>/dev/null || true)}" +firefly_require APP_ORIGIN || return 2>/dev/null || exit 1 +PREVIEW_URL="$APP_ORIGIN" # Phase 9 guest-entry URL (historical var name) +store_secret APP_ORIGIN "$APP_ORIGIN" +store_secret PREVIEW_URL "$PREVIEW_URL" +# Minted in an earlier phase, so a fresh shell arrives without it: recover before +# storing, or this line is a save that saves nothing. +: "${GUEST_API_SECRET:=$(read_secret GUEST_API_SECRET 2>/dev/null || true)}" +store_secret GUEST_API_SECRET "$GUEST_API_SECRET" +``` + +### 8e. Verify production serves the origin `BETTER_AUTH_URL` points at + +```bash +# #19 reports success at every earlier step and only surfaces later as "Invalid token" +# on the guest login link, so assert the match rather than assuming it. +# +# Re-derive the Vercel context: these come from 8a, and in a new shell they are +# empty. The curl then sends "Authorization: Bearer " and this step reports that +# production does not serve $APP_ORIGIN when the deployment is in fact correct. +firefly_vercel_context "$REPO_DIR" + +SERVING=$(curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" \ + | python3 -c 'import json,sys; t=(json.load(sys.stdin).get("targets") or {}).get("production") or {}; print("\n".join(t.get("alias") or []))') +if [ -z "${APP_ORIGIN:-}" ]; then + # "Production does not serve " with no hostname reads as a domain mismatch when the + # deployment is in fact correct and aliased. The cause is a lost variable, so say that. + echo "APP_ORIGIN is empty, so this check cannot run — it is NOT a domain mismatch." + echo " Recover it with: APP_ORIGIN=\$(read_secret APP_ORIGIN) and re-run this block." + return 2>/dev/null || exit 1 +fi +grep -qxF "${APP_ORIGIN#https://}" <<<"$SERVING" \ + || { echo "Production does not serve $APP_ORIGIN"; exit 1; } +``` + +--- + +## Phase 9 — Verify + +### App running + +```bash +databricks apps get "$AGENT_APP_NAME" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); \ + print('app_status:', d['app_status']['state'])" +# Expected: app_status: RUNNING +``` + +### Enterprise network controls (check this BEFORE blaming the app) + +If your workspace restricts access by IP, the deployed frontend is a third party to +it: Vercel calls Databricks from its own egress addresses, which are not your +office or VPN ranges. Every data call then returns a bare `403`, and the UI shows +`Failed to load SQL warehouses — Databricks API error: Forbidden`. Nothing is +wrong with the deployment; the workspace is refusing the caller. + +```bash +# Does this workspace enforce an IP allowlist? Same shared helper as Phase 1b, so +# the two cannot disagree — which they did, in opposite directions. +ACL_STATUS="$(firefly_ip_allowlist_status "$DB_PROFILE")" +case "$ACL_STATUS" in + enabled:*) + cat </dev/null || true + vercel env add GUEST_API_SECRET production --value "$GUEST_API_SECRET" \ + --force --non-interactive --scope "$VERCEL_TEAM" + store_secret GUEST_API_SECRET "$GUEST_API_SECRET" + vercel deploy --prod --yes --scope "$VERCEL_TEAM" >/dev/null +fi +# Guest SP credentials (created in Phase 6b) +GUEST_SP_CLIENT_ID=$(read_secret GUEST_SP_CLIENT_ID) +GUEST_SP_SECRET=$(read_secret GUEST_SP_SECRET) + +# 1. Create a workspace record +WS=$(curl -s -X POST "$PREVIEW_URL/api/guest/workspaces" \ + -H "X-API-Key: $GUEST_API_SECRET" -H "Content-Type: application/json" \ + -d "{\"name\":\"Test\",\"workspaceUrl\":\"$DATABRICKS_HOST\"}") +WS_ID=$(echo "$WS" | python3 -c "import sys,json; print(json.load(sys.stdin)['workspace']['id'])") + +# 2. Register the guest SP (clientId = applicationId from Phase 6b) +SPN=$(curl -s -X POST "$PREVIEW_URL/api/guest/spns" \ + -H "X-API-Key: $GUEST_API_SECRET" -H "Content-Type: application/json" \ + -d "{\"name\":\"Test SPN\",\"clientId\":\"$GUEST_SP_CLIENT_ID\", \ + \"clientSecret\":\"$GUEST_SP_SECRET\",\"guestWorkspaceId\":\"$WS_ID\"}") +SPN_ID=$(echo "$SPN" | python3 -c "import sys,json; print(json.load(sys.stdin)['spn']['id'])") + +# 3. Create guest user → get login URL +GU=$(curl -s -X POST "$PREVIEW_URL/api/guest/users" \ + -H "X-API-Key: $GUEST_API_SECRET" -H "Content-Type: application/json" \ + -d "{\"orgName\":\"Test Org\",\"spnId\":\"$SPN_ID\"}") +echo "$GU" | python3 -c "import sys,json; print(json.load(sys.stdin)['guestUser']['loginUrl'])" +# Navigate to the loginUrl in a browser → agent panel should appear → ask a question. +``` + +### Memory round-trip + +```bash +# In the agent panel, turn 1: state a distinctive fact. +# Open New Chat (turn 2): ask the fact back. +# The agent should recall from /memories/... without being told again. +# Requires Phase 5 (setup_memory_store.py) to have run successfully. +``` + +### Deployment summary — REQUIRED final output + +End with a deployment-summary **table**, and make the **guest login URL the first row** — it is +the one thing the user needs to open the app, so it belongs at the top of the table, verbatim +and clickable, never buried below the resource rows. The login link is **one-time and expires +~10 minutes** after it is minted (the "Guest login" block above), so mint it as the LAST step. + +**Render every URL as a clickable markdown link** — `[]()` — so the user can click +straight from the summary (substitute the real value into *both* the label and the target): + +The row below the link used to read ``Expired or already used? | `bash scripts/new-guest-link.sh --open` ``, +which is an **operator** command sitting in the one table an operator forwards to the person +who will actually use the app. That person has no clone of this repo, no +`.firefly-bootstrap/state.env`, and no business holding `GUEST_API_SECRET` or the guest +SP's client id and secret — all of which the script requires and refuses to run without. +So the instruction could not work for them, and the only way to make it work would be to +hand over credentials. Tell the recipient the one thing they can act on: ask you. + +| Resource | Value | +|---|---| +| **▶ Guest login URL** (one-time, ~10 min) | **[``]()** | +| Expired or already used? | Ask whoever sent you this link for a fresh one — it takes them a few seconds | +| Frontend (preview) | [``]() | +| Agent app | `` — RUNNING · [``]() | +| Lakebase | `` | +| Neon project | `` | +| UC memory store | `..firefly_managed_memory` | +| Agent SP / Guest SP | `` | + +**For you, the operator — not for the table above.** When the recipient tells you the link +is expired or already used, mint a fresh one and send it: + +```bash +bash scripts/new-guest-link.sh --open +``` + +This has to run from your `$REPO_DIR`, because it reads `PREVIEW_URL`, +`GUEST_API_SECRET`, `GUEST_SP_CLIENT_ID` and `GUEST_SP_SECRET` out of +`$REPO_DIR/.firefly-bootstrap/state.env` and exits naming whichever one is missing. +Those are credentials: keep them on your machine and send the recipient only the URL. + +It replays only the three "Guest login" POSTs above, reading `PREVIEW_URL` / +`GUEST_API_SECRET` / guest-SP creds from `$REPO_DIR/.firefly-bootstrap/state.env`, and +prints a new URL in about two seconds. Prefer `--open` — copy-pasting into a tab that +already has the app loaded can consume the link before you read it. + +--- + +## Next steps — no UC data + +Apply this section **only if `$UC_CATALOG.$UC_SCHEMA` is still empty after Phase 6c** — +i.e. `SEED_STATUS` was `declined` (you answered `no` to `SEED_SAMPLE_DATA`), +`source-denied` (no `SELECT` on the sample source), or `source-error`. + +> `SEED_STATUS=not-this-phase` is **not** one of them. Seeding belongs to Phase 3f; +> Phase 6c only grants, so it reports that seeding was not its job rather than +> claiming you declined it. Read Phase 3f's own `seed=` line for the real outcome. +Bootstrap can complete successfully — app, guest login, and memory may all work — +but Genie will not answer data questions until queryable tables exist in a schema +the agent SP can read. + +> If `SEED_STATUS` was `source-not-ready` or `source-empty`, **do not follow this +> section** — nothing is wrong with your entitlements. The samples catalog had not +> finished provisioning. Re-run Phase 6c's script (below) and it will seed. + +To seed after the fact, re-run just Phase 6c's script; it is idempotent: + +```bash +cd "$REPO_DIR" && bash scripts/genie-data-setup.sh \ + --catalog "$UC_CATALOG" --schema "$UC_SCHEMA" --profile "$DB_PROFILE" \ + --seed yes --create-space yes \ + --agent-sp "$SP_CLIENT_ID" \ + --guest-sp "${GUEST_SP_CLIENT_ID:-}" \ + --grant-guest "${GRANT_GUEST_SPACE_ACCESS:-yes}" +``` + +**Recommended next steps for the user:** + +1. **Choose a data source** — ingest production/analytics data, copy a slice from + the workspace `samples` catalog, or use your team's standard demo dataset. +2. **Place tables in a granted schema** — the agent SP needs `USE CATALOG`, + `USE SCHEMA`, and `SELECT` on whatever schema holds the data (Phase 6). + Grant the guest SP the same if guest users will ask data questions (Phase 6b). +3. **Confirm warehouse access** — a running SQL warehouse with `CAN_USE` for the + agent SP (Phase 6) is required for Genie to execute queries. +4. **Re-check** — re-run the Phase 6c `tables list` / `SHOW TABLES` check. +5. **Verify in the agent panel** — ask a question that requires real data (not just + "does the panel load?"). Empty or evasive answers after data is loaded usually + mean missing grants on the new schema/tables, not a broken deploy. + +Seeding is offered as a Phase 0 blocking ask (`SEED_SAMPLE_DATA`), so it is always +the user's decision. Never seed a schema that already holds tables you did not +create, and never overwrite an existing table — Phase 6c reports +`already-populated` and leaves such a schema alone. + +--- + +## Dependency map + +| Concern | Tooling | Auth method | +|---|---|---| +| Databricks provisioning | `databricks` CLI + REST | `databricks auth login` (U2M OAuth, browser) | +| UC memory store | Python SDK / REST | same CLI profile | +| Neon DB | `neonctl` CLI | `neonctl auth` (browser OAuth) | +| Neon runtime | `drizzle-orm`, `@neondatabase/serverless` | `DATABASE_URL` (Postgres) | +| Vercel deploy | `vercel` CLI | `vercel login` (browser OAuth) | +| Vercel runtime | none (host only) | — | +| GitHub / submodule | `git`, optionally `gh` | `gh auth login` or SSH | + +--- + +## Report bootstrap issues + +If anything fails, behaves unexpectedly, or required a workaround during this +runbook, **construct and file a GitHub issue on the user's behalf** so the +bootstrap can be improved. Do not tell the user to "go file an issue" and stop — +you draft it, they confirm, you submit. + +### When to file + +- A phase command failed or needed a manual fix not documented here +- Phase 9 verification failed (or passed only after an undocumented workaround) +- Phase 6c was empty and Genie could not answer data questions after the user loaded data +- Any `[ASK — REQUIRED, BLOCKING]` value had to be overridden because the runbook was wrong + +### What you do + +1. **Collect during the run** — note phase, command, stderr/stdout (redact secrets), + non-default inputs (`$UC_CATALOG`, `$AGENT_APP_NAME`, etc.), and expected vs actual. +2. **Draft the issue** — fill the template below from the session. Omit sections that + don't apply; never paste tokens, client secrets, or URLs with embedded credentials. +3. **Show the user** — present the title and body; ask for a one-line confirmation or + edits. +4. **File it** — write the body to a temp file, then create the issue from `$REPO_DIR` + (or this repo root) with `gh` authenticated: + +**Title:** `bootstrap(): ` — substitute the branch you are on + +**Body template:** + +```markdown +## Summary + + +## Phase / step + + +## Commands run + + +## Error output + + +## Environment +- Bootstrap branch: `` (do NOT hardcode a + branch name — after genie-agent merges, readers will be on the default branch and an + issue that claims otherwise points maintainers at code the reporter never ran) +- DATABRICKS_HOST: +- UC_CATALOG / UC_SCHEMA: +- AGENT_APP_NAME: +- Fresh workspace: yes / no + +## Expected vs actual +- Expected: … +- Actual: … + +## Workaround used (if any) +… + +## Suggested runbook fix +… +``` + +```bash +ISSUE_BODY=$(mktemp) +cat > "$ISSUE_BODY" <<'EOF' + +EOF + +gh issue create \ + --repo databrickslabs/firefly \ + --title "bootstrap($(git rev-parse --abbrev-ref HEAD)): " \ + --body-file "$ISSUE_BODY" + +rm -f "$ISSUE_BODY" +``` + +5. **Return the issue URL** to the user. + +Do not open Databricks support tickets for bootstrap/runbook problems — use the +repo issue tracker instead (see README project support note). diff --git a/CLAUDE.md b/CLAUDE.md index 7e3ec1a..e844721 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,4 +34,58 @@ - Trust the mutation's `onSettled` + `refetchOnWindowFocus` for consistency - Do not delete folders with rm -rf -- only delete files! \ No newline at end of file +- only delete files! + +## Agent panel (managed-memory agent) + +- The Agent panel embeds a Databricks App (Genie + memory) via a Vercel-native + reverse proxy at `src/app/api/agent-proxy/[[...path]]/route.ts` — NOT the Go + proxy. The route mints the user/guest SPN token + (`src/lib/databricks-spn-authtoken.ts`) and forwards HTTP + SSE to + `DATABRICKS_AGENT_APP_URL`. +- This proxy route intentionally sets `Cache-Control: no-store` on the rewritten + HTML document (it injects `` + forced light theme per request). This + is the one place the "never use custom Cache-Control" rule does not apply, + because the app's ETag never changes and a 304 would serve stale injected HTML. +- Panel UI: `src/components/agent/agent-panel.tsx` (gated by + `NEXT_PUBLIC_AGENT_ENABLED`), store in `src/stores/agent-panel-store.ts`. +- The agent app source is the `vendor/app-templates` submodule + `agent/` overlay, + merged by `scripts/assemble_agent.sh` into the gitignored `agent-build/`. Do not + hand-edit `vendor/**` (pristine submodule); put deltas in `agent/`. + +## Bootstrap runbook (ENV-0, #69) + +Applies to `BOOTSTRAP.md`, `README.md`, `scripts/bootstrap.sh`, `scripts/lib/**`. +**ALWAYS run `bash scripts/check-runbook-invariants.sh` after touching any of them.** + +- **NEVER use `corepack enable` / `corepack prepare` to install pnpm.** corepack ignores + the npm registry setting and fetches from `registry.npmjs.org`, so it dies wherever + public npm is blocked. **ALWAYS use `npm install -g pnpm@`** (npm honors + the user's configured registry). The pin is required — pnpm's `latest` tag has shipped a + 12.x alpha that fails with `ERR_PNPM_IGNORED_BUILDS`. +- **NEVER put corporate-network setup in `BOOTSTRAP.md` as prose only.** Phase 0 must have + runnable commands. Both the runbook and `bootstrap.sh` source the single implementation + in `scripts/lib/corp-network.sh` — **NEVER duplicate that logic** into either one. +- **ALWAYS keep `README.md` and `BOOTSTRAP.md` consistent** about the pnpm install method. +- **NEVER use `npm ping` to test registry reachability** — corporate mirrors 404 `/-/ping` + while serving packages normally. Probe the actual operation (`npm view pnpm@`). +- **NEVER bypass a blocked registry** via `--registry https://registry.npmjs.org`, disabling + TLS, or `/etc/hosts` edits. Bridge the user's own mirror; never hardcode one. +- **NEVER let `BOOTSTRAP.md` call a helper it does not source.** Every helper lives in a + `scripts/lib/*.sh` that the runbook sources in Phase 0a. It called `store_secret` / + `read_secret` with no definition anywhere in the file, and `bootstrap.sh`'s version had + an incompatible signature, so copying it across did not help either. +- **ALWAYS keep runnable blocks bash- AND zsh-parseable** (zsh is the macOS default; macOS + bash is 3.2). No `${!var}`, no `declare -F` as a function test, no `mapfile`. +- **NEVER ship a block that only looks executable** — `{ # comment }` and `{ : ; }` were + the Phase 1b/1e CLI installers, and installed nothing. +- **NEVER `grep` structured output.** The `sync.exclude` comment names `pyproject.toml`, so + a bare grep reports it excluded. Use `check_sync_exclude_rules` / the shared parsers. +- **NEVER trust a `create` response for an id you then act on.** `neonctl projects create` + succeeds server-side before the parse fails, orphaning a project. Create, then look up + by name, then fail closed on empty. + +This already regressed once: the guard was deleted (`e91322d`), then a docs-sync commit +(`99f2cc2`) reintroduced corepack. Keep the rationale next to the rule. That same commit +also caused three of the defects above, by syncing prose while dropping the code behind +it — edit the shared library, never re-sync the two files by hand. \ No newline at end of file diff --git a/Makefile b/Makefile index 0b97e69..3adcb90 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help db-generate db-push db-migrate db-studio db-drop db-check db-up db-pull auth-generate auth-migrate +.PHONY: help check db-generate db-push db-migrate db-studio db-drop db-check db-up db-pull auth-generate auth-migrate # Load environment variables from .env.local -include .env.local @@ -39,3 +39,18 @@ auth-generate: ## Generate Better Auth schema (output to auth-schema.ts for refe auth-setup: ## Setup auth tables (uses Drizzle push) @echo "Pushing Better Auth schema to database using Drizzle..." pnpm drizzle-kit push + +# GitHub Actions is disabled on this repository ({"enabled": false} on +# /actions/permissions), so the runbook invariants and the TypeScript build are enforced by +# nothing automatic -- they hold only when somebody runs them. Repo-local git hooks cannot +# help either: core.hooksPath points at ~/.databricks/githooks globally, and that directory +# is not ours to edit. So the least-bad option is to make the check trivial to invoke and +# name it in the contributing path. Run this before pushing anything that touches +# BOOTSTRAP.md, scripts/, or src/. +check: ## Run the runbook invariants, agent unit tests, and typecheck (CI is disabled; this is the only gate) + @bash scripts/check-runbook-invariants.sh + @# The invariants read source; these EXERCISE it. Both are needed: static checks could not + @# see that the two Genie backends expose different tool names, which shipped a broken + @# default past nine green end-to-end runs. + @cd agent && python3 -m unittest discover -s tests -p 'test_*.py' -q && echo "agent tests: clean" + @npx --yes tsc --noEmit -p tsconfig.json && echo "typecheck: clean" diff --git a/README.md b/README.md index b98ed67..f9b1f66 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,32 @@ # FireFly Analytics - Databricks Custom Frontend +[![Open in Cursor](https://img.shields.io/badge/Open%20in-Cursor-black?logo=cursor)](https://cursor.com/link/prompt?text=Clone%20https%3A%2F%2Fgithub.com%2Fdatabrickslabs%2Ffirefly%20into%20this%20workspace%20and%20work%20through%20BOOTSTRAP.md%20top%20to%20bottom%2C%20running%20the%20Phase%200a%20corporate-network%20setup%20commands%20before%20any%20Phase%201%20command.%0A%0A%5BASK%20-%20REQUIRED%2C%20BLOCKING%5D%20FIREFLY_BRANCH%20-%20which%20branch%20to%20clone.%20Default%3A%20main.%20Before%20going%20further%2C%20confirm%20the%20branch%20you%20cloned%20actually%20contains%20BOOTSTRAP.md%3B%20if%20it%20does%20not%2C%20say%20so%20and%20ask%20again%20rather%20than%20continuing%2C%20because%20the%20runbook%20is%20not%20on%20every%20branch%20and%20a%20missing%20BOOTSTRAP.md%20three%20phases%20later%20reads%20as%20a%20broken%20repo.) [![Start here: BOOTSTRAP.md](https://img.shields.io/badge/Start%20here-BOOTSTRAP.md-black?logo=markdown)](./BOOTSTRAP.md) A Next.js application that provides a customized frontend for Databricks with multiple authentication strategies and embedded Databricks apps. +## Quick Start (AI-assisted) + +For a fully automated, interactive-auth bootstrap — no manual token wrangling — use: + +| File | Purpose | +|---|---| +| [`BOOTSTRAP.md`](./BOOTSTRAP.md) | Harness-agnostic runbook; an AI agent works through it top-to-bottom, prompting for each value | +| [`scripts/bootstrap.sh`](./scripts/bootstrap.sh) | Executable version of the same runbook with `--dry-run` and `--stop-after ` flags | + +```bash +# Dry run — see every command without executing anything +bash scripts/bootstrap.sh --dry-run + +# Full interactive run (opens browser for each auth step) +bash scripts/bootstrap.sh +``` + +The runbook covers Phases 0–9 end-to-end (with sub-phases 3f, 6b and 6c): Databricks +provisioning, sample-data seeding and Genie space creation, Neon DB, Vercel deploy, guest +service principal, and end-to-end verification. Manual setup instructions follow below. + +--- + ## Table of Contents - [Prerequisites](#prerequisites) @@ -10,6 +34,7 @@ A Next.js application that provides a customized frontend for Databricks with mu - [Database Setup](#database-setup) - [Databricks OAuth Configuration](#databricks-oauth-configuration) - [Go Proxy Setup (VSCode Editor)](#go-proxy-setup-vscode-editor) +- [Agent Panel (Managed-Memory Agent)](#agent-panel-managed-memory-agent) - [Local Development](#local-development) - [Deployment to Vercel](#deployment-to-vercel) - [Architecture](#architecture) @@ -22,6 +47,25 @@ A Next.js application that provides a customized frontend for Databricks with mu - Go 1.21+ (for the proxy server) - Vercel account (for deployment) +> **Installing pnpm — use npm, not corepack (ENV-0).** Node ships pnpm via `corepack`, but +> `corepack enable` / `corepack prepare` fetch the pnpm release from `registry.npmjs.org` +> and ignore your configured npm registry, so they fail on corporate networks that block or +> blackhole public npm (`ECONNREFUSED` / `ETIMEDOUT` / `503`, and the `pnpm` shim never +> works). Install from your approved registry instead, pinning the version: +> +> ```bash +> corepack disable >/dev/null 2>&1 || true # an enabled shim shadows the install below +> npm install -g pnpm@10.34.5 # uses your configured (approved) npm registry +> pnpm --version # must print 10.34.5 +> ``` +> +> The pin matters: pnpm's `latest` dist-tag has shipped a 12.x alpha that ignores +> `onlyBuiltDependencies` and fails with `ERR_PNPM_IGNORED_BUILDS`. This is the same step as +> [`BOOTSTRAP.md`](./BOOTSTRAP.md) Phase 1a, and it is enforced by +> `scripts/check-runbook-invariants.sh`. On macOS, `brew install pnpm` is another +> approved-CDN option. Do **not** work around a block with +> `--registry https://registry.npmjs.org` or by disabling TLS. + ## Environment Setup ### 1. Copy Environment Variables @@ -204,6 +248,311 @@ For production, deploy the Go proxy to: Update `NEXT_PUBLIC_PROXY_URL` in your environment to point to the deployed proxy. +## Agent Panel (Managed-Memory Agent) + +The optional **Agent panel** is a slide-out chat assistant (Genie + long-term +memory) available in the SSO-SPN organization view. It embeds a Databricks App +built from the [`databricks/app-templates`](https://github.com/databricks/app-templates) +`agent-openai-agents-sdk` template, vendored here as a git submodule under +`vendor/app-templates`. + +### How the agent uses Genie + +The agent answers data questions over **Genie MCP**, authenticating with the agent App's +service principal. The `ask_genie` tool (`agent/agent_server/genie_tools.py`) calls +`genie_ask` and polls `genie_poll_response` until completion. + +**The endpoint is resolved in exactly one place** — `genie_mcp_path()` in +`agent/agent_server/genie_mcp.py` — and both `agent.py` and `genie_tools.py` call it. They used +to disagree: `genie_tools.py` held its own `GENIE_MCP_PATH` constant and POSTed to workspace-wide +Genie while `agent.py` correctly connected the space-scoped server, so a deployment configured +for a curated space answered from the wrong backend while every probe reported `mode=space`. Do +not reintroduce a second path. + +The agent is scoped to a **Genie space** named by `GENIE_SPACE_ID`, reached at +`/api/2.0/mcp/genie/`. Answers come from that space's curated tables, joins, and +instructions. `BOOTSTRAP.md` **Phase 3f** creates the space (or adopts space ids you supplied at +Phase 0) *before* Phase 4 deploys, so the app is born in its final mode; **Phase 6c** then grants +the agent and guest service principals on it. + +A Genie space is also the only object a guest service principal can be granted `CAN_RUN` on, and +guests have no workspace access — so the space is what makes the guest flow work at all. + +Genie is configured at the **agent App layer** in `agent/databricks.yml` (not the frontend +`.env.local`): + +| Env var | Purpose | +| --- | --- | +| `GENIE_MCP_MODE` | `space` — the default, and the supported configuration. Bundle variable `genie_mcp_mode` | +| `GENIE_SPACE_ID` | Required. Bundle variable `genie_space_id`, default **`none`** — a placeholder, not an id. It cannot default to empty: the bundle would render `{"name": "GENIE_SPACE_ID"}` with no `value` and the Apps API rejects the deploy ("Must specify environment variable source using either value or valueFrom"). `genie_mcp_path()` raises `ValueError` on an unset id rather than silently falling back to a different backend | +| `DATABRICKS_HOST`, `DATABRICKS_WORKSPACE_ID` | **Auto-injected** by the Databricks Apps runtime — never set in the bundle. Used to identify the workspace; **no attribution link is built from them** (see below) | + +The panel shows plain-text "Powered by Genie" attribution with **no link**. There was one, and it +was wrong in two ways at once: the audience for this panel is guest users who have no Databricks +workspace access, so the link led somewhere they could not open; and it named a backend that +never saw the question. `GENIE_ONE_URL` no longer exists. + +Pointing an existing deployment at a different space means passing the id: + +```bash +databricks bundle deploy -t dev --profile "$DB_PROFILE" \ + --var catalog="$UC_CATALOG" --var schema="$UC_SCHEMA" \ + --var genie_space_id= +``` + +The `GENIE_INSTRUCTIONS` prompt (composed onto the agent's memory instructions in +`agent/agent_server/agent.py`) forces Genie-first behavior for any question about tables, +catalogs, dashboards, or "my data". + +### Grant the agent's service principal access to your data + +The agent queries Genie Agent as the **agent App's service principal** (see above), not +the signed-in user. Genie only returns Unity Catalog data that this service principal can +access, so on a workspace where the SP has no grants, data questions come back empty even +though the agent and Genie are otherwise working. Grant the app's service principal access +to the catalogs/schemas you want it to answer over: + +- `USE CATALOG` on the catalog and `USE SCHEMA` on the schema, +- `SELECT` on the tables (or on the schema), and +- access to a SQL warehouse Genie can execute against. + +> **A fresh workspace has no data and no running warehouse — Genie answers "empty" until both +> exist.** If you bootstrap with `BOOTSTRAP.md`, Phase 3f handles this: it resolves a SQL +> warehouse and, when `SEED_SAMPLE_DATA=yes` (the default) and the target schema is **empty**, +> copies a slice of `samples.wanderbricks` into `$UC_CATALOG.$UC_SCHEMA`. It never overwrites an +> existing table, so pointing it at a populated schema is safe and does nothing. +> +> Setting up by hand instead, or answering `no`: start a (serverless) SQL warehouse and create at +> least one UC schema with a few tables, then apply the grants above to the app's SP. Without +> data *and* a warehouse the Agent panel looks broken even though the agent, Genie, and memory +> are all working. + + + +### How it differs from the Go proxy + +Unlike the VSCode/notebook editors (which use the Go proxy), the agent is embedded +through a **Vercel-native reverse proxy** — a Next.js route at +`src/app/api/agent-proxy/[[...path]]/route.ts`. That route: + +- resolves the current user's (or guest's) mapped **service principal** and mints + a Databricks bearer token via M2M OAuth (`src/lib/databricks-spn-authtoken.ts`), + so guests never hit the Databricks OAuth wall; +- forwards HTTP + SSE (streaming chat) to `DATABRICKS_AGENT_APP_URL`, injecting the + bearer and relaxing frame headers for same-origin embedding; +- rewrites the app's HTML (`` + forced light theme) so relative assets + resolve under `/api/agent-proxy` and the chat UI matches Firefly's light UI. + +**No Go proxy or Cloud Run is required for the agent.** The Go proxy is only for the +code/notebook editors. + +### Enable it + +1. Set the environment variables (see `.env.example`): + ```env + NEXT_PUBLIC_AGENT_ENABLED=true + DATABRICKS_AGENT_APP_URL=https://your-agent-app.databricksapps.com + ``` +2. Ensure SPN auth is configured (`SPN_AUTH_*` and `FIREFLY_SPN_*`), since the proxy + reuses the same SSO→SPN mapping used elsewhere. +3. Make sure the runtime prerequisites are met for the signed-in (or guest) user, + or `/api/agent-proxy` returns `400`/`401`: + - the user's **active organization** has a `workspaceUrl` set, and + - there is an **SPN mapping** (`userSpns` row) for that user's email in that org + (the proxy mints the workspace bearer from this mapping). + +### Provision the agent's Databricks resources (prerequisite) + +`agent/databricks.yml` binds the app to three workspace-specific resources that +**must exist before `databricks bundle deploy`**: + +- an **MLflow experiment** (agent tracing), +- a **Lakebase (Databricks Postgres)** autoscaling instance — backs the frontend's + chat persistence (`ai_chatbot` schema) and the OpenAI Agents SDK session store. + This is **not** the durable-memory store (see below); the two are often confused. +- a **UC volume** — default `workspace.default.firefly_wheels` (the bundle grants the + app `READ_VOLUME` on it). The catalog/schema/name are bundle variables + (`catalog`/`schema`/`wheels_volume_name`), so on a workspace whose catalog is + `main` you pass `--var catalog=main` instead of editing the YAML. + +A fourth resource — the **UC memory store** for durable cross-session memory — is +created *after* the app exists (it needs the app SP for its grant), so it is not in +this pre-deploy list; see "Deploy → run → enable memory" below. It is a distinct UC +securable (`DATABRICKS_MEMORY_STORE`, default `workspace.default.firefly_managed_memory`), +**not** the Lakebase instance above. + +For a fresh workspace, create them as follows. + +**1. Experiment + Lakebase (one command).** First fetch the submodule and assemble +`agent-build/` (both are prerequisites of this step, even though the full build is +documented later): + +```bash +git submodule update --init # empty on a fresh clone; assemble fails without it +bash scripts/assemble_agent.sh +cd agent-build + +# Creates the MLflow experiment AND a new autoscaling Lakebase project+branch in +# one pass, and writes their IDs into agent-build/databricks.yml + .env. +# --python 3.12 is REQUIRED: the dep tree (whenever/PyO3) caps at 3.13, so a bare +# `uv run` that picks 3.14 fails to build. must be new/unique +# (no reuse — an existing name errors). Use lowercase alphanumerics + hyphens. +uv run --python 3.12 quickstart --profile --lakebase-create-new +``` + +> **Do not re-run `assemble_agent.sh` after `quickstart` in a single deploy pass.** +> Assemble does `rm -rf agent-build`, which wipes the `.env` (Lakebase creds) and +> the resource IDs quickstart just wrote — and the SP-grant step below reads that +> `.env`. Order that avoids the trap: assemble once → quickstart → (mirror IDs to the +> overlay) → vendor → deploy. If you must re-assemble, first copy the quickstart IDs +> into the tracked overlay `agent/databricks.yml` (next note). + +**2. Wheels volume.** Create the UC volume the bundle grants read on (matches the +`catalog`/`schema`/`wheels_volume_name` bundle-variable defaults): + +```bash +databricks volumes create workspace default firefly_wheels MANAGED -p +``` + +> **The volume must _exist_ for the resource binding; the build reads wheels from +> synced source, not the volume.** UC volumes are **not mounted during the Apps +> build**, so dependencies install from the pre-vendored `vendor-wheels/` directory +> (synced with the source) via `UV_FIND_LINKS` — see "Vendor the build wheels" +> below. An empty volume satisfies the `READ_VOLUME` grant. + +> **Copy only the quickstart-managed IDs into the tracked overlay.** `quickstart` +> writes the new `experiment_id` and `postgres` (`branch`/`database`) into +> **`agent-build/databricks.yml`**, which is gitignored and rebuilt from scratch by +> `scripts/assemble_agent.sh`. Copy those two resource blocks into the tracked overlay +> **`agent/databricks.yml`**, or they are lost on the next assemble. You do **not** +> edit `DATABRICKS_HOST` or `DATABRICKS_WORKSPACE_ID` any more — they are +> auto-injected at runtime (see "Derive — don't store"), and `GENIE_ONE_URL` no +> longer exists. And you do +> **not** edit the memory-store / wheels-volume namespace unless overriding the +> `workspace.default` default via `--var`. + +### Build & deploy the agent app + +The deployable app is assembled from the pristine submodule plus the local overlay +in `agent/` (agent code, chat-UI patches, bundle config): + +```bash +# Fetch the submodule (first time only) — pulls vendor/app-templates +git submodule update --init + +# Merge vendor submodule + agent/ overlay into ./agent-build (gitignored). +# The script also runs a local-only `git init` on agent-build so the bundle +# picks it up (the parent repo gitignores agent-build/, and `databricks bundle` +# respects the enclosing repo's ignore rules — without its own git boundary the +# sync would find "no files to sync" and deploy an empty app). +bash scripts/assemble_agent.sh + +cd agent-build + +# Vendor the build wheels (once, before deploy). Pre-fetches linux/cp311 wheels +# into vendor-wheels/ so the Apps build installs offline via UV_FIND_LINKS and +# never depends on the build container's PyPI egress (an unsanctioned .dev proxy, a lagging +# .cloud mirror, and no offline fallback are what made online installs flaky and +# blow past the 10-min startup limit). Requires local `uv` + `pip`. +bash scripts/vendor_wheels.sh + +# Validate first. A healthy bundle reports 0 "no files to sync" warnings; if you +# see that warning, agent-build is missing its git boundary (re-run the script). +databricks bundle validate -p + +# Deploy: one command does deploy -> run -> enable-memory (see below). +# On a non-default catalog, pass it through: ... --var catalog=main +bash scripts/deploy_agent.sh +``` + +> **On a non-default namespace** (e.g. a workspace whose catalog is `main`), append +> `--var catalog=main` (and/or `--var schema=...`) to `deploy_agent.sh` (it forwards +> them to every `bundle` command) so the wheels-volume binding and +> `DATABRICKS_MEMORY_STORE` line up. You can also set `BUNDLE_VAR_catalog=main`. + +Then point `DATABRICKS_AGENT_APP_URL` at the deployed app URL. + +#### Deploy → run → enable memory + +`scripts/deploy_agent.sh` runs three steps; here is what it does by hand, and why: + +```bash +# From agent-build/: +databricks bundle deploy -p # create app + SP, upload source +databricks bundle run agent_openai_agents_sdk -p # build + start + +# Resolve the SP (exists after create) and enable durable memory: +SP=$(databricks apps get -p \ + --output json | jq -r '.service_principal_client_id') +uv run --python 3.12 python scripts/setup_memory_store.py "$SP" --profile +``` + +**No manual Lakebase grant is required for the app to come up.** The frontend *does* +run Drizzle DB migrations during its first build, but the bundle binds the +Postgres/Lakebase resource with `CAN_CONNECT_AND_CREATE` (`agent/databricks.yml`), +applied at `bundle deploy`. That binding auto-provisions the app SP's Postgres login +role, so the migrations connect as the SP, `CREATE` their own schema/tables (the SP +becomes **owner** → full DML), and succeed. A no-grant deploy comes up `RUNNING` with +`ai_chatbot.{Chat,Message,Vote}` present and owned by the SP — verified end-to-end. +(`scripts/grant_lakebase_permissions.py` exists for a *different* topology — a +Lakebase-backed session/checkpoint store — and is **not** needed here; this app's +managed memory is a UC memory store, below.) + +**Durable memory is a Unity Catalog memory store — you MUST create it and grant the +SP.** The `save_memory`/`get_memory` tools call the UC memory-store API +(`/api/2.1/unity-catalog/memory-stores//entries`), not Lakebase. +That store is **not** created by quickstart or the bundle, and the app SP has no +rights on it by default, so a fresh deploy answers but silently fails to persist +("memory store not found", then "does not have READ/WRITE MEMORY STORE"). The last +line above fixes both — it creates `DATABRICKS_MEMORY_STORE` (default +`workspace.default.firefly_managed_memory`) and grants the SP +`READ_MEMORY_STORE`+`WRITE_MEMORY_STORE`. It is idempotent. There is no `databricks` +CLI for memory stores (CLI v0.298.0); the script uses the REST API via the SDK. + +**Operational notes** + +- **First request returns HTTP 503.** After `bundle run`, the container runs + `uv sync`, `npm install`, and the Vite build for the chat UI before it serves + traffic (can take a few minutes) — the 503 is normal, not a failure. Note the + app is behind Databricks app auth, so an unauthenticated request 302-redirects + to OIDC rather than returning `200`. +- **Deploy state ≠ health; read the *live* app state.** The platform marks the + deployment `SUCCEEDED` the moment the container command *starts* — before the + build finishes or the port binds (observed: `SUCCEEDED` ~45s before the backend + bound `:8000`). So `active_deployment.status.state` is always green and is not a + health signal. Judge health from the **live `app_status.state`** (`RUNNING`) or + the **runtime logs** — wait for `Both frontend and backend are ready!`. + `start_app.py` builds the frontend *before* the backend binds the port, so a + frontend/migration failure takes the whole container down (honest `app_status`) + rather than leaving the backend serving behind a broken UI. +- **Two different Python versions are in play.** `uv run quickstart` runs locally + and needs **3.12** (its dependency tree uses PyO3 capped at 3.13, so `uv` picking + 3.14 fails — pass `--python 3.12`). The **Apps runtime is 3.11 (cp311)**, not 3.12 + as older docs claimed, so `scripts/vendor_wheels.sh` pins `PY=3.11` and downloads + cp311 wheels. If you regenerate `uv.lock`, make sure the configured PyPI proxy is + `pypi-proxy.cloud.databricks.com` (the `.dev` host is unsanctioned and stamping it into + the lock is what caused the original install timeouts). `bootstrap.sh` Phase 4 + fails if `agent-build/uv.lock` or `databricks-apps/guest-manager/uv.lock` still + contain `.dev`. From the workspace root you can also run + `bash scripts/check-no-dev-pypi-proxy.sh`. Bootstrap also refuses to bridge + pip → uv when the index is `pypi-proxy.dev.databricks.com`. +- **Verify the overlay applied** before deploying (quick sanity check): + `agent-build/agent_server/agent.py` contains `GENIE_INSTRUCTIONS`, + `e2e-chatbot-app-next/client/vite.config.ts` has `base: "./"`, and + `client/src/main.tsx` contains `__FIREFLY_PROXY_BASENAME__`. +- **Genie/memory config lives in the bundle**, not the frontend — see + `agent/databricks.yml` (`GENIE_MCP_MODE`, and `DATABRICKS_MEMORY_STORE` built + from the `catalog`/`schema` bundle variables). `DATABRICKS_HOST` and + `DATABRICKS_WORKSPACE_ID` are intentionally absent (auto-injected at runtime); + `GENIE_ONE_URL` was removed along with the attribution link. + +Re-run `bash scripts/assemble_agent.sh` after any change under `agent/` (overlay) +or a submodule bump; it rebuilds `agent-build/` from scratch each time. + ## Local Development ### 1. Start the Development Server @@ -250,7 +599,7 @@ pnpm format ### 1. Install Vercel CLI (Optional) ```bash -pnpm install -g vercel +npm install -g vercel@56.3.1 # pinned; matches BOOTSTRAP.md Phase 1d ``` ### 2. Connect to Vercel @@ -276,22 +625,40 @@ Navigate to your project in the Vercel dashboard: DATABASE_URL BETTER_AUTH_SECRET BETTER_AUTH_URL (use your production URL) - NEXT_PUBLIC_BETTER_AUTH_URL (use your production URL) + # NEXT_PUBLIC_BETTER_AUTH_URL — omit for preview/dynamic URLs; see .env.example ENCRYPTION_KEY NEXT_PUBLIC_PROXY_URL DATABRICKS_APP_URL + # Agent panel (optional; see "Agent Panel" below) + NEXT_PUBLIC_AGENT_ENABLED + DATABRICKS_AGENT_APP_URL + # The Agent panel also needs the SSO->SPN mapping vars below (the proxy mints + # the signed-in/guest user's mapped SPN token — see "Enable it"). Omit these + # and /api/agent-proxy returns 400/401 at runtime: + SPN_AUTH_DATABRICKS_ACCOUNT_ID + SPN_AUTH_DATABRICKS_ACCOUNTS_URL + SPN_AUTH_DATABRICKS_WORKSPACE_URL + SPN_AUTH_OKTA_CLIENT_ID + SPN_AUTH_OKTA_CLIENT_SECRET + SPN_AUTH_OKTA_ISSUER + FIREFLY_SPN_GLOBAL_ADMIN_CLIENT_ID + FIREFLY_SPN_GLOBAL_ADMIN_CLIENT_SECRET + FIREFLY_WORKSPACE_SPN_SECRET_SCOPE_NAME + # ...and, to create/manage guest users for the panel: + GUEST_API_SECRET ``` **Important**: For production deployment, you must use your actual domain name for certain URLs: -- **BETTER_AUTH_URL**: Use your production domain (e.g., `https://www.firefly-analytics.com`) -- **NEXT_PUBLIC_BETTER_AUTH_URL**: Use your production domain (e.g., `https://www.firefly-analytics.com`) +- **BETTER_AUTH_URL**: Use your production domain (e.g., `https://www.firefly-analytics.com`). This is server-side only and must match the Databricks OAuth redirect URI. +- **NEXT_PUBLIC_BETTER_AUTH_URL**: **Do not set this unless you have a stable custom domain attached to every deployment.** This variable is baked in at Next.js build time. If it points to a different origin than the URL serving the page (e.g. a Vercel preview URL), the browser will block auth API calls with a CORS error. Leave it unset — the auth client automatically uses `window.location.origin`, which is always correct. - **NEXT_PUBLIC_PROXY_URL**: Use your deployed Go proxy URL (e.g., `https://proxy.firefly-analytics.com`) -For our production deployment at FireFly Analytics: +For production with a custom domain: ```env BETTER_AUTH_URL=https://www.firefly-analytics.com -NEXT_PUBLIC_BETTER_AUTH_URL=https://www.firefly-analytics.com +# NEXT_PUBLIC_BETTER_AUTH_URL — only set if using a custom domain on all deployments +# NEXT_PUBLIC_BETTER_AUTH_URL=https://www.firefly-analytics.com NEXT_PUBLIC_PROXY_URL=https://app-proxy.firefly-analytics.com ``` @@ -316,7 +683,26 @@ Replace with your actual production domain. For our deployment, we use: https://www.firefly-analytics.com/api/oauth/databricks/callback ``` -### 5. Deploy +### 5. Disable Vercel Preview Deployment Protection (if using preview URLs) + +Vercel projects have **SSO protection enabled by default** for preview deployments. This blocks unauthenticated requests — including the guest provisioning API (`/api/guest/*`) and any programmatic calls — with a `401 Protected deployment` response. + +To disable it for a project: + +```bash +# Via Vercel CLI (Vercel CLI 54+) +vercel project protection disable --sso + +# Or via Vercel API +curl -X PATCH "https://api.vercel.com/v9/projects/" \ + -H "Authorization: Bearer $VERCEL_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"ssoProtection": null}' +``` + +This only affects preview deployments. Production deployments with a custom domain are not protected by default. + +### 6. Deploy #### Option A: Deploy via Git @@ -334,7 +720,7 @@ vercel --prod vercel ``` -### 6. Verify Deployment +### 7. Verify Deployment After deployment: - Check that all environment variables are set correctly @@ -342,7 +728,7 @@ After deployment: - Verify database connectivity - Ensure the Go proxy is accessible -### 7. Deploy Go Proxy Separately +### 8. Deploy Go Proxy Separately The Go proxy should be deployed separately (not on Vercel): @@ -353,6 +739,55 @@ The Go proxy should be deployed separately (not on Vercel): Update `NEXT_PUBLIC_PROXY_URL` in Vercel environment variables to point to your deployed proxy. +## Guest User Provisioning + +The guest login path lets external users access a specific Databricks workspace via a pre-provisioned service principal, without going through the Databricks OAuth wall. Provisioning is a three-step API sequence secured by `GUEST_API_SECRET` (`X-API-Key` header). + +### Prerequisites + +- `GUEST_API_SECRET` set in Vercel env (64-char hex, `openssl rand -hex 64`) +- A Databricks M2M service principal with client-id + client-secret that has access to the target workspace +- Vercel preview protection disabled if testing against a preview URL (see step 5 above) + +### Step 1 — Register the guest workspace + +```bash +curl -X POST https:///api/guest/workspaces \ + -H "X-API-Key: $GUEST_API_SECRET" \ + -H "Content-Type: application/json" \ + -d '{"name": "Acme Corp Workspace", "workspaceUrl": "https://dbc-xxxx.cloud.databricks.com"}' +# Response: { "workspace": { "id": "", "name": "...", "workspaceUrl": "..." } } +``` + +### Step 2 — Register the guest service principal + +```bash +curl -X POST https:///api/guest/spns \ + -H "X-API-Key: $GUEST_API_SECRET" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Acme Guest SPN", + "clientId": "", + "clientSecret": "", + "guestWorkspaceId": "" + }' +# Response: { "spn": { "id": "", "name": "...", "clientId": "...", "guestWorkspaceId": "..." } } +``` + +### Step 3 — Create the guest user and get a login URL + +```bash +curl -X POST https:///api/guest/users \ + -H "X-API-Key: $GUEST_API_SECRET" \ + -H "Content-Type: application/json" \ + -d '{"orgName": "Acme Corp", "spnId": ""}' +# Response: { "guestUser": { "id": "...", "email": "...", "loginUrl": "https:///guest-login?token=", "expiresAt": "...", ... } } +``` + +Send the `loginUrl` to your end-user. It contains a one-time token valid for 10 minutes. The user clicks it, the token is verified, and they are redirected to their organization's dashboard — no Databricks SSO required. + +> **Note**: The `loginUrl` hostname comes from the server-side `BETTER_AUTH_URL` env var. The auth client verifies the token against `window.location.origin` (or `NEXT_PUBLIC_BETTER_AUTH_URL` if set). These two must match, so `BETTER_AUTH_URL` should be set to the URL where the app is served. Do not set `NEXT_PUBLIC_BETTER_AUTH_URL` to a different origin (see `.env.example`). + ## Architecture ### Authentication Strategies @@ -362,15 +797,17 @@ This application supports multiple authentication strategies: 1. **Login With Databricks**: Per-workspace authentication using Databricks native OAuth 2. **Custom Federation**: Multi-tenant authentication with custom identity providers 3. **Login With Okta**: Tenant-based authentication with service principal identity mapping -4. **Login With Guest User**: Coming Soon +4. **Login With Guest User**: Provisioned via a private REST API secured with `GUEST_API_SECRET`. See [Guest User Provisioning](#guest-user-provisioning) below. ### Key Features - **Organization Support**: Multi-tenant architecture with organization management - **Embedded Databricks Apps**: VSCode editor embedded without SSO exposure +- **Agent Panel**: Slide-out Genie + managed-memory chat assistant, embedded via a Vercel-native SPN proxy (see [Agent Panel](#agent-panel-managed-memory-agent)) - **Notebooks**: Interactive notebooks with full Databricks functionality - **SQL Editor**: Advanced SQL editor with visual query builder - **Data Catalog**: Browse Unity Catalog with a modern interface +- **Pipeline Editor**: Visual node-based pipeline design with Delta Live Tables integration ### Technology Stack @@ -383,9 +820,22 @@ This application supports multiple authentication strategies: ## Documentation -For detailed architectural documentation, visit: -- [Embedding Databricks Apps w/o SSO](http://localhost:3000/docs/architecture/lakehouse-apps-proxy) -- [Login With Databricks Authentication](http://localhost:3000/docs/architecture/authentication/databricks-identity) +### Solutions + +- [All Solutions](/docs/solutions) +- [Embedding Databricks Apps w/o SSO](/docs/solutions/embedding-apps) +- [Notebook Editor](/docs/solutions/notebook-editor) +- [Code Editor](/docs/solutions/code-editor) +- [Agent Panel](/docs/solutions/agent) +- [SQL Editor](/docs/solutions/sql-editor) +- [Data Catalog](/docs/solutions/data-catalog) +- [Pipeline Editor](/docs/solutions/pipeline-editor) + +### Architecture + +- [Embedding Databricks Apps via Proxy (hub)](/docs/architecture/lakehouse-apps-proxy) +- [Architecture Overview](/docs/architecture/overview) +- [Login With Databricks Authentication](/docs/architecture/authentication/databricks-identity) ## Project Support diff --git a/agent/agent_server/agent.py b/agent/agent_server/agent.py new file mode 100644 index 0000000..fbebb5b --- /dev/null +++ b/agent/agent_server/agent.py @@ -0,0 +1,153 @@ +import logging +import os +from contextlib import AsyncExitStack +from datetime import datetime +from typing import AsyncGenerator + +import mlflow +from agents import Agent, Runner, function_tool, set_default_openai_api, set_default_openai_client +from agents.tracing import set_trace_processors +from databricks.sdk import WorkspaceClient +from databricks_openai import AsyncDatabricksOpenAI +from databricks_openai.agents import McpServer +from fastapi import HTTPException +from mlflow.genai.agent_server import invoke, stream +from mlflow.types.responses import ( + ResponsesAgentRequest, + ResponsesAgentResponse, + ResponsesAgentStreamEvent, +) + +from agent_server.utils import ( + build_mcp_url, + get_session_id, + process_agent_stream_events, +) +from agent_server.genie_mcp import genie_mcp_path +from agent_server.genie_tools import GENIE_INSTRUCTIONS, GENIE_TOOLS +from agent_server.utils_memory import ( + MEMORY_INSTRUCTIONS, + MEMORY_TOOLS, + MemoryContext, + resolve_scope, +) + +# Skill-first: utils_memory.py stays a verbatim copy of the managed-memory skill; +# the Genie deviation is composed here at wire time. +INSTRUCTIONS = MEMORY_INSTRUCTIONS + "\n\n" + GENIE_INSTRUCTIONS + +logger = logging.getLogger(__name__) + +set_default_openai_client(AsyncDatabricksOpenAI()) +set_default_openai_api("chat_completions") +set_trace_processors([]) +mlflow.openai.autolog() +logging.getLogger("mlflow.utils.autologging_utils").setLevel(logging.ERROR) + + +@function_tool +def get_current_time() -> str: + """Get the current date and time.""" + return datetime.now().isoformat() + + +def _app_workspace_client() -> WorkspaceClient: + """App service principal (Databricks Apps OAuth M2M), not the proxy caller token.""" + return WorkspaceClient() + + + + +def _genie_mcp_url(app_wc: WorkspaceClient) -> str: + """Full URL for whichever Genie backend genie_mcp_path() resolved.""" + path_or_url = genie_mcp_path() + if path_or_url.startswith("http"): + return path_or_url.rstrip("/") + if not path_or_url.startswith("/"): + path_or_url = f"/{path_or_url}" + return build_mcp_url(path_or_url.rstrip("/"), workspace_client=app_wc) + + +async def init_genie_mcp_server() -> McpServer: + app_wc = _app_workspace_client() + url = _genie_mcp_url(app_wc) + logger.info("Genie MCP mode=%s url=%s", os.environ.get("GENIE_MCP_MODE", "one"), url) + return McpServer( + url=url, + name="Genie", + workspace_client=app_wc, + timeout=60.0, + ) + + +async def connect_healthy_mcp_servers( + stack: AsyncExitStack, servers: list[McpServer] +) -> tuple[list[McpServer], list[str]]: + healthy: list[McpServer] = [] + unavailable: list[str] = [] + for server in servers: + name = getattr(server, "name", "MCP server") + try: + connected = await stack.enter_async_context(server) + await connected.list_tools() + healthy.append(connected) + except Exception: + logger.warning("MCP server %r unavailable; continuing without it.", name, exc_info=True) + unavailable.append(name) + return healthy, unavailable + + +def create_agent(mcp_servers: list[McpServer] | None = None) -> Agent: + return Agent( + name="Agent", + instructions=INSTRUCTIONS, + model="databricks-gpt-5-2", + tools=[get_current_time, *GENIE_TOOLS, *MEMORY_TOOLS], + mcp_servers=mcp_servers or [], + ) + + +def _require_scope(request: ResponsesAgentRequest) -> str: + scope = resolve_scope(request) + if not scope: + raise HTTPException( + status_code=401, + detail="No end-user identity — refusing a shared memory scope.", + ) + return scope + + +async def _mcp_servers_for_request(stack: AsyncExitStack) -> list[McpServer]: + servers, _ = await connect_healthy_mcp_servers(stack, [await init_genie_mcp_server()]) + return servers + + +@invoke() +async def invoke_handler(request: ResponsesAgentRequest) -> ResponsesAgentResponse: + if session_id := get_session_id(request): + mlflow.update_current_trace(metadata={"mlflow.trace.session": session_id}) + scope = _require_scope(request) + async with AsyncExitStack() as stack: + agent = create_agent(mcp_servers=await _mcp_servers_for_request(stack)) + messages = [i.model_dump() for i in request.input] + result = await Runner.run(agent, messages, context=MemoryContext(scope=scope)) + return ResponsesAgentResponse( + output=[item.to_input_item() for item in result.new_items] + ) + + +@stream() +async def stream_handler( + request: ResponsesAgentRequest, +) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + if session_id := get_session_id(request): + mlflow.update_current_trace(metadata={"mlflow.trace.session": session_id}) + scope = _require_scope(request) + async with AsyncExitStack() as stack: + agent = create_agent(mcp_servers=await _mcp_servers_for_request(stack)) + messages = [i.model_dump() for i in request.input] + result = Runner.run_streamed( + agent, input=messages, context=MemoryContext(scope=scope) + ) + async for event in process_agent_stream_events(result.stream_events()): + yield event diff --git a/agent/agent_server/genie_mcp.py b/agent/agent_server/genie_mcp.py new file mode 100644 index 0000000..1b9291a --- /dev/null +++ b/agent/agent_server/genie_mcp.py @@ -0,0 +1,107 @@ +"""Which Genie backend this deployment talks to — resolved in exactly one place. + +There were two answers to that question. agent.py resolved the mode properly and pointed +the MCP server at `/api/2.0/mcp/genie/`, while genie_tools.py held its own +module-level constant `GENIE_MCP_PATH = "/api/2.0/mcp/genie"` and POSTed straight to it. +Both were attached to the same Agent, and GENIE_INSTRUCTIONS told the model it "MUST call +ask_genie first" -- so a deployment configured for a curated space answered from +workspace-wide Genie anyway, silently, with the space-scoped server connected and unused. + +That is worse than a wrong answer. Choosing a space is how you scope answers to curated +tables and joins, and it is the only object a guest can be granted CAN_RUN on. Guests have +no workspace access at all, so answering them from workspace-wide Genie cannot work -- +while every configuration probe still reports mode=space, because the deploy really did +pass that variable. Nothing observed the endpoint a question actually reached. + +The Genie Agent IS the space. Workspace-wide is a deliberate opt-out for someone who wants +it as their MCP, or the fallback where no space can exist (no tables to build one over). +""" + +import logging +import os + +logger = logging.getLogger(__name__) + +# Workspace-wide unified Genie: discovers whatever the calling SP can read, scoped to +# nothing. Reachable only by principals with workspace access, which excludes guests. +GENIE_WORKSPACE_MCP_PATH = "/api/2.0/mcp/genie" + +# `space` is canonical. `agent` is accepted as a forward-looking alias only: the product +# name is Genie Agent, so the value will likely be renamed, but renaming it NOW would +# propagate through the bundle, the runbook, the harness probes and the regression matrix +# for no reader benefit -- and every in-flight `--var genie_mcp_mode=space` would break. +# Accepting both costs one tuple entry and makes that rename a later, safe change. +_SPACE_MODES = ("space", "agent") + +_logged_once = False + + +def _space_id_or_raise(mode: str) -> str: + """The configured Genie space id, or a ValueError naming the fix. + + "none" is the bundle's unset sentinel, not an id. An EMPTY default cannot be used: + the bundle renders {"name": "GENIE_SPACE_ID"} with no `value`, and the Apps API + rejects the deploy with "Must specify environment variable source using either + value or valueFrom". So the variable ships a placeholder and the emptiness check + lives here. + """ + space_id = os.environ.get("GENIE_SPACE_ID", "").strip() + if space_id.lower() in ("", "none", "null"): + raise ValueError( + f"GENIE_MCP_MODE={mode} needs GENIE_SPACE_ID. Set it to a Genie space id, " + "or set GENIE_MCP_MODE=one to use workspace-wide Genie deliberately " + "(note: guest users cannot query workspace-wide Genie)." + ) + return space_id + + +def genie_tool_names() -> tuple[str, str]: + """The (ask, poll) MCP tool names for this deployment's backend. + + The two backends do not expose the same tools, and resolving the path per mode + while leaving the tool names hardcoded to the workspace-wide pair is the same + defect as the one above, one layer down: a space-scoped deployment POSTed + `genie_ask` and got `-32602 BAD_REQUEST: Tool genie_ask does not exist`. + + workspace-wide -> genie_ask, genie_poll_response + space-scoped -> query_space_, poll_response_ + + Their arguments and response fields differ too; genie_tools.py adapts both to one + shape. Verified against tools/list on a live workspace, not inferred from docs. + """ + mode = os.environ.get("GENIE_MCP_MODE", "space").strip().lower() + if mode in _SPACE_MODES: + space_id = _space_id_or_raise(mode) + return f"query_space_{space_id}", f"poll_response_{space_id}" + return "genie_ask", "genie_poll_response" + + +def genie_mcp_path() -> str: + """The Genie MCP path this deployment must use. Same answer for every caller. + + Raises when the mode asks for a space and no id was supplied, rather than quietly + falling back to workspace-wide: a silent fallback is what made this defect invisible. + """ + global _logged_once + + mode = os.environ.get("GENIE_MCP_MODE", "space").strip().lower() + explicit = os.environ.get("GENIE_MCP_URL", "").strip() + + if mode in _SPACE_MODES: + path = f"{GENIE_WORKSPACE_MCP_PATH}/{_space_id_or_raise(mode)}" + elif explicit and not explicit.rstrip("/").endswith(GENIE_WORKSPACE_MCP_PATH): + path = explicit if explicit.startswith("/") else f"/{explicit}" + else: + path = GENIE_WORKSPACE_MCP_PATH + + # Logged once, because "which backend answered" was previously unobservable: the + # startup line recorded the configured mode, not the endpoint each tool called. + if not _logged_once: + logger.info("Genie MCP resolved: mode=%s path=%s", mode, path) + _logged_once = True + return path + + +def is_space_mode() -> bool: + """True when this deployment is scoped to a Genie space (the default).""" + return os.environ.get("GENIE_MCP_MODE", "space").strip().lower() in _SPACE_MODES diff --git a/agent/agent_server/genie_tools.py b/agent/agent_server/genie_tools.py new file mode 100644 index 0000000..08376a8 --- /dev/null +++ b/agent/agent_server/genie_tools.py @@ -0,0 +1,236 @@ +"""Genie Agent MCP helpers — ask + poll until completed.""" + +import json +import logging +import time + +from agents import function_tool +from databricks.sdk import WorkspaceClient + +from agent_server.genie_mcp import genie_mcp_path, genie_tool_names, is_space_mode + +logger = logging.getLogger(__name__) + +MAX_POLLS = 45 +POLL_INTERVAL_SEC = 2.0 + +# Genie message states, which are NOT the workspace-wide tool's lowercase enum +# (in_progress/completed/failed/...). Both vocabularies reach _normalize below. +_SPACE_TERMINAL_OK = ("COMPLETED",) +_SPACE_TERMINAL_BAD = ("FAILED", "CANCELLED", "QUERY_RESULT_EXPIRED") + + +def _app_workspace_client() -> WorkspaceClient: + return WorkspaceClient() + + +def _mcp_tool_call(name: str, arguments: dict) -> dict: + body = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + } + # Resolved per call from GENIE_MCP_MODE, not a module constant. The constant that + # used to live here hardcoded workspace-wide Genie, so a space-scoped deployment + # answered from the wrong backend while every config probe still said mode=space. + raw = _app_workspace_client().api_client.do( + "POST", genie_mcp_path(), body=body + ) + if not isinstance(raw, dict): + return {"error": f"non-dict MCP response: {str(raw)[:300]}"} + # A JSON-RPC error carries `error` and no `result`. Dropping it on the floor and + # returning {} is what surfaced a precise server message -- "BAD_REQUEST: Tool + # genie_ask does not exist" -- to the user as the uninformative "Genie ask + # failed: {}", with nothing anywhere naming the tool or the endpoint. + if raw.get("error") is not None: + err = raw["error"] + message = err.get("message") if isinstance(err, dict) else str(err) + logger.error("Genie MCP tool %s failed: %s", name, message) + return {"error": message or json.dumps(err)[:500]} + result = raw.get("result") or {} + structured = result.get("structuredContent") + if structured: + return structured + content = result.get("content") or [] + if content and isinstance(content[0], dict) and content[0].get("text"): + try: + return json.loads(content[0]["text"]) + except json.JSONDecodeError: + return {"text": content[0]["text"]} + return result + + +def _render_space_answer(content: dict) -> str: + """Markdown from a space-scoped response's `content`. + + Space mode has no `final_answer`; the answer lives in textAttachments plus + queryAttachments (description, SQL, and an inline statement_response). + """ + parts: list[str] = [] + for text in content.get("textAttachments") or []: + if text: + parts.append(str(text)) + + for attachment in content.get("queryAttachments") or []: + if not isinstance(attachment, dict): + continue + if attachment.get("description"): + parts.append(str(attachment["description"])) + if attachment.get("query"): + parts.append(f"```sql\n{attachment['query']}\n```") + table = _render_statement(attachment.get("statement_response")) + if table: + parts.append(table) + + return "\n\n".join(parts).strip() + + +def _render_statement(statement: object) -> str: + """A markdown table from a Genie statement_response, or "" if there is none.""" + if not isinstance(statement, dict): + return "" + columns = [ + col.get("name", "") + for col in ( + ((statement.get("manifest") or {}).get("schema") or {}).get("columns") or [] + ) + ] + rows = ((statement.get("result") or {}).get("data_array")) or [] + if not columns or not rows: + return "" + + def cells(row: object) -> list[str]: + # JSON_ARRAY comes back as {"values": [{"string_value": "x"}]}, but plain + # lists of scalars also occur; render both rather than picking one and + # emitting an empty table for the other. + if isinstance(row, dict): + values = row.get("values") or [] + return [ + str(v.get("string_value", "")) if isinstance(v, dict) else str(v) + for v in values + ] + if isinstance(row, list): + return ["" if v is None else str(v) for v in row] + return [str(row)] + + lines = [ + "| " + " | ".join(columns) + " |", + "| " + " | ".join("---" for _ in columns) + " |", + ] + for row in rows[:100]: + lines.append("| " + " | ".join(cells(row)) + " |") + if len(rows) > 100: + lines.append(f"\n_({len(rows)} rows; first 100 shown.)_") + return "\n".join(lines) + + +def _normalize(payload: dict) -> dict: + """One shape for both backends: conversation_id, response_id, status, final_answer. + + Space mode returns camelCase ids, uppercase Genie message states, and the answer + inside `content`. Normalizing at the boundary keeps ask/poll single-path instead + of branching on the mode at every field access. + """ + if payload.get("error"): + return {"status": "failed", "error": payload["error"]} + + if "conversationId" not in payload and "messageId" not in payload: + return payload # already the workspace-wide shape + + raw_status = str(payload.get("status") or "").upper() + if raw_status in _SPACE_TERMINAL_OK: + status = "completed" + elif raw_status in _SPACE_TERMINAL_BAD: + status = "failed" + else: + status = "in_progress" + + normalized = { + "conversation_id": payload.get("conversationId"), + "response_id": payload.get("messageId"), + "status": status, + "genie_status": raw_status, + } + answer = _render_space_answer(payload.get("content") or {}) + if answer: + normalized["final_answer"] = answer + return normalized + + +def _poll_genie(conversation_id: str, response_id: str) -> dict: + _, poll_tool = genie_tool_names() + # The space-scoped tool names its second argument message_id; the workspace-wide + # one calls it response_id. Same value, and passing the wrong key fails as an + # invalid-argument error rather than anything mentioning Genie. + id_key = "message_id" if is_space_mode() else "response_id" + for _ in range(MAX_POLLS): + payload = _normalize( + _mcp_tool_call( + poll_tool, + {"conversation_id": conversation_id, id_key: response_id}, + ) + ) + status = payload.get("status") + if status == "completed": + return payload + if status == "failed": + return payload + time.sleep(POLL_INTERVAL_SEC) + return {"status": "timeout", "message": "Genie poll timed out"} + + +_BROAD_DATA_PROMPTS = ( + "tell me about my data", + "what is my data", + "what's my data", + "describe my data", + "my data", +) + +_TABLE_DETAIL_SUFFIX = ( + " List the catalogs, schemas, and tables available. For each table, give its " + "purpose, key columns, and approximate row count. Prioritize concrete tables " + "and their columns over dashboards or Genie spaces." +) + + +def _augment_broad_question(question: str) -> str: + normalized = question.strip().lower().rstrip("?.! ") + if normalized in _BROAD_DATA_PROMPTS: + return question.strip() + _TABLE_DETAIL_SUFFIX + return question + + +@function_tool +def ask_genie(question: str) -> str: + """Query Genie Agent over workspace data. Use for any question about tables, catalogs, dashboards, or 'my data'. For broad questions, it automatically requests table- and column-level detail. Polls until Genie completes.""" + question = _augment_broad_question(question) + ask_tool, _ = genie_tool_names() + # Space mode calls the question `query`; workspace-wide calls it `question`. + arg = "query" if is_space_mode() else "question" + ask = _normalize(_mcp_tool_call(ask_tool, {arg: question})) + if ask.get("status") == "completed" and ask.get("final_answer"): + return ask["final_answer"] + conversation_id = ask.get("conversation_id") + response_id = ask.get("response_id") + if not conversation_id or not response_id: + # Name the endpoint and tool. Without them this read "Genie ask failed: {}", + # which says nothing about which backend was asked or what it objected to. + detail = ask.get("error") or json.dumps(ask)[:2000] + return ( + f"Genie ask failed via tool '{ask_tool}' at {genie_mcp_path()}: {detail}" + ) + result = _poll_genie(conversation_id, response_id) + if result.get("status") == "completed": + return result.get("final_answer") or json.dumps(result)[:8000] + return json.dumps(result)[:2000] + + +GENIE_TOOLS = [ask_genie] + +# Composed onto MEMORY_INSTRUCTIONS in agent.py so utils_memory.py stays a pristine, +# regenerable copy of the managed-memory skill (this is the local Genie deviation). +GENIE_INSTRUCTIONS = """For any question about workspace data, tables, catalogs, dashboards, metrics, or phrases like "my data", you MUST call ask_genie first with the user's question — do not ask the user to clarify catalog/schema first. When the user's question is broad (e.g. "tell me about my data"), phrase the ask_genie question to request concrete data assets: the catalogs, schemas, and tables available, plus each table's purpose, key columns, and row counts where possible (for example: "What catalogs, schemas, and tables do I have? For each table, give its purpose, key columns, and approximate row count."). If the first Genie answer stays high-level (only dashboards or Genie spaces, no tables/columns), call ask_genie again asking specifically for the tables and their columns in the catalogs that were returned. Use get_current_time for the current date and time. + +After ask_genie or memory tools return, answer in clear markdown prose (headings, bullets, tables). When Genie returns dashboard or asset links, include them as markdown links. Prefer surfacing concrete tables and their key columns over a list of dashboards. Never paste raw tool JSON in the final reply.""" diff --git a/agent/agent_server/start_server.py b/agent/agent_server/start_server.py new file mode 100644 index 0000000..731cf4a --- /dev/null +++ b/agent/agent_server/start_server.py @@ -0,0 +1,29 @@ +import os +from pathlib import Path + +from dotenv import load_dotenv +from mlflow.genai.agent_server import AgentServer, setup_mlflow_git_based_version_tracking + +# Load env vars from .env before importing the agent for proper auth +load_dotenv(dotenv_path=Path(__file__).parent.parent / ".env", override=True) + +# Need to import the agent to register the functions with the server +import agent_server.agent # noqa: E402 + +_enable_chat_proxy = os.environ.get("ENABLE_CHAT_PROXY", "true").lower() not in ( + "0", + "false", + "no", +) +agent_server = AgentServer("ResponsesAgent", enable_chat_proxy=_enable_chat_proxy) +# Define the app as a module level variable to enable multiple workers +app = agent_server.app # noqa: F841 +setup_mlflow_git_based_version_tracking() + + +def main(): + agent_server.run(app_import_string="agent_server.start_server:app") + + +if __name__ == "__main__": + main() diff --git a/agent/agent_server/utils.py b/agent/agent_server/utils.py new file mode 100644 index 0000000..c0bb8fa --- /dev/null +++ b/agent/agent_server/utils.py @@ -0,0 +1,67 @@ +import logging +from typing import AsyncGenerator, AsyncIterator, Optional +from uuid import uuid4 + +from agents.result import StreamEvent +from databricks.sdk import WorkspaceClient +from mlflow.genai.agent_server import get_request_headers +from mlflow.types.responses import ResponsesAgentRequest, ResponsesAgentStreamEvent + + +def get_session_id(request: ResponsesAgentRequest) -> str | None: + if request.context and request.context.conversation_id: + return request.context.conversation_id + if request.custom_inputs and isinstance(request.custom_inputs, dict): + return request.custom_inputs.get("session_id") + return None + + +def get_databricks_host(workspace_client: WorkspaceClient | None = None) -> Optional[str]: + workspace_client = workspace_client or WorkspaceClient() + try: + return workspace_client.config.host + except Exception as e: + logging.exception(f"Error getting databricks host from env: {e}") + return None + + +def build_mcp_url(path: str, workspace_client: WorkspaceClient | None = None) -> str: + if not path.startswith("/"): + return path + hostname = get_databricks_host(workspace_client) + return f"{hostname}{path}" + + +def get_user_workspace_client() -> WorkspaceClient: + headers = get_request_headers() + token = headers.get("x-forwarded-access-token") + if not token: + auth = headers.get("authorization") or headers.get("Authorization") or "" + if auth.lower().startswith("bearer "): + token = auth[7:].strip() + if not token: + return WorkspaceClient() + return WorkspaceClient(token=token, auth_type="pat") + + +async def process_agent_stream_events( + async_stream: AsyncIterator[StreamEvent], +) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + curr_item_id = str(uuid4()) + async for event in async_stream: + if event.type == "raw_response_event": + event_data = event.data.model_dump() + if event_data["type"] == "response.output_item.added": + curr_item_id = str(uuid4()) + event_data["item"]["id"] = curr_item_id + elif event_data.get("item") is not None and event_data["item"].get("id") is not None: + event_data["item"]["id"] = curr_item_id + elif event_data.get("item_id") is not None: + event_data["item_id"] = curr_item_id + yield event_data + elif event.type == "run_item_stream_event" and event.item.type == "tool_call_output_item": + # Required for chat history: Edit/Regenerate must replay tool_call + output pairs. + yield ResponsesAgentStreamEvent( + type="response.output_item.done", + item=event.item.to_input_item(), + ) diff --git a/agent/agent_server/utils_memory.py b/agent/agent_server/utils_memory.py new file mode 100644 index 0000000..dfadaa7 --- /dev/null +++ b/agent/agent_server/utils_memory.py @@ -0,0 +1,205 @@ +# GENERATED from vendor/app-templates/agent-openai-agents-sdk/.claude/skills/managed-memory/SKILL.md +# submodule pin: 2a4c79296aae89c8a05e7825c61e0d94ad34c944 +# Regenerate verbatim by re-running the managed-memory skill; do NOT hand-edit. +# local deviations: NONE (Genie instructions composed in agent.py via genie_tools.GENIE_INSTRUCTIONS). +"""Databricks managed memory (UC memory-store) tools for the OpenAI Agents SDK.""" + +import os + +from agents import RunContextWrapper, function_tool +from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError +from dataclasses import dataclass +from mlflow.genai.agent_server import get_request_headers +from mlflow.types.responses import ResponsesAgentRequest + +from agent_server.utils import get_user_workspace_client + +_client: WorkspaceClient | None = None + + +def _ws() -> WorkspaceClient: + global _client + if _client is None: + _client = WorkspaceClient() + return _client + + +def _entries(suffix: str = "") -> str: + store = os.getenv("DATABRICKS_MEMORY_STORE") + if not store: + raise RuntimeError( + "DATABRICKS_MEMORY_STORE is not set — it must be the full catalog.schema.name." + ) + return f"/api/2.1/unity-catalog/memory-stores/{store}/entries{suffix}" + + +def resolve_scope(request: ResponsesAgentRequest | None = None) -> str | None: + """End-user id used as memory scope; fail closed when unknown in production.""" + headers = get_request_headers() or {} + if headers.get("x-forwarded-access-token"): + return get_user_workspace_client().current_user.me().id + if os.getenv("DATABRICKS_APP_NAME"): + return None + ci = dict(getattr(request, "custom_inputs", None) or {}) + return headers.get("x-forwarded-user") or ci.get("user_id") + + +def _save(scope: str, path: str, description: str, contents: str = "") -> str: + try: + _ws().api_client.do( + "POST", + _entries(), + query={"scope": scope}, + body={ + "path": path, + "contents": contents, + "description": description, + "creation_reason": "CREATION_REASON_AGENT_INFERRED", + "creation_source": "CREATION_SOURCE_ONLINE_AGENT", + }, + ) + except DatabricksError as e: + if e.error_code == "ALREADY_EXISTS": + return f"A memory already exists at {path}; use update_memory to revise it." + return f"Could not save {path}: {getattr(e, 'message', str(e))}" + return f"Saved memory at {path}." + + +def _get(scope: str, path: str) -> str: + try: + entry = _ws().api_client.do( + "GET", _entries(":get"), query={"scope": scope, "path": path} + ) + except DatabricksError as e: + if e.error_code == "NOT_FOUND": + return f"No memory at {path}." + return f"Could not read {path}: {getattr(e, 'message', str(e))}" + return entry.get("contents") or entry.get("description") or f"(empty memory at {path})" + + +def _list(scope: str) -> str: + try: + resp = _ws().api_client.do("GET", _entries(), query={"scope": scope}) + except DatabricksError as e: + return f"Could not list memories: {getattr(e, 'message', str(e))}" + items = resp.get("entries", []) + if not items: + return "No memories yet." + lines = [ + ("[has_contents] " if e.get("has_contents") else "") + + f"- {e['path']}: {e.get('description', '')}" + for e in items + ] + return f"{len(items)} memories total:\n" + "\n".join(lines) + + +def _update( + scope: str, path: str, op: dict | None = None, description: str | None = None +) -> str: + op = op or {} + if len(op) > 1: + return "Pass at most one contents edit (str_replace / insert / replace_all)." + if not op and description is None: + return "Provide a new description and/or one contents edit." + body: dict = {"scope": scope, "path": path, **op} + if description is not None: + body["description"] = description + try: + _ws().api_client.do("PATCH", _entries(), body=body) + except DatabricksError as e: + if e.error_code == "NOT_FOUND": + return f"No memory at {path} to update — check list_memories or save it first." + return f"Could not update {path}: {getattr(e, 'message', str(e))}" + return f"Updated {path}." + + +def _delete(scope: str, path: str) -> str: + try: + _ws().api_client.do( + "DELETE", _entries(), query={"scope": scope, "path": path} + ) + except DatabricksError as e: + if e.error_code == "NOT_FOUND": + return f"No memory at {path} (already gone)." + return f"Could not delete {path}: {getattr(e, 'message', str(e))}" + return f"Deleted {path}." + + +@dataclass +class MemoryContext: + """Per-request run context; scope partitions memories by end user.""" + + scope: str + + +def _scope(ctx: RunContextWrapper[MemoryContext]) -> str: + if not ctx.context.scope: + raise RuntimeError("No end-user scope for this request — refusing a shared memory bucket.") + return ctx.context.scope + + +@function_tool(strict_mode=False) +async def save_memory( + ctx: RunContextWrapper[MemoryContext], + path: str, + description: str, + contents: str = "", +) -> str: + """Create ONE durable memory at a stable /memories/... path. Check list_memories first.""" + return _save(_scope(ctx), path, description, contents) + + +@function_tool +async def get_memory(ctx: RunContextWrapper[MemoryContext], path: str) -> str: + """Read the full contents of one memory by exact path from list_memories.""" + return _get(_scope(ctx), path) + + +@function_tool +async def list_memories(ctx: RunContextWrapper[MemoryContext]) -> str: + """List saved memories (path + description only). Call before save or recall.""" + return _list(_scope(ctx)) + + +@function_tool(strict_mode=False) +async def update_memory( + ctx: RunContextWrapper[MemoryContext], + path: str, + description: str | None = None, + str_replace: dict | None = None, + insert: dict | None = None, + replace_all: dict | None = None, +) -> str: + """Revise an existing memory. Pass description and/or exactly one contents edit op.""" + op = { + k: v + for k, v in ( + ("str_replace", str_replace), + ("insert", insert), + ("replace_all", replace_all), + ) + if v + } + return _update(_scope(ctx), path, op, description) + + +@function_tool +async def delete_memory(ctx: RunContextWrapper[MemoryContext], path: str) -> str: + """Permanently remove one memory by exact path.""" + return _delete(_scope(ctx), path) + + +MEMORY_TOOLS = [save_memory, get_memory, list_memories, update_memory, delete_memory] + +MEMORY_INSTRUCTIONS = """You have durable, cross-session memory about whoever (or whatever) this conversation is scoped to. Use it deliberately, not by reflex. + +Recall whenever the answer is about the user or calls for personalized information — anything that might draw on preferences, decisions, or workflows they've shared before — and you don't already have it from this conversation; also list once before saving, to find the right existing topic. Don't tell the user you don't know their preferences without checking — list_memories first. Skip memory only when the answer truly doesn't depend on who's asking (general knowledge, math, coding) or you already have what you need. A `[has_contents]` entry has a body to get_memory; one without is fully captured by its description. Open a memory with get_memory before you state its specifics, and never assert a fact that isn't stored — if nothing relevant is stored, just answer without it. Don't re-list what you've already seen this turn. + +Save only what will still matter in a future, unrelated conversation — a stable preference, fact, decision, or ongoing project the user actually stated or decided. Don't save your own suggestions or guesses, passing chatter, secrets, or anything scoped to this chat ("for now", a one-off label). +- Write each memory so it stands on its own out of context, under one broad, stable /memories/... topic per subject with the specifics inside it. +- Check the list first and update_memory an existing topic instead of minting a near-duplicate. +- For a very broad question that touches many memories, summarize from the list's descriptions; reserve get_memory for the specific entry you actually need. +- If the user's info changes or contradicts what's stored, update or replace it rather than keeping both — but don't rewrite a memory that already says the same thing. +- delete_memory what's stale. +- Briefly tell the user whenever you save, update, or delete.""" diff --git a/agent/databricks.yml b/agent/databricks.yml new file mode 100644 index 0000000..8185656 --- /dev/null +++ b/agent/databricks.yml @@ -0,0 +1,161 @@ +bundle: + name: firefly_openai_managed_mem + +# --- Parameterize (bundle variables) -------------------------------------- +# Namespace for the wheels volume and the managed-memory store. Defaults target +# `workspace.default` so a fresh Default-Storage workspace (which has no `main` +# catalog) deploys with zero edits (GAP-3). Override per target below, or at the +# command line via `--var catalog=main` / env `BUNDLE_VAR_catalog=main`, or with +# a variable-overrides.json. +variables: + catalog: + description: UC catalog holding the wheels volume and managed-memory store. + default: workspace + schema: + description: UC schema within ${var.catalog}. + default: default + wheels_volume_name: + description: UC volume that holds the pre-vendored build wheels. + default: firefly_wheels + memory_store_name: + description: Managed-memory store (Lakebase-backed) name. + default: firefly_managed_memory + # Genie backend. The Genie Agent IS a curated Genie space, so `space` is the default and + # the space id is supplied by the phase that creates the space BEFORE this bundle is + # first deployed. The app is therefore never even briefly workspace-wide. + # + # The VALUE stays `space` deliberately. The product is called Genie Agent and the value + # will likely follow, but renaming it now would ripple through the runbook, the harness + # probes and the regression matrix at the same time as the reordering below -- two + # changes at once, each able to mask the other. genie_mcp.py already accepts `agent` as + # an alias, so that rename is a later one-line change. + # + # `one` is workspace-wide unified Genie and is an explicit OPT-OUT, not a fallback: it + # discovers whatever the calling SP can read, scoped to nothing, and GUEST USERS CANNOT + # QUERY IT AT ALL, having no workspace access. Reaching it by accident is precisely what + # this default prevents -- genie_tools.py used to hardcode workspace-wide, so a + # space-scoped deployment answered from the wrong backend while every configuration + # probe still reported the space. + # + # These two never move independently. genie_mcp.py raises when the mode wants a space + # and the id is empty, so the app fails to boot instead of degrading silently. + genie_mcp_mode: + description: "`space` (default) for a curated Genie space, or `one` to opt into workspace-wide Genie. `agent` is accepted as an alias for `space`." + default: space + genie_space_id: + description: Genie space id — required when genie_mcp_mode=space (the default), `none` only when opting into `one`. + # NOT empty. An empty value makes the bundle render this env entry as + # `{"name": "GENIE_SPACE_ID"}` with no `value` key, and the Apps API refuses + # the deploy: "Must specify environment variable source using either value + # or valueFrom". That broke Phase 4 on the DEFAULT (Genie Agent) path, which is + # the path most deployments take. agent.py treats `none` as unset, so the + # mode=space guard still raises rather than querying a bogus space. + default: "none" + +sync: + exclude: + # uv.lock is intentionally NOT synced. The Apps build runs a plain `uv sync` + # (not `uv sync --locked`), so it can satisfy dependencies from the vendored + # wheels via UV_FIND_LINKS without the lock appearing stale (GAP-15). + # NOTE: pyproject.toml and vendor-wheels/ MUST sync — do not add them here + # (GAP-7: no deps file => app crashes; GAP-14: vendored wheels feed the build). + - uv.lock + - databricks.yml + - .python-version + - .vendor-req.txt + - .vendor-req-export.txt + - .vendor-req-filtered.txt + - requirements.txt.volume.bak + +resources: + apps: + agent_openai_agents_sdk: + name: "firefly-openai-managed-mem-v2" + description: "OpenAI Agents SDK agent with managed memory and Genie MCP" + source_code_path: ./ + config: + command: ["python", "scripts/start_app.py"] + env: + - name: MLFLOW_TRACKING_URI + value: "databricks" + - name: MLFLOW_REGISTRY_URI + value: "databricks-uc" + - name: API_PROXY + value: "http://localhost:8000/invocations" + - name: CHAT_APP_PORT + value: "3000" + - name: CHAT_PROXY_TIMEOUT_SECONDS + value: "300" + # Install the pre-vendored linux/cp311 wheels first so the build never + # depends on the build container's PyPI egress (GAP-8/11/12). This path + # is the synced source_code root inside the Apps container. Run + # scripts/vendor_wheels.sh once before `databricks bundle deploy`. + - name: UV_FIND_LINKS + value: "/app/python/source_code/vendor-wheels" + # Pin every dependency to the vendored/tested versions (#64). The build runs a + # plain `uv sync` (uv.lock is excluded — GAP-15), which can otherwise re-resolve a + # transitive dep to a newer release that lacks a Linux wheel (the greenlet 3.5.4 + # crash). constraints.txt is generated by vendor_wheels.sh (Phase 3d) and synced to + # the source root; UV_CONSTRAINT *bounds* versions without the exact-match rigidity + # that made `uv sync --locked` fail (GAP-15). Absent file → uv ignores it (safe). + - name: UV_CONSTRAINT + value: "/app/python/source_code/constraints.txt" + # --- Leave alone (resource bindings) --------------------------------- + # Populated by `uv run quickstart` (experiment id + Lakebase endpoint). + # Never fold these into a plaintext value:. + - name: MLFLOW_EXPERIMENT_ID + value_from: "experiment" + - name: LAKEBASE_AUTOSCALING_ENDPOINT + value_from: "postgres" + # --- Parameterize ---------------------------------------------------- + - name: DATABRICKS_MEMORY_STORE + value: "${var.catalog}.${var.schema}.${var.memory_store_name}" + # --- Derive, don't store (GAP-4 fix) --------------------------------- + # Neither DATABRICKS_HOST nor DATABRICKS_WORKSPACE_ID is stored here: the + # Databricks Apps runtime auto-injects both into every app. Bundle + # substitutions cannot do this portably — ${workspace.host} is empty when + # the host comes from a CLI profile (the `deploy -p ` flow), and + # ${workspace.workspace_id} does not resolve into app env. Runtime + # injection means nothing to hand-rewrite and nothing to go stale. + # + # There is no GENIE_ONE_URL. The frontend used to compose a workspace + # attribution link from those two vars, but this panel's audience is + # guest users with no workspace access, so the link was dead for them — + # and with GENIE_MCP_MODE defaulting to a space it also named the wrong + # backend. Attribution is plain text now. + - name: GENIE_MCP_URL + value: "/api/2.0/mcp/genie" + - name: GENIE_MCP_MODE + value: ${var.genie_mcp_mode} + # Empty unless Phase 6c resolved a space. GENIE_MCP_MODE=one ignores it. + - name: GENIE_SPACE_ID + value: ${var.genie_space_id} + resources: + - name: wheels_volume + uc_securable: + securable_full_name: "${var.catalog}.${var.schema}.${var.wheels_volume_name}" + securable_type: VOLUME + permission: READ_VOLUME + # Leave alone — quickstart writes experiment_id and the postgres branch + # for the target workspace; the committed values are placeholders. + - name: experiment + experiment: + experiment_id: "123237888438046" + permission: CAN_MANAGE + - name: 'postgres' + postgres: + branch: "projects/firefly-openai-managed-mem-v2-ui-lb-202607090900/branches/firefly-openai-managed-mem-v2-ui-lb-202607090900-branch" + database: "projects/firefly-openai-managed-mem-v2-ui-lb-202607090900/branches/firefly-openai-managed-mem-v2-ui-lb-202607090900-branch/databases/databricks-postgres" + permission: 'CAN_CONNECT_AND_CREATE' + +targets: + dev: + mode: development + default: true + + prod: + mode: production + resources: + apps: + agent_openai_agents_sdk: + name: firefly-openai-managed-mem diff --git a/agent/patches/e2e-chatbot-app-next/client/src/components/chat.tsx b/agent/patches/e2e-chatbot-app-next/client/src/components/chat.tsx new file mode 100644 index 0000000..b525abd --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/components/chat.tsx @@ -0,0 +1,354 @@ +import type { DataUIPart, LanguageModelUsage, UIMessageChunk } from 'ai'; +import { useChat } from '@ai-sdk/react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useSWRConfig } from 'swr'; +import { ChatHeader } from '@/components/chat-header'; +import { fetchWithErrorHandlers, generateUUID } from '@/lib/utils'; +import { MultimodalInput } from './multimodal-input'; +import { Messages } from './messages'; +import type { + Attachment, + ChatMessage, + CustomUIDataTypes, + FeedbackMap, + VisibilityType, +} from '@chat-template/core'; +import { unstable_serialize } from 'swr/infinite'; +import { getChatHistoryPaginationKey } from './sidebar-history'; +import { toast } from './toast'; +import { useSearchParams } from 'react-router-dom'; +import { useChatVisibility } from '@/hooks/use-chat-visibility'; +import { ChatSDKError } from '@chat-template/core/errors'; +import { useDataStream } from './data-stream-provider'; +import { isCredentialErrorMessage } from '@/lib/oauth-error-utils'; +import { ChatTransport } from '../lib/ChatTransport'; +import type { ClientSession } from '@chat-template/auth'; +import { softNavigateToChatId } from '@/lib/navigation'; +import { useAppConfig } from '@/contexts/AppConfigContext'; +import { Greeting } from './greeting'; +import { GenieAttribution } from './genie-attribution'; + +export function Chat({ + id, + initialMessages, + initialChatModel, + initialVisibilityType, + isReadonly, + initialLastContext, + feedback = {}, + title, +}: { + id: string; + initialMessages: ChatMessage[]; + initialChatModel: string; + initialVisibilityType: VisibilityType; + isReadonly: boolean; + session: ClientSession; + initialLastContext?: LanguageModelUsage; + feedback?: FeedbackMap; + title?: string; +}) { + const { visibilityType } = useChatVisibility({ + chatId: id, + initialVisibilityType, + }); + + const { mutate } = useSWRConfig(); + const { setDataStream } = useDataStream(); + const { chatHistoryEnabled, genieConfig } = useAppConfig(); + + const [input, setInput] = useState(''); + const [_usage, setUsage] = useState( + initialLastContext, + ); + + const [lastPart, setLastPart] = useState(); + const lastPartRef = useRef(lastPart); + lastPartRef.current = lastPart; + + // Single counter for resume attempts - reset when stream parts are received + const resumeAttemptCountRef = useRef(0); + const maxResumeAttempts = 3; + + const abortController = useRef(new AbortController()); + useEffect(() => { + return () => { + abortController.current?.abort('ABORT_SIGNAL'); + }; + }, []); + + const fetchWithAbort = useMemo(() => { + return async (input: RequestInfo | URL, init?: RequestInit) => { + // useChat does not cancel /stream requests when the component is unmounted + const signal = abortController.current?.signal; + return fetchWithErrorHandlers(input, { ...init, signal }); + }; + }, []); + + const stop = useCallback(() => { + abortController.current?.abort('USER_ABORT_SIGNAL'); + }, []); + + const isNewChat = initialMessages.length === 0; + const didFetchHistoryOnNewChat = useRef(false); + const fetchChatHistory = useCallback(() => { + mutate(unstable_serialize(getChatHistoryPaginationKey)); + }, [mutate]); + + // For new chats, the title arrives via a `data-title` stream part + // once backend title generation completes — no separate fetch needed. + const [streamTitle, setStreamTitle] = useState(); + const [titlePending, setTitlePending] = useState(false); + const displayTitle = title ?? streamTitle; + + const { + messages, + setMessages, + sendMessage, + status, + resumeStream, + clearError, + addToolApprovalResponse, + regenerate, + } = useChat({ + id, + messages: initialMessages, + experimental_throttle: 100, + generateId: generateUUID, + resume: id !== undefined && initialMessages.length > 0, // Enable automatic stream resumption + transport: new ChatTransport({ + onStreamPart: (part) => { + if (isNewChat && !didFetchHistoryOnNewChat.current) { + fetchChatHistory(); + if (chatHistoryEnabled) { + setTitlePending(true); + } + didFetchHistoryOnNewChat.current = true; + } + // Reset resume attempts when we successfully receive stream parts + resumeAttemptCountRef.current = 0; + setLastPart(part); + }, + api: '/api/chat', + fetch: fetchWithAbort, + prepareSendMessagesRequest({ messages, id, body }) { + const lastMessage = messages.at(-1); + const isUserMessage = lastMessage?.role === 'user'; + + // For continuations (non-user messages like tool results), we must always + // send previousMessages because the tool result only exists client-side + // and hasn't been saved to the database yet. + const needsPreviousMessages = !chatHistoryEnabled || !isUserMessage; + + return { + body: { + id, + // Only include message field for user messages (new messages) + // For continuation (assistant messages with tool results), omit message field + ...(isUserMessage ? { message: lastMessage } : {}), + selectedChatModel: initialChatModel, + selectedVisibilityType: visibilityType, + nextMessageId: generateUUID(), + // Send previous messages when: + // 1. Database is disabled (ephemeral mode) - always need client-side messages + // 2. Continuation request (tool results) - tool result only exists client-side + ...(needsPreviousMessages + ? { + previousMessages: isUserMessage + ? messages.slice(0, -1) + : messages, + } + : {}), + ...body, + }, + }; + }, + prepareReconnectToStreamRequest({ id }) { + return { + api: `/api/chat/${id}/stream`, + credentials: 'include', + }; + }, + }), + onData: (dataPart) => { + setDataStream((ds) => + ds ? [...ds, dataPart as DataUIPart] : [], + ); + if (dataPart.type === 'data-usage') { + setUsage(dataPart.data as LanguageModelUsage); + } + if (dataPart.type === 'data-title') { + setStreamTitle(dataPart.data as string); + setTitlePending(false); + fetchChatHistory(); + } + }, + onFinish: ({ + isAbort, + isDisconnect, + isError, + messages: finishedMessages, + }) => { + didFetchHistoryOnNewChat.current = false; + setTitlePending(false); + + // If user aborted, don't try to resume + if (isAbort) { + console.log('[Chat onFinish] Stream was aborted by user, not resuming'); + fetchChatHistory(); + return; + } + + // Check if the last message contains an OAuth credential error + // If so, don't try to resume - the user needs to authenticate first + const lastMessage = finishedMessages?.at(-1); + const hasOAuthError = lastMessage?.parts?.some( + (part) => + part.type === 'data-error' && + typeof part.data === 'string' && + isCredentialErrorMessage(part.data), + ); + + if (hasOAuthError) { + console.log( + '[Chat onFinish] OAuth credential error detected, not resuming', + ); + fetchChatHistory(); + clearError(); + return; + } + + // Determine if we should attempt to resume: + // 1. Stream didn't end with a 'finish' part (incomplete) + // 2. It was a disconnect/error that terminated the stream + // 3. We haven't exceeded max resume attempts + const streamIncomplete = lastPartRef.current?.type !== 'finish'; + const shouldResume = + streamIncomplete && + (isDisconnect || isError || lastPartRef.current === undefined); + + if (shouldResume && resumeAttemptCountRef.current < maxResumeAttempts) { + console.log( + '[Chat onFinish] Resuming stream. Attempt:', + resumeAttemptCountRef.current + 1, + ); + resumeAttemptCountRef.current++; + // Ref: https://github.com/vercel/ai/issues/8477#issuecomment-3603209884 + queueMicrotask(() => { + resumeStream(); + }) + } else { + // Stream completed normally or we've exhausted resume attempts + if (resumeAttemptCountRef.current >= maxResumeAttempts) { + console.warn('[Chat onFinish] Max resume attempts reached'); + } + fetchChatHistory(); + } + }, + onError: (error) => { + console.log('[Chat onError] Error occurred:', error); + + // Only show toast for explicit ChatSDKError (backend validation errors) + // Other errors (network, schema validation) are handled silently or in message parts + if (error instanceof ChatSDKError) { + toast({ + type: 'error', + description: error.message, + }); + } else { + // Non-ChatSDKError: Could be network error or in-stream error + // Log but don't toast - errors during streaming may be informational + console.warn('[Chat onError] Error during streaming:', error.message); + } + // Note: We don't call resumeStream here because onError can be called + // while the stream is still active (e.g., for data-error parts). + // Resume logic is handled exclusively in onFinish. + }, + }); + + const [searchParams] = useSearchParams(); + const query = searchParams.get('query'); + + const [hasAppendedQuery, setHasAppendedQuery] = useState(false); + + useEffect(() => { + if (query && !hasAppendedQuery) { + sendMessage({ + role: 'user' as const, + parts: [{ type: 'text', text: query }], + }); + + setHasAppendedQuery(true); + softNavigateToChatId(id, chatHistoryEnabled); + } + }, [query, sendMessage, hasAppendedQuery, id, chatHistoryEnabled]); + + const [attachments, setAttachments] = useState>([]); + + const inputElement = + + const genieFooter = genieConfig ? ( + + ) : null; + + if (messages.length === 0) { + return ( +
+ +
+
+ + {inputElement} + {genieFooter} +
+
+
+ ); + } + + return ( + <> +
+ + + + + + +
+ {!isReadonly && ( + inputElement + )} + {genieFooter} +
+
+ + ); +} diff --git a/agent/patches/e2e-chatbot-app-next/client/src/components/elements/mcp-tool.tsx b/agent/patches/e2e-chatbot-app-next/client/src/components/elements/mcp-tool.tsx new file mode 100644 index 0000000..45862d2 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/components/elements/mcp-tool.tsx @@ -0,0 +1,261 @@ +import { Button } from '@/components/ui/button'; +import { CollapsibleTrigger } from '@/components/ui/collapsible'; +import { cn } from '@/lib/utils'; +import { + ChevronDownIcon, + ServerIcon, + ShieldAlertIcon, + ShieldCheckIcon, + ShieldXIcon, + Sparkles, +} from 'lucide-react'; +import { + ToolContainer, + ToolContent, + ToolInput, + ToolOutput, + ToolStatusBadge, + type ToolState, +} from './tool'; + +// MCP-specific container with distinct styling +type McpToolProps = Parameters[0]; + +export const McpTool = ({ className, ...props }: McpToolProps) => ( + +); + +// Re-export shared components for convenience +export { + ToolContent as McpToolContent, + ToolInput as McpToolInput, + ToolOutput as McpToolOutput, +}; + +// MCP-specific header with banner +type McpToolHeaderProps = { + serverName?: string; + toolName: string; + state: ToolState; + // Used when state is 'approval-responded' to determine approval outcome + approved?: boolean; + className?: string; +}; + +// Badge component for approval status in the banner +// Uses AI SDK native tool states directly +type ApprovalStatusBadgeProps = { + state: ToolState; + // Used when state is 'approval-responded' to determine approval outcome + approved?: boolean; +}; + +const ApprovalStatusBadge = ({ state, approved }: ApprovalStatusBadgeProps) => { + // Pending: waiting for user approval + if (state === 'approval-requested') { + return ( + + + Pending + + ); + } + + // Allowed: tool executed successfully or user approved (waiting for execution) + if ( + state === 'output-available' || + (state === 'approval-responded' && approved === true) + ) { + return ( + + + Allowed + + ); + } + + // Denied: user explicitly denied the tool + if ( + state === 'output-denied' || + (state === 'approval-responded' && approved === false) + ) { + return ( + + + Denied + + ); + } + + // Fallback for any other state - show as pending + return ( + + + Pending + + ); +}; + +function isGenieServer(serverName?: string): boolean { + return !!serverName && /genie/i.test(serverName); +} + +export const McpToolHeader = ({ + className, + serverName, + toolName, + state, + approved, +}: McpToolHeaderProps) => ( +
+ {/* MCP Banner */} +
+ + + Tool Call Request + + {isGenieServer(serverName) ? ( + <> + • + + + Powered by Genie + + + ) : ( + serverName && ( + <> + • + {serverName} + + ) + )} + • + +
+ {/* Tool header */} + +
+ {toolName} +
+
+ {/* Only show tool status badge when tool is running/completed (approved) */} + {(state === 'output-available' || + (state === 'approval-responded' && approved === true)) && ( + + )} + +
+ +
+); + +// MCP-specific approval actions +type McpApprovalActionsProps = { + onApprove: () => void; + onDeny: () => void; + isSubmitting: boolean; +}; + +export const McpApprovalActions = ({ + onApprove, + onDeny, + isSubmitting, +}: McpApprovalActionsProps) => ( +
+
+ +

+ This tool requires your permission to run. +

+
+
+ + +
+
+); + +// MCP-specific approval status display +type McpApprovalStatusProps = { + approved: boolean; + reason?: string; +}; + +export const McpApprovalStatus = ({ + approved, + reason, +}: McpApprovalStatusProps) => ( +
+ {approved ? ( + + ) : ( + + )} + + {approved ? 'Allowed' : 'Denied'} + + {reason && ( + <> + • + {reason} + + )} +
+); diff --git a/agent/patches/e2e-chatbot-app-next/client/src/components/genie-attribution.tsx b/agent/patches/e2e-chatbot-app-next/client/src/components/genie-attribution.tsx new file mode 100644 index 0000000..f5632a2 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/components/genie-attribution.tsx @@ -0,0 +1,87 @@ +import { ExternalLink, Sparkles } from 'lucide-react'; +import { cn } from '@/lib/utils'; + +/** + * `workspaceHost` / `workspaceId` are retained because callers pass them and + * they identify the workspace for display, but there is deliberately no longer a + * link built from them. See the note on the removed attribution link below. + */ +export type GenieAttributionProps = { + workspaceHost?: string; + workspaceId?: string; + links?: string[]; + variant?: 'inline' | 'footer' | 'compact'; + className?: string; +}; + +function linkLabel(url: string, index: number): string { + try { + const parsed = new URL(url); + if (parsed.pathname.includes('/dashboard')) { + return 'AI/BI dashboard'; + } + if (parsed.pathname.includes('/genie/')) { + return 'Genie Agent'; + } + const path = parsed.pathname.split('/').filter(Boolean).slice(-2).join('/'); + return path || parsed.hostname || `Related link ${index + 1}`; + } catch { + return `Related link ${index + 1}`; + } +} + +export function GenieAttribution({ + links = [], + variant = 'inline', + className, +}: GenieAttributionProps) { + const uniqueLinks = [...new Set(links.filter((link) => link.startsWith('http')))]; + + return ( +
+
+ {/* + Attribution is plain text on purpose. There used to be a "Genie One" + link here, and it was wrong twice over: the audience for this panel is + guest users who have no Databricks workspace access, so the link led + somewhere they cannot open; and once the agent defaults to a Genie + space, that "Genie One" label named a backend that never saw the question. A dead + link labelled with the wrong backend is worse than no link. + */} + + + Powered by Genie + +
+ {uniqueLinks.length > 0 && ( + + )} +
+ ); +} diff --git a/agent/patches/e2e-chatbot-app-next/client/src/components/message.tsx b/agent/patches/e2e-chatbot-app-next/client/src/components/message.tsx new file mode 100644 index 0000000..3b0c1da --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/components/message.tsx @@ -0,0 +1,531 @@ +import React, { memo, useState } from 'react'; +import { AnimatedAssistantIcon } from './animation-assistant-icon'; +import { Response } from './elements/response'; +import { MessageContent } from './elements/message'; +import { + Tool, + ToolHeader, + ToolContent, + ToolInput, + ToolOutput, + type ToolState, +} from './elements/tool'; +import { + McpTool, + McpToolHeader, + McpToolContent, + McpToolInput, + McpApprovalActions, +} from './elements/mcp-tool'; +import { MessageActions } from './message-actions'; +import { PreviewAttachment } from './preview-attachment'; +import equal from 'fast-deep-equal'; +import { cn, sanitizeText, formatToolOutput } from '@/lib/utils'; +import { MessageEditor } from './message-editor'; +import { MessageReasoning } from './message-reasoning'; +import { Shimmer } from './ui/shimmer'; +import type { UseChatHelpers } from '@ai-sdk/react'; +import type { ChatMessage, Feedback } from '@chat-template/core'; +import { useDataStream } from './data-stream-provider'; +import { + createMessagePartSegments, + formatNamePart, + isNamePart, + joinMessagePartSegments, +} from './databricks-message-part-transformers'; +import { MessageError } from './message-error'; +import { MessageOAuthError } from './message-oauth-error'; +import { isCredentialErrorMessage } from '@/lib/oauth-error-utils'; +import { + groupConsecutiveToolSegments, + type ChatPart, + type ToolPart, +} from '@/lib/tool-group-segments'; +import { Streamdown } from 'streamdown'; +import { useApproval } from '@/hooks/use-approval'; +import { useAppConfig } from '@/contexts/AppConfigContext'; +import { GenieAttribution } from './genie-attribution'; +import { + collectGenieLinksFromMessage, + extractGenieLinks, + isGenieToolPart, +} from '@/lib/genie-attribution'; + +const PurePreviewMessage = ({ + message, + allMessages, + isLoading, + setMessages, + addToolApprovalResponse, + sendMessage, + regenerate, + isReadonly, + requiresScrollPadding, + initialFeedback, +}: { + message: ChatMessage; + allMessages: ChatMessage[]; + isLoading: boolean; + setMessages: UseChatHelpers['setMessages']; + addToolApprovalResponse: UseChatHelpers['addToolApprovalResponse']; + sendMessage: UseChatHelpers['sendMessage']; + regenerate: UseChatHelpers['regenerate']; + isReadonly: boolean; + requiresScrollPadding: boolean; + initialFeedback?: Feedback; +}) => { + const [mode, setMode] = useState<'view' | 'edit'>('view'); + const [showErrors, setShowErrors] = useState(false); + + // Hook for handling MCP approval requests + const { submitApproval, isSubmitting, pendingApprovalId } = useApproval({ + addToolApprovalResponse, + sendMessage, + }); + const { genieConfig } = useAppConfig(); + const genieLinks = React.useMemo( + () => collectGenieLinksFromMessage(message.parts), + [message.parts], + ); + const showGenieAttribution = + message.role === 'assistant' && !!genieConfig; + + const attachmentsFromMessage = message.parts.filter( + (part) => part.type === 'file', + ); + + // Extract non-OAuth error parts separately (OAuth errors are rendered inline) + const errorParts = React.useMemo( + () => + message.parts + .filter((part) => part.type === 'data-error') + .filter((part) => { + // OAuth errors are rendered inline, not in the error section + return !isCredentialErrorMessage(part.data); + }), + [message.parts], + ); + + useDataStream(); + + const partSegments = React.useMemo( + /** + * We segment message parts into segments that can be rendered as a single component. + * Used to render citations as part of the associated text. + * Note: OAuth errors are included here for inline rendering, non-OAuth errors are filtered out. + */ + () => + createMessagePartSegments( + message.parts.filter( + (part) => + part.type !== 'data-error' || isCredentialErrorMessage(part.data), + ), + ), + [message.parts], + ); + + const renderBlocks = React.useMemo( + () => groupConsecutiveToolSegments(partSegments), + [partSegments], + ); + + // Check if message only contains non-OAuth errors (no other content) + const hasOnlyErrors = React.useMemo(() => { + const nonErrorParts = message.parts.filter( + (part) => part.type !== 'data-error', + ); + // Only consider non-OAuth errors for this check + return errorParts.length > 0 && nonErrorParts.length === 0; + }, [message.parts, errorParts.length]); + + return ( +
+
+ {partSegments.length === 0 && errorParts.length === 0 && message.role === 'assistant' && ( + + )} + +
+ {attachmentsFromMessage.length > 0 && ( +
+ {attachmentsFromMessage.map((attachment) => ( + + ))} +
+ )} + + {renderBlocks.map((block) => { + if (block.kind === 'tool-group') { + return ( + + ); + } + + const parts = block.parts; + const index = block.index; + const [part] = parts; + const { type } = part; + const key = `message-${message.id}-part-${index}`; + + if (type === 'reasoning' && part.text?.trim().length > 0) { + return ( + + ); + } + + if (type === 'text') { + if (isNamePart(part)) { + return ( + {`# ${formatNamePart(part)}`} + ); + } + if (mode === 'view') { + return ( +
+ + + {sanitizeText(joinMessagePartSegments(parts))} + + +
+ ); + } + + if (mode === 'edit') { + return ( +
+
+
+ +
+
+ ); + } + } + + // dynamic-tool parts are rendered by MessageToolGroup above. + + // Support for citations/annotations + if (type === 'source-url') { + return ( + + [{part.title || part.url}] + + ); + } + + // Render OAuth errors inline + if (type === 'data-error' && isCredentialErrorMessage(part.data)) { + return ( + + ); + } + })} + + {showGenieAttribution && genieConfig && ( + + )} + + {!isReadonly && !hasOnlyErrors && ( + setShowErrors(!showErrors)} + initialFeedback={initialFeedback} + /> + )} + + {errorParts.length > 0 && (hasOnlyErrors || showErrors) && ( +
+ {errorParts.map((part, index) => ( + + ))} +
+ )} +
+
+
+ ); +}; + +export const PreviewMessage = memo( + PurePreviewMessage, + (prevProps, nextProps) => { + if (prevProps.isLoading !== nextProps.isLoading) return false; + // While streaming, re-render whenever the AI SDK produces a new message + // object (each throttled update). We use reference equality rather than + // deep-equal on parts because fast-deep-equal short-circuits on identical + // references — and the SDK may mutate parts in place during streaming. + if (nextProps.isLoading && prevProps.message !== nextProps.message) + return false; + + if (prevProps.message.id !== nextProps.message.id) return false; + if (prevProps.requiresScrollPadding !== nextProps.requiresScrollPadding) + return false; + if (!equal(prevProps.message.parts, nextProps.message.parts)) return false; + if (prevProps.initialFeedback?.feedbackType !== nextProps.initialFeedback?.feedbackType) + return false; + + return true; // Props are equal, skip re-render + }, +); + + +const MessageToolGroup = ({ + tools, + isLoading, + submitApproval, + isSubmitting, + pendingApprovalId, +}: { + tools: ToolPart[]; + isLoading: boolean; + submitApproval: ReturnType['submitApproval']; + isSubmitting: boolean; + pendingApprovalId: string | null; +}) => { + const isMultiple = tools.length > 1; + return ( +
+ {tools.map((tool) => ( + + ))} +
+ ); +}; + +const ToolPartRenderer = ({ + part, + isLoading, + submitApproval, + isSubmitting, + pendingApprovalId, +}: { + part: ToolPart; + isLoading: boolean; + submitApproval: ReturnType['submitApproval']; + isSubmitting: boolean; + pendingApprovalId: string | null; +}) => { + const { genieConfig } = useAppConfig(); + const { toolCallId, input, state, errorText, output, toolName } = part; + + const isMcpApproval = + part.callProviderMetadata?.databricks?.approvalRequestId != null; + const mcpServerName = + part.callProviderMetadata?.databricks?.mcpServerName?.toString(); + const isGenieTool = isGenieToolPart(part); + const genieLinks = + isGenieTool && output != null ? extractGenieLinks(output) : []; + + const approved: boolean | undefined = + 'approval' in part ? part.approval?.approved : undefined; + + const effectiveState: ToolState = (() => { + if (part.providerExecuted && !isLoading && state === 'input-available') { + return 'output-available'; + } + return state; + })(); + + if (isMcpApproval) { + return ( + + + + + {state === 'approval-requested' && ( + + submitApproval({ approvalRequestId: toolCallId, approve: true }) + } + onDeny={() => + submitApproval({ + approvalRequestId: toolCallId, + approve: false, + }) + } + isSubmitting={isSubmitting && pendingApprovalId === toolCallId} + /> + )} + {state === 'output-available' && output != null && ( + + Error: {errorText} +
+ ) : ( +
+
+ {formatToolOutput(output)} +
+ {isGenieTool && genieConfig && ( + + )} +
+ ) + } + errorText={undefined} + /> + )} + + + ); + } + + return ( + + + + + {state === 'output-available' && ( + + Error: {errorText} + + ) : ( +
+
+ {formatToolOutput(output)} +
+ {isGenieTool && genieConfig && ( + + )} +
+ ) + } + errorText={undefined} + /> + )} + + + ); +}; + +export const AwaitingResponseMessage = () => { + const role = 'assistant'; + + return ( +
+
+ Generating response +
+
+ ); +}; diff --git a/agent/patches/e2e-chatbot-app-next/client/src/contexts/AppConfigContext.tsx b/agent/patches/e2e-chatbot-app-next/client/src/contexts/AppConfigContext.tsx new file mode 100644 index 0000000..3883fa2 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/contexts/AppConfigContext.tsx @@ -0,0 +1,69 @@ +import { createContext, useContext, type ReactNode } from 'react'; +import useSWR from 'swr'; +import { fetcher } from '@/lib/utils'; + +export interface GenieConfig { + workspaceHost: string; + workspaceId?: string; +} + +interface ConfigResponse { + features: { + chatHistory: boolean; + feedback: boolean; + }; + obo?: { + missingScopes: string[]; + }; + genie?: GenieConfig | null; +} + +interface AppConfigContextType { + config: ConfigResponse | undefined; + isLoading: boolean; + error: Error | undefined; + chatHistoryEnabled: boolean; + feedbackEnabled: boolean; + oboMissingScopes: string[]; + genieConfig: GenieConfig | null; +} + +const AppConfigContext = createContext( + undefined, +); + +export function AppConfigProvider({ children }: { children: ReactNode }) { + const { data, error, isLoading } = useSWR( + '/api/config', + fetcher, + { + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 60000, + }, + ); + + const value: AppConfigContextType = { + config: data, + isLoading, + error, + chatHistoryEnabled: data?.features.chatHistory ?? true, + feedbackEnabled: data?.features.feedback ?? false, + oboMissingScopes: data?.obo?.missingScopes ?? [], + genieConfig: data?.genie ?? null, + }; + + return ( + + {children} + + ); +} + +export function useAppConfig() { + const context = useContext(AppConfigContext); + if (context === undefined) { + throw new Error('useAppConfig must be used within an AppConfigProvider'); + } + return context; +} diff --git a/agent/patches/e2e-chatbot-app-next/client/src/lib/genie-attribution.ts b/agent/patches/e2e-chatbot-app-next/client/src/lib/genie-attribution.ts new file mode 100644 index 0000000..f71b67c --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/lib/genie-attribution.ts @@ -0,0 +1,105 @@ +import type { ChatMessage } from '@chat-template/core'; + +const URL_RE = /https?:\/\/[^\s"'<>)\]]+/gi; +const MARKDOWN_LINK_RE = /\[[^\]]*\]\((https?:\/\/[^)]+)\)/gi; + +export function isGenieServerName(name?: string): boolean { + if (!name) return false; + return /genie/i.test(name); +} + +export function isGenieToolPart( + part: ChatMessage['parts'][number], +): part is Extract { + if (part.type !== 'dynamic-tool') return false; + const server = part.callProviderMetadata?.databricks?.mcpServerName?.toString(); + if (isGenieServerName(server)) return true; + const toolName = part.toolName ?? ''; + return /genie/i.test(toolName); +} + +export function messageUsesGenie(parts: ChatMessage['parts']): boolean { + return parts.some(isGenieToolPart); +} + +function normalizeUrl(url: string): string { + return url.replace(/[.,;]+$/g, ''); +} + +function isNavigableGenieLink(url: string): boolean { + try { + const parsed = new URL(url); + return parsed.protocol === 'http:' || parsed.protocol === 'https:'; + } catch { + return false; + } +} + +export function extractMarkdownLinks(text: string): string[] { + const urls: string[] = []; + for (const match of text.matchAll(MARKDOWN_LINK_RE)) { + const url = match[1]; + if (url && isNavigableGenieLink(url)) { + urls.push(normalizeUrl(url)); + } + } + return urls; +} + +function collectUrls(value: unknown, out: Set, depth = 0): void { + if (depth > 12 || value == null) return; + + if (typeof value === 'string') { + extractMarkdownLinks(value).forEach((url) => out.add(url)); + const matches = value.match(URL_RE); + matches?.forEach((match) => { + const normalized = normalizeUrl(match); + if (isNavigableGenieLink(normalized)) { + out.add(normalized); + } + }); + return; + } + + if (Array.isArray(value)) { + value.forEach((item) => collectUrls(item, out, depth + 1)); + return; + } + + if (typeof value === 'object') { + for (const [key, nested] of Object.entries(value as Record)) { + if ( + /(url|link|href|dashboard|visualization|citation)/i.test(key) && + typeof nested === 'string' && + nested.startsWith('http') + ) { + out.add(normalizeUrl(nested)); + } + collectUrls(nested, out, depth + 1); + } + } +} + +export function extractGenieLinks(output: unknown): string[] { + const urls = new Set(); + collectUrls(output, urls); + return [...urls]; +} + +export function collectGenieLinksFromMessage( + parts: ChatMessage['parts'], +): string[] { + const urls = new Set(); + for (const part of parts) { + if (isGenieToolPart(part) && part.output != null) { + extractGenieLinks(part.output).forEach((url) => urls.add(url)); + } + if (part.type === 'text' && typeof part.text === 'string') { + extractGenieLinks(part.text).forEach((url) => urls.add(url)); + } + if (part.type === 'source-url' && typeof part.url === 'string') { + urls.add(normalizeUrl(part.url)); + } + } + return [...urls]; +} diff --git a/agent/patches/e2e-chatbot-app-next/client/src/lib/utils.ts b/agent/patches/e2e-chatbot-app-next/client/src/lib/utils.ts new file mode 100644 index 0000000..e130743 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/lib/utils.ts @@ -0,0 +1,129 @@ +import type { UIMessagePart } from 'ai'; +import { type ClassValue, clsx } from 'clsx'; +import { twMerge } from 'tailwind-merge'; +import type { DBMessage } from '@chat-template/db'; +import { ChatSDKError, type ErrorCode } from '@chat-template/core/errors'; +import type { + ChatMessage, + ChatTools, + CustomUIDataTypes, +} from '@chat-template/core'; +import { formatISO } from 'date-fns'; + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} + +export const fetcher = async (url: string) => { + const response = await fetch(url); + + if (!response.ok) { + const { code, cause } = await response.json(); + throw new ChatSDKError(code as ErrorCode, cause); + } + + // Handle 204 No Content - return empty chat history response + if (response.status === 204) { + return { chats: [], hasMore: false }; + } + + return response.json(); +}; + +export async function fetchWithErrorHandlers( + input: RequestInfo | URL, + init?: RequestInit, +) { + try { + const response = await fetch(input, init); + + if (!response.ok) { + const parsedResponse = await response.json(); + console.log('parsedResponse', parsedResponse); + const { code, cause } = parsedResponse; + throw new ChatSDKError(code as ErrorCode, cause); + } + + return response; + } catch (error: unknown) { + if (typeof navigator !== 'undefined' && !navigator.onLine) { + throw new ChatSDKError('offline:chat'); + } + + throw error; + } +} + +export function generateUUID(): string { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { + const r = (Math.random() * 16) | 0; + const v = c === 'x' ? r : (r & 0x3) | 0x8; + return v.toString(16); + }); +} + +export function sanitizeText(text: string) { + return text.replace('', ''); +} + +export function convertToUIMessages(messages: DBMessage[]): ChatMessage[] { + return messages.map((message) => ({ + id: message.id, + role: message.role as 'user' | 'assistant' | 'system', + parts: message.parts as UIMessagePart[], + metadata: { + createdAt: formatISO(message.createdAt), + }, + })); +} + +export function getTextFromMessage(message: ChatMessage): string { + return message.parts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join(''); +} + +/** Best-effort human-readable rendering for MCP / Genie tool outputs. */ +export function formatToolOutput(output: unknown): string { + if (output == null) return ''; + if (typeof output === 'string') { + const trimmed = output.trim(); + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + return formatToolOutput(JSON.parse(trimmed)); + } catch { + return output; + } + } + return output; + } + + if (Array.isArray(output)) { + const parts = output + .map((item) => formatToolOutput(item)) + .filter((s) => s.length > 0); + return parts.join('\n\n'); + } + + if (typeof output === 'object') { + const record = output as Record; + if (typeof record.text === 'string' && record.text.trim()) { + return record.text; + } + if (typeof record.message === 'string' && record.message.trim()) { + return record.message; + } + if (typeof record.result === 'string' && record.result.trim()) { + return record.result; + } + if (Array.isArray(record.content)) { + return formatToolOutput(record.content); + } + if (record.type === 'text' && typeof record.text === 'string') { + return record.text; + } + } + + return JSON.stringify(output, null, 2); +} diff --git a/agent/patches/e2e-chatbot-app-next/client/src/main.tsx b/agent/patches/e2e-chatbot-app-next/client/src/main.tsx new file mode 100644 index 0000000..86c17d7 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/src/main.tsx @@ -0,0 +1,44 @@ +import ReactDOM from 'react-dom/client'; +import { BrowserRouter } from 'react-router-dom'; +import App from './App'; +import './index.css'; + +// Firefly proxy-embed shim (source-side; keeps the reverse proxy stock). +// When embedded, the iframe is served under a dynamic mount: +// - Vercel-native route: /api/agent-proxy/... +// - Go proxy (optional): /app-proxy/{toolId}/... +// Derive that mount prefix from the current path and (1) drive the router +// basename and (2) prefix absolute /api/ fetches so they route back through +// the proxy instead of hitting the frontend origin root. +declare global { + interface Window { + __FIREFLY_PROXY_BASENAME__?: string; + } +} + +const proxyPrefix = + window.location.pathname.match(/^\/(?:app-proxy\/[^/]+|api\/agent-proxy)/)?.[0] ?? + ''; +const routerBasename = proxyPrefix || '/'; + +if (proxyPrefix) { + window.__FIREFLY_PROXY_BASENAME__ = proxyPrefix; + const originalFetch = window.fetch.bind(window); + window.fetch = (input, init) => { + if (typeof input === 'string' && input.startsWith('/api/')) { + input = proxyPrefix + input; + } + return originalFetch(input, init); + }; +} + +const rootElement = document.getElementById('root'); +if (!rootElement) { + throw new Error('Failed to find the root element with ID "root"'); +} + +ReactDOM.createRoot(rootElement).render( + + + , +); diff --git a/agent/patches/e2e-chatbot-app-next/client/vite.config.ts b/agent/patches/e2e-chatbot-app-next/client/vite.config.ts new file mode 100644 index 0000000..de3798f --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/client/vite.config.ts @@ -0,0 +1,54 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; +import path from "node:path"; +import type { ProxyOptions } from "vite"; + +export function simulateNetworkError( + _timeout: number, +): ProxyOptions["configure"] { + return (proxy, _options) => { + proxy.on("proxyReq", (proxyReq, _req, res) => { + setTimeout(() => { + // Destroy the socket connection to the browser + res.socket?.destroy(new Error("simulated network error")); + + // Also clean up the backend connection + proxyReq.socket?.destroy(); + }, 2000); + }); + }; +} + +const proxyTarget = "http://localhost:3001"; + +// https://vitejs.dev/config/ +export default defineConfig({ + // Firefly proxy-embed: emit relative asset URLs so they resolve under the + // dynamic /app-proxy/{toolId}/ mount instead of the proxy origin root. + base: "./", + plugins: [react()], + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, + server: { + port: process.env.PORT ? Number.parseInt(process.env.PORT) : 3000, + proxy: { + "/api/chat": { + target: proxyTarget, + changeOrigin: true, + // Uncomment this to test situations where the stream will time out. + // configure: simulateNetworkError(2000), + }, + "/api": { + target: process.env.BACKEND_URL || "http://localhost:3001", + changeOrigin: true, + }, + }, + }, + build: { + outDir: "dist", + sourcemap: false, + }, +}); diff --git a/agent/patches/e2e-chatbot-app-next/server/src/lib/sanitize-ui-messages.ts b/agent/patches/e2e-chatbot-app-next/server/src/lib/sanitize-ui-messages.ts new file mode 100644 index 0000000..941f22c --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/server/src/lib/sanitize-ui-messages.ts @@ -0,0 +1,50 @@ +import type { ChatMessage } from '@chat-template/core'; + +const COMPLETED_TOOL_STATES = new Set([ + 'output-available', + 'output-denied', + 'output-error', +]); + +/** + * Drop assistant tool parts that never received outputs. Orphan function_call + * items break OpenAI replay (Edit / Regenerate) with "tool_call_ids did not + * have response messages". + */ +export function sanitizeUiMessagesForModel( + messages: ChatMessage[], +): ChatMessage[] { + return messages + .map((message) => { + if (message.role !== 'assistant' || !message.parts?.length) { + return message; + } + + const parts = message.parts.filter((part) => { + if (part.type !== 'dynamic-tool') { + return true; + } + return COMPLETED_TOOL_STATES.has(part.state); + }); + + if (parts.length === message.parts.length) { + return message; + } + + return { ...message, parts }; + }) + .filter((message) => { + if (message.role !== 'assistant') { + return true; + } + if (!message.parts?.length) { + return false; + } + return message.parts.some( + (part) => + part.type === 'text' || + part.type === 'reasoning' || + part.type === 'dynamic-tool', + ); + }); +} diff --git a/agent/patches/e2e-chatbot-app-next/server/src/routes/chat.ts b/agent/patches/e2e-chatbot-app-next/server/src/routes/chat.ts new file mode 100644 index 0000000..1922940 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/server/src/routes/chat.ts @@ -0,0 +1,630 @@ +import { + Router, + type Request, + type Response, + type Router as RouterType, +} from 'express'; +import { + convertToModelMessages, + createUIMessageStream, + streamText, + generateText, + type LanguageModelUsage, + pipeUIMessageStreamToResponse, +} from 'ai'; +import type { LanguageModelV3Usage } from '@ai-sdk/provider'; + +// Convert ai's LanguageModelUsage to @ai-sdk/provider's LanguageModelV3Usage +function toV3Usage(usage: LanguageModelUsage): LanguageModelV3Usage { + return { + inputTokens: { + total: usage.inputTokens, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { + total: usage.outputTokens, + text: undefined, + reasoning: undefined, + }, + }; +} +import { + authMiddleware, + requireAuth, + requireChatAccess, + getIdFromRequest, +} from '../middleware/auth'; +import { + deleteChatById, + getMessagesByChatId, + saveChat, + saveMessages, + updateChatLastContextById, + updateChatVisiblityById, + isDatabaseAvailable, + updateChatTitleById, +} from '@chat-template/db'; +import { + type ChatMessage, + checkChatAccess, + convertToUIMessages, + generateUUID, + myProvider, + postRequestBodySchema, + type PostRequestBody, + StreamCache, + type VisibilityType, + CONTEXT_HEADER_CONVERSATION_ID, + CONTEXT_HEADER_USER_ID, +} from '@chat-template/core'; +import { ChatSDKError } from '@chat-template/core/errors'; +import { storeMessageMeta } from '../lib/message-meta-store'; +import { drainStreamToWriter, fallbackToGenerateText } from '../lib/stream-fallback'; +import { sanitizeUiMessagesForModel } from '../lib/sanitize-ui-messages'; + +export const chatRouter: RouterType = Router(); + +const streamCache = new StreamCache(); +// Apply auth middleware to all chat routes +chatRouter.use(authMiddleware); + +/** + * POST /api/chat - Send a message and get streaming response + * + * Note: Works in ephemeral mode when database is disabled. + * Streaming continues normally, but no chat/message persistence occurs. + */ +chatRouter.post('/', requireAuth, async (req: Request, res: Response) => { + const dbAvailable = isDatabaseAvailable(); + if (!dbAvailable) { + console.log('[Chat] Running in ephemeral mode - no persistence'); + } + + let requestBody: PostRequestBody; + + try { + requestBody = postRequestBodySchema.parse(req.body); + } catch (_) { + console.error('Error parsing request body:', _); + const error = new ChatSDKError('bad_request:api'); + const response = error.toResponse(); + return res.status(response.status).json(response.json); + } + + try { + const { + id, + message, + selectedChatModel, + selectedVisibilityType, + }: { + id: string; + message?: ChatMessage; + selectedChatModel: string; + selectedVisibilityType: VisibilityType; + } = requestBody; + + const session = req.session; + if (!session) { + const error = new ChatSDKError('unauthorized:chat'); + const response = error.toResponse(); + return res.status(response.status).json(response.json); + } + + const { chat, allowed, reason } = await checkChatAccess( + id, + session?.user.id, + ); + + if (reason !== 'not_found' && !allowed) { + const error = new ChatSDKError('forbidden:chat'); + const response = error.toResponse(); + return res.status(response.status).json(response.json); + } + + let titlePromise: Promise | undefined; + + if (!chat) { + // Only create new chat if we have a message (not a continuation) + if (isDatabaseAvailable() && message) { + await saveChat({ + id, + userId: session.user.id, + title: 'New chat', + visibility: selectedVisibilityType, + }); + + titlePromise = generateTitleFromUserMessage({ message }) + .then(async (title) => { + await updateChatTitleById({ chatId: id, title }); + return title; + }) + .catch(async (error) => { + console.error('Error generating title:', error); + const textFromUserMessage = message?.parts.find( + (part) => part.type === 'text', + )?.text; + if (textFromUserMessage) { + const fallback = truncatePreserveWords( + textFromUserMessage, + 128, + ); + await updateChatTitleById({ chatId: id, title: fallback }); + return fallback; + } + return null; + }); + } + } else { + if (chat.userId !== session.user.id) { + const error = new ChatSDKError('forbidden:chat'); + const response = error.toResponse(); + return res.status(response.status).json(response.json); + } + } + + const messagesFromDb = await getMessagesByChatId({ id }); + + // Use previousMessages from request body when: + // 1. Ephemeral mode (DB not available) - always use client-side messages + // 2. Continuation request (no message) - tool results only exist client-side + const useClientMessages = + !dbAvailable || (!message && requestBody.previousMessages); + const previousMessages = useClientMessages + ? (requestBody.previousMessages ?? []) + : convertToUIMessages(messagesFromDb); + + // If message is provided, add it to the list and save it + // If not (continuation/regeneration), just use previous messages + let uiMessages: ChatMessage[]; + if (message) { + uiMessages = [...previousMessages, message]; + await saveMessages({ + messages: [ + { + chatId: id, + id: message.id, + role: 'user', + parts: message.parts, + attachments: [], + createdAt: new Date(), + traceId: null, + }, + ], + }); + } else { + // Continuation: use existing messages without adding new user message + uiMessages = previousMessages as ChatMessage[]; + + // For continuations with database enabled, save any updated assistant messages + // This ensures tool-result parts (like MCP approval responses) are persisted + if (dbAvailable && requestBody.previousMessages) { + const assistantMessages = requestBody.previousMessages.filter( + (m: ChatMessage) => m.role === 'assistant', + ); + if (assistantMessages.length > 0) { + await saveMessages({ + messages: assistantMessages.map((m: ChatMessage) => ({ + chatId: id, + id: m.id, + role: m.role, + parts: m.parts, + attachments: [], + createdAt: m.metadata?.createdAt + ? new Date(m.metadata.createdAt) + : new Date(), + traceId: null, + })), + }); + } + } + + // Whether the user DENIED a tool call decides whether we call the model, which has + // nothing to do with persistence. This check used to sit inside the `dbAvailable` + // guard above (and inside its `assistantMessages.length > 0` guard), so in ephemeral + // mode -- a documented, supported configuration -- a continuation carrying a denial + // fell straight through to streamText and the model ran against a tool the user had + // just refused. Only saveMessages belongs behind the database check. + // Only the MOST RECENT tool interaction decides this. Scanning the whole history with + // .some() meant that once a user denied anything, every later continuation matched that + // stale part and returned early -- so approving a subsequent tool silently dropped the + // continuation and the conversation stalled with no way forward. A denied part stays in + // the client's message history forever, so "any denial anywhere" is never the question; + // "was the thing we are continuing FROM denied" is. + const toolParts = (requestBody.previousMessages ?? []).flatMap( + (m: ChatMessage) => + (m.parts ?? []).filter((p) => p.type === 'dynamic-tool'), + ); + const lastToolPart = toolParts[toolParts.length - 1]; + const hasMcpDenial = + !!lastToolPart && + (lastToolPart.state === 'output-denied' || + ('approval' in lastToolPart && lastToolPart.approval?.approved === false)); + + if (hasMcpDenial) { + // The user denied the tool call we would be continuing from, so there is nothing to + // ask the model. + res.end(); + return; + } + } + + // Clear any previous active stream for this chat + streamCache.clearActiveStream(id); + + let finalUsage: LanguageModelUsage | undefined; + let traceId: string | null = null; + const streamId = generateUUID(); + + const model = await myProvider.languageModel(selectedChatModel); + const modelMessages = await convertToModelMessages( + sanitizeUiMessagesForModel(uiMessages), + ); + const requestHeaders = { + [CONTEXT_HEADER_CONVERSATION_ID]: id, + [CONTEXT_HEADER_USER_ID]: session.user.email ?? session.user.id, + // Forward OBO user token to the backend/serving endpoint + ...(req.headers['x-forwarded-access-token'] + ? { 'x-forwarded-access-token': req.headers['x-forwarded-access-token'] as string } + : {}), + }; + + const result = streamText({ + model, + messages: modelMessages, + providerOptions: { + databricks: { includeTrace: true }, + }, + includeRawChunks: true, + headers: requestHeaders, + onChunk: ({ chunk }) => { + if (chunk.type === 'raw') { + const raw = chunk.rawValue as any; + // Extract trace in Databricks serving endpoint output format, if present + if (raw?.type === 'response.output_item.done') { + const traceIdFromChunk = + raw?.databricks_output?.trace?.info?.trace_id; + if (typeof traceIdFromChunk === 'string') { + traceId = traceIdFromChunk; + } + } + // Extract trace from MLflow AgentServer output format, if present + if (!traceId && typeof raw?.trace_id === 'string') { + traceId = raw.trace_id; + } + } + }, + onFinish: ({ usage }) => { + finalUsage = usage; + }, + }); + + /** + * We manually read from toUIMessageStream instead of using writer.merge + * so the execute promise (and thus the outer stream) stays alive if we + * need to fall back to generateText after a streaming error. + */ + const stream = createUIMessageStream({ + // Pass originalMessages so that continuation responses reuse the existing + // assistant message ID. Without this, handleUIMessageStreamFinish generates + // a fresh ID, causing the client to push a second assistant message instead + // of replacing the existing one. + originalMessages: uiMessages, + // The DB Message.id column is typed as uuid, so we must generate UUIDs + // rather than the AI SDK's default short-id format (e.g. "Xt8nZiQRj1fS4yiU"). + generateId: generateUUID, + execute: async ({ writer }) => { + // Manually drain the AI stream so we can append the traceId data part + // after all model chunks are processed (traceId is captured via onChunk). + // result.toUIMessageStream() converts TextStreamPart → UIMessageChunk: + // - text-delta: maps TextStreamPart.text → UIMessageChunk.delta + // - start-step/finish-step: strips extra fields + // - finish: strips rawFinishReason/totalUsage + // - raw: dropped (trace_id captured via onChunk above) + const aiStream = result.toUIMessageStream({ + sendReasoning: true, + sendSources: true, + sendFinish: false, + onError: (error) => { + const msg = + error instanceof Error ? error.message : String(error); + writer.onError?.(error); + return msg; + }, + }); + + const { failed } = await drainStreamToWriter(aiStream, writer); + + if (failed) { + console.log('Streaming failed, falling back to generateText...'); + const fallbackResult = await fallbackToGenerateText( + { model, messages: modelMessages, headers: requestHeaders }, + writer, + ); + + finalUsage = fallbackResult?.usage; + traceId = fallbackResult?.traceId ?? null; + } + if (titlePromise) { + const generatedTitle = await titlePromise; + if (generatedTitle) { + writer.write({ type: 'data-title', data: generatedTitle }); + } + } + + // Write traceId so the client knows whether feedback is supported. + writer.write({ type: 'data-traceId', data: traceId }); + }, + onFinish: async ({ responseMessage }) => { + // Store in-memory for ephemeral mode (also useful when DB is available) + storeMessageMeta(responseMessage.id, id, traceId); + + try { + await saveMessages({ + messages: [ + { + id: responseMessage.id, + role: responseMessage.role, + parts: responseMessage.parts, + createdAt: new Date(), + attachments: [], + chatId: id, + traceId, // Store trace ID for feedback + }, + ], + }); + } catch (err) { + console.error('[onFinish] Failed to save assistant message:', err); + } + + if (finalUsage) { + try { + await updateChatLastContextById({ + chatId: id, + context: toV3Usage(finalUsage), + }); + } catch (err) { + console.warn('Unable to persist last usage for chat', id, err); + } + } + + streamCache.clearActiveStream(id); + }, + }); + + pipeUIMessageStreamToResponse({ + stream, + response: res, + consumeSseStream({ stream }) { + streamCache.storeStream({ + streamId, + chatId: id, + stream, + }); + }, + }); + } catch (error) { + console.error('[Chat] Caught error in chat API:', { + errorType: error?.constructor?.name, + message: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + error, + }); + + if (error instanceof ChatSDKError) { + const response = error.toResponse(); + return res.status(response.status).json(response.json); + } + + const chatError = new ChatSDKError('offline:chat'); + const response = chatError.toResponse(); + return res.status(response.status).json(response.json); + } +}); + +/** + * DELETE /api/chat?id=:id - Delete a chat + */ +chatRouter.delete( + '/:id', + [requireAuth, requireChatAccess], + async (req: Request, res: Response) => { + const id = getIdFromRequest(req); + if (!id) return; + + const deletedChat = await deleteChatById({ id }); + return res.status(200).json(deletedChat); + }, +); + +/** + * GET /api/chat/:id + */ + +chatRouter.get( + '/:id', + [requireAuth, requireChatAccess], + async (req: Request, res: Response) => { + const id = getIdFromRequest(req); + if (!id) return; + + const { chat } = await checkChatAccess(id, req.session?.user.id); + + return res.status(200).json(chat); + }, +); + +/** + * GET /api/chat/:id/stream - Resume a stream + */ +chatRouter.get( + '/:id/stream', + [requireAuth], + async (req: Request, res: Response) => { + const chatId = getIdFromRequest(req); + if (!chatId) return; + const cursor = req.headers['x-resume-stream-cursor'] as string; + + console.log(`[Stream Resume] Cursor: ${cursor}`); + + console.log(`[Stream Resume] GET request for chat ${chatId}`); + + // Check if there's an active stream for this chat first + const streamId = streamCache.getActiveStreamId(chatId); + + if (!streamId) { + console.log(`[Stream Resume] No active stream for chat ${chatId}`); + const streamError = new ChatSDKError('empty:stream'); + const response = streamError.toResponse(); + return res.status(response.status).json(response.json); + } + + const { allowed, reason } = await checkChatAccess( + chatId, + req.session?.user.id, + ); + + // If chat doesn't exist in DB, it's a temporary chat from the homepage - allow it + if (reason === 'not_found') { + console.log( + `[Stream Resume] Resuming stream for temporary chat ${chatId} (not yet in DB)`, + ); + } else if (!allowed) { + console.log( + `[Stream Resume] User ${req.session?.user.id} does not have access to chat ${chatId} (reason: ${reason})`, + ); + const streamError = new ChatSDKError('forbidden:chat', reason); + const response = streamError.toResponse(); + return res.status(response.status).json(response.json); + } + + // Get all cached chunks for this stream + const stream = streamCache.getStream(streamId, { + cursor: cursor ? Number.parseInt(cursor) : undefined, + }); + + if (!stream) { + console.log(`[Stream Resume] No stream found for ${streamId}`); + const streamError = new ChatSDKError('empty:stream'); + const response = streamError.toResponse(); + return res.status(response.status).json(response.json); + } + + console.log(`[Stream Resume] Resuming stream ${streamId}`); + + // Set headers for SSE + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + // Pipe the cached stream directly to the response + stream.pipe(res); + + // Handle stream errors + stream.on('error', (error) => { + console.error('[Stream Resume] Stream error:', error); + if (!res.headersSent) { + res.status(500).end(); + } + }); + }, +); + +/** + * POST /api/chat/title - Generate title from message + */ +chatRouter.post('/title', requireAuth, async (req: Request, res: Response) => { + try { + const { message } = req.body; + const title = await generateTitleFromUserMessage({ message }); + res.json({ title }); + } catch (error) { + console.error('Error generating title:', error); + res.status(500).json({ error: 'Failed to generate title' }); + } +}); + +/** + * PATCH /api/chat/:id/visibility - Update chat visibility + */ +chatRouter.patch( + '/:id/visibility', + [requireAuth, requireChatAccess], + async (req: Request, res: Response) => { + try { + const id = getIdFromRequest(req); + if (!id) return; + const { visibility } = req.body; + + if (!visibility || !['public', 'private'].includes(visibility)) { + return res.status(400).json({ error: 'Invalid visibility type' }); + } + + await updateChatVisiblityById({ chatId: id, visibility }); + res.json({ success: true }); + } catch (error) { + console.error('Error updating visibility:', error); + res.status(500).json({ error: 'Failed to update visibility' }); + } + }, +); + +// Helper function to generate title from user message +async function generateTitleFromUserMessage({ + message, + maxMessageLength = 256, +}: { + message: ChatMessage; + maxMessageLength?: number; +}) { + const model = await myProvider.languageModel('title-model'); + + // Truncate each text part to the maxMessageLength + const truncatedMessage = { + ...message, + parts: message.parts.map((part) => + part.type === 'text' + ? { ...part, text: part.text.slice(0, maxMessageLength) } + : part, + ), + }; + + const { text: title } = await generateText({ + model, + system: `\n + - you will generate a short title based on the first message a user begins a conversation with + - ensure it is not more than 80 characters long + - the title should be a summary of the user's message + - do not use quotes or colons. do not include other expository content ("I'll help...")`, + prompt: JSON.stringify(truncatedMessage), + }); + + return title; +} + +function truncatePreserveWords(input: string, maxLength: number): string { + if (maxLength <= 0) return ''; + if (input.length <= maxLength) return input; + + // Take the raw slice first + const slice = input.slice(0, maxLength); + + // Find the last whitespace within the slice + const lastSpaceIndex = slice.lastIndexOf(' '); + + // If no whitespace found, we must break mid-word + if (lastSpaceIndex === -1) { + return slice; + } + + // If the whitespace is too close to the start (e.g., leading space), + // fallback to mid-word break to avoid returning an empty string + if (lastSpaceIndex === 0) { + return slice; + } + + return slice.slice(0, lastSpaceIndex); +} + diff --git a/agent/patches/e2e-chatbot-app-next/server/src/routes/config.ts b/agent/patches/e2e-chatbot-app-next/server/src/routes/config.ts new file mode 100644 index 0000000..ca0ca42 --- /dev/null +++ b/agent/patches/e2e-chatbot-app-next/server/src/routes/config.ts @@ -0,0 +1,93 @@ +import { + Router, + type Request, + type Response, + type Router as RouterType, +} from 'express'; +import { isDatabaseAvailable } from '@chat-template/db'; +import { getEndpointOboInfo } from '@chat-template/ai-sdk-providers'; + +export const configRouter: RouterType = Router(); + +/** + * Extract OAuth scopes from a JWT token (without verification). + * Databricks tokens use 'scope' (space-separated string) or 'scp' (array). + */ +function getScopesFromToken(token: string): string[] { + try { + const parts = token.split('.'); + if (parts.length !== 3) return []; + const payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf-8')); + if (typeof payload.scope === 'string') return payload.scope.split(' '); + if (Array.isArray(payload.scp)) return payload.scp as string[]; + return []; + } catch { + return []; + } +} + +function workspaceHostFromEnv(): string { + const host = process.env.DATABRICKS_HOST?.trim(); + if (host) { + return host.replace(/\/$/, ''); + } + const mcpUrl = process.env.GENIE_MCP_URL?.trim(); + if (!mcpUrl) { + return ''; + } + try { + return new URL(mcpUrl).origin; + } catch { + return ''; + } +} + +/** + * No attribution URL is exposed. The panel's audience is guest users with no + * Databricks workspace access, so any workspace link is dead for them — and once + * GENIE_MCP_MODE defaults to a space, a "Genie Agent" link names a backend that + * never answered the question. The UI shows plain-text attribution instead. + */ +function genieConfigFromEnv() { + const workspaceHost = workspaceHostFromEnv(); + const workspaceId = process.env.DATABRICKS_WORKSPACE_ID?.trim(); + if (!workspaceHost) { + return null; + } + return { + workspaceHost, + workspaceId: workspaceId || undefined, + }; +} + +/** + * GET /api/config - Get application configuration + * Returns feature flags and OBO status based on environment configuration. + * If the user's OBO token is present, decodes it to check which required + * scopes are missing — the banner only shows missing scopes. + */ +configRouter.get('/', async (req: Request, res: Response) => { + const oboInfo = await getEndpointOboInfo(); + + let missingScopes = oboInfo.endpointRequiredScopes; + + const userToken = req.headers['x-forwarded-access-token'] as string | undefined; + if (userToken && oboInfo.isEndpointOboEnabled) { + const tokenScopes = getScopesFromToken(userToken); + missingScopes = oboInfo.endpointRequiredScopes.filter(required => { + const parent = required.split('.')[0]; + return !tokenScopes.some(ts => ts === required || ts === parent); + }); + } + + res.json({ + features: { + chatHistory: isDatabaseAvailable(), + feedback: !!process.env.MLFLOW_EXPERIMENT_ID, + }, + obo: { + missingScopes, + }, + genie: genieConfigFromEnv(), + }); +}); diff --git a/agent/scripts/deploy_agent.sh b/agent/scripts/deploy_agent.sh new file mode 100755 index 0000000..24c766c --- /dev/null +++ b/agent/scripts/deploy_agent.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# One-command agent deploy: create -> run -> enable memory. +# +# What actually needs an ordering here (proven end-to-end on a fresh workspace): +# +# * Frontend Drizzle migrations DO run at the first container build, but they do +# NOT need a manual grant. The bundle binds the Postgres/Lakebase resource with +# CAN_CONNECT_AND_CREATE (agent/databricks.yml), which is applied at `bundle +# deploy` and auto-provisions the app SP's Postgres login role. The migrations +# connect as that SP, CREATE their schema/tables (SP becomes OWNER -> full DML), +# and succeed. A no-grant deploy comes up RUNNING with ai_chatbot.{Chat,Message, +# Vote} present. (This is why the old "grant Lakebase before build" step is gone.) +# +# * The durable, cross-session MEMORY feature is a Unity Catalog *memory store*, +# not Lakebase. It must (a) EXIST as a UC securable and (b) grant the app SP +# READ/WRITE — neither of which quickstart or the bundle does. Without it the +# agent silently no-ops ("memory store not found" -> "does not have READ/WRITE +# MEMORY STORE"). scripts/setup_memory_store.py does both; it needs the SP, which +# exists only after the app is created, so it runs after deploy. +# +# Idempotent — safe to re-run. +# +# Usage (from agent-build/, after assemble_agent.sh + quickstart): +# bash scripts/deploy_agent.sh [extra bundle args, e.g. --var catalog=main] +set -euo pipefail +cd "$(dirname "$0")/.." # agent-build root + +PROFILE="${1:?usage: deploy_agent.sh [--var ...]}"; shift || true +EXTRA=("$@") # forwarded to bundle deploy/run (e.g. --var catalog=main) +# ${EXTRA[@]+"${EXTRA[@]}"} safely expands a possibly-empty array under `set -u` +# (bare "${EXTRA[@]}" errors "unbound variable" on macOS/older bash when empty). +APP_KEY="agent_openai_agents_sdk" + +APP="$(databricks bundle summary -o json -p "$PROFILE" ${EXTRA[@]+"${EXTRA[@]}"} | jq -r ".resources.apps.${APP_KEY}.name")" +[ -n "$APP" ] && [ "$APP" != "null" ] || { echo "ERROR: could not resolve app name from bundle summary"; exit 1; } +echo "==> App: $APP (profile: $PROFILE)" + +# 0) Ensure vendored wheels exist so the build installs offline (skip if already done). +if [ -z "$(ls -A vendor-wheels 2>/dev/null || true)" ]; then + echo "==> vendor-wheels/ empty — vendoring build wheels" + bash scripts/vendor_wheels.sh +fi + +# 1) Create/update the app + upload source, then build + start. +echo "==> bundle deploy (creates app + SP, uploads source)" +databricks bundle deploy -p "$PROFILE" ${EXTRA[@]+"${EXTRA[@]}"} +echo "==> bundle run (build + start; frontend migrations succeed via the resource binding)" +databricks bundle run "$APP_KEY" -p "$PROFILE" ${EXTRA[@]+"${EXTRA[@]}"} + +# 2) Resolve the app SP (exists after create). +SP="$(databricks apps get "$APP" -p "$PROFILE" --output json | jq -r '.service_principal_client_id // empty')" +[ -n "$SP" ] || { echo "ERROR: failed to obtain the app's service principal"; exit 1; } +echo "==> App service principal: $SP" + +# 3) Enable managed memory: create the UC memory store + grant the SP READ/WRITE. +# The store name is whatever the app runs with (DATABRICKS_MEMORY_STORE, built from +# the bundle's catalog/schema vars). setup_memory_store.py reads it from the env, +# so pass the same catalog/schema overrides you passed to bundle (if any). +STORE="$(databricks bundle summary -o json -p "$PROFILE" ${EXTRA[@]+"${EXTRA[@]}"} \ + | jq -r ".resources.apps.${APP_KEY}.config.env[]? | select(.name==\"DATABRICKS_MEMORY_STORE\") | .value")" +echo "==> Enabling managed memory on store: ${STORE:-}" +uv run --python 3.12 python scripts/setup_memory_store.py "$SP" \ + ${STORE:+--memory-store "$STORE"} --profile "$PROFILE" + +# 4) Assert health from the LIVE app state (PAT-readable). NOTE: judge health from +# app_status.state, NOT active_deployment.status.state — the latter goes green when +# the container command starts, before the port binds (see start_app.py). +echo "==> Verifying live app state (PAT-readable)" +STATE="$(databricks apps get "$APP" -p "$PROFILE" --output json | jq -r '.app_status.state // "?"')" +echo " app_status.state = $STATE" +case "$STATE" in + RUNNING) + echo " ✓ healthy — point DATABRICKS_AGENT_APP_URL at the app URL." ;; + *) + echo " ✗ not RUNNING yet. Confirm from the runtime logs (needs an OAuth profile):" + echo " databricks apps logs $APP -p # wait for 'Both frontend and backend are ready!'" + exit 1 ;; +esac diff --git a/agent/scripts/setup_memory_store.py b/agent/scripts/setup_memory_store.py new file mode 100644 index 0000000..0e37853 --- /dev/null +++ b/agent/scripts/setup_memory_store.py @@ -0,0 +1,96 @@ +"""Create the UC managed-memory store and grant the app's service principal on it. + +This is what actually makes the durable, cross-session memory feature work — NOT +`grant_lakebase_permissions.py` (that grants Postgres/Lakebase table privileges, +which this Genie-One + UC-memory topology never uses). The memory tools in +`agent_server/utils_memory.py` talk to the Unity Catalog memory-store API +(`/api/2.1/unity-catalog/memory-stores//entries`), so the +store must EXIST as a UC securable and the app's SP must hold READ/WRITE on it. +Neither is created by `quickstart` or the bundle, so without this step the agent +silently no-ops memory ("memory store not found", then "does not have READ/WRITE +MEMORY STORE"). Idempotent — safe to re-run. + +Usage (from agent-build/, after the app exists so its SP is known): + SP=$(databricks apps get -p --output json | jq -r .service_principal_client_id) + uv run --python 3.12 python scripts/setup_memory_store.py "$SP" \ + --memory-store workspace.default.firefly_managed_memory --profile + +`--memory-store` defaults to $DATABRICKS_MEMORY_STORE (the value the app runs with). +""" + +import argparse +import os +import sys + +from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError + +MEMORY_STORE_API = "/api/2.1/unity-catalog/memory-stores" +PERMISSIONS_API = "/api/2.1/unity-catalog/permissions/memory_store" +PRIVILEGES = ["READ_MEMORY_STORE", "WRITE_MEMORY_STORE"] + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Create the UC memory store and grant the app SP READ/WRITE on it." + ) + parser.add_argument( + "sp_client_id", + help="App service principal client ID (UUID). Get it via: " + "databricks apps get --output json | jq -r '.service_principal_client_id'", + ) + parser.add_argument( + "--memory-store", + default=os.getenv("DATABRICKS_MEMORY_STORE"), + help="Fully-qualified store name catalog.schema.name " + "(default: DATABRICKS_MEMORY_STORE from env).", + ) + parser.add_argument( + "--profile", + default=None, + help="Databricks CLI profile to authenticate with (optional).", + ) + args = parser.parse_args() + + if not args.memory_store or args.memory_store.count(".") != 2: + print( + "Error: --memory-store must be a fully-qualified catalog.schema.name " + f"(got: {args.memory_store!r}). Set DATABRICKS_MEMORY_STORE or pass --memory-store.", + file=sys.stderr, + ) + sys.exit(1) + + catalog, schema, name = args.memory_store.split(".") + w = WorkspaceClient(profile=args.profile) if args.profile else WorkspaceClient() + + # 1) Create the store (idempotent). + print(f"==> Ensuring memory store {args.memory_store} exists") + try: + w.api_client.do( + "POST", + MEMORY_STORE_API, + body={"catalog_name": catalog, "schema_name": schema, "name": name}, + ) + print(" created.") + except DatabricksError as e: + if getattr(e, "error_code", "") == "ALREADY_EXISTS" or "already exists" in str(e).lower(): + print(" already exists, skipping.") + else: + raise + + # 2) Grant the app SP READ + WRITE on the store. + print(f"==> Granting {PRIVILEGES} to SP {args.sp_client_id}") + w.api_client.do( + "PATCH", + f"{PERMISSIONS_API}/{args.memory_store}", + body={"changes": [{"principal": args.sp_client_id, "add": PRIVILEGES}]}, + ) + + # 3) Read back for confirmation. + perms = w.api_client.do("GET", f"{PERMISSIONS_API}/{args.memory_store}") + print(f"==> Current grants: {perms.get('privilege_assignments', [])}") + print("==> Done. The agent's save_memory/get_memory tools can now persist per user.") + + +if __name__ == "__main__": + main() diff --git a/agent/scripts/start_app.py b/agent/scripts/start_app.py new file mode 100644 index 0000000..890c909 --- /dev/null +++ b/agent/scripts/start_app.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +""" +Start script for running frontend and backend processes concurrently. + +Requirements: +1. Not reporting ready until BOTH frontend and backend processes are ready +2. Exiting as soon as EITHER process fails +3. Printing error logs if either process fails + +Usage: + start-app [OPTIONS] + +All options are passed through to the backend server (start-server). +See 'uv run start-server --help' for available options. +""" + +import argparse +import os +import re +import shutil +import socket +import subprocess +import sys +import threading +import time +from pathlib import Path + +from dotenv import load_dotenv + +# Readiness patterns +BACKEND_READY = [r"Uvicorn running on", r"Application startup complete", r"Started server process"] +FRONTEND_READY = [r"Server is running on http://localhost"] + + +def backend_command(backend_args: list[str] | None = None) -> list[str]: + """Console script on Apps (pip install); uv locally.""" + if shutil.which("start-server"): + cmd = ["start-server"] + else: + cmd = ["uv", "run", "start-server"] + if backend_args: + cmd.extend(backend_args) + return cmd + + +def check_port_available(port: int) -> bool: + """Check if a port is available (nothing is actively listening on it).""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + s.connect(("localhost", port)) + return False # Something is listening + except (ConnectionRefusedError, OSError): + return True # Nothing listening = available + + +class ProcessManager: + def __init__(self, port=8000, no_ui=False): + self.backend_process = None + self.frontend_process = None + self.backend_ready = False + self.frontend_ready = False + self.failed = threading.Event() + self.backend_log = None + self.frontend_log = None + self.port = port + self.no_ui = no_ui + + def check_ports(self): + """Check that required ports are available before starting processes.""" + backend_port = self.port + + errors = [] + if not check_port_available(backend_port): + errors.append( + f"Port {backend_port} (backend) is already in use.\n" + f" To free it: lsof -ti :{backend_port} | xargs kill -9" + ) + + if not self.no_ui: + frontend_port = int(os.environ.get("CHAT_APP_PORT", os.environ.get("PORT", "3000"))) + + if backend_port == frontend_port: + print( + f"ERROR: Backend and frontend are both configured to use port {backend_port}." + ) + print(" Set CHAT_APP_PORT in .env to a different port (e.g., CHAT_APP_PORT=3000).") + sys.exit(1) + + if not check_port_available(frontend_port): + port_source = ( + "CHAT_APP_PORT" + if os.environ.get("CHAT_APP_PORT") + else "PORT" + if os.environ.get("PORT") + else "default" + ) + errors.append( + f"Port {frontend_port} (frontend, source: {port_source}) is already in use.\n" + f" To free it: lsof -ti :{frontend_port} | xargs kill -9\n" + f" Or set a different port: CHAT_APP_PORT= in .env" + ) + + if errors: + print("ERROR: Port(s) already in use:\n") + for error in errors: + print(f" {error}\n") + sys.exit(1) + + def monitor_process(self, process, name, log_file, patterns): + is_ready = False + try: + for line in iter(process.stdout.readline, ""): + if not line: + break + + line = line.rstrip() + log_file.write(line + "\n") + print(f"[{name}] {line}") + + # Check readiness + if not is_ready and any(re.search(p, line, re.IGNORECASE) for p in patterns): + is_ready = True + if name == "backend": + self.backend_ready = True + else: + self.frontend_ready = True + print(f"✓ {name.capitalize()} is ready!") + + if self.no_ui and self.backend_ready: + print("\n" + "=" * 50) + print("✓ Backend is ready! (running without UI)") + print(f"✓ API available at http://localhost:{self.port}") + print("=" * 50 + "\n") + elif self.backend_ready and self.frontend_ready: + print("\n" + "=" * 50) + print("✓ Both frontend and backend are ready!") + print(f"✓ Open the frontend at http://localhost:{self.port}") + print("=" * 50 + "\n") + + process.wait() + if process.returncode != 0: + self.failed.set() + + except Exception as e: + print(f"Error monitoring {name}: {e}") + self.failed.set() + + def apply_chat_ui_patches(self): + """Overlay Firefly-specific chat UI fixes onto the cloned template.""" + patch_root = Path(__file__).resolve().parent.parent / "patches" / "e2e-chatbot-app-next" + target_root = Path("e2e-chatbot-app-next") + if not patch_root.is_dir() or not target_root.is_dir(): + return + for src in patch_root.rglob("*"): + if not src.is_file(): + continue + rel = src.relative_to(patch_root) + dest = target_root / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + print("Applied chat UI patches from patches/e2e-chatbot-app-next") + + def clone_frontend_if_needed(self): + if Path("e2e-chatbot-app-next").exists(): + self.apply_chat_ui_patches() + return True + + print("Cloning e2e-chatbot-app-next...") + for url in [ + "https://github.com/databricks/app-templates.git", + "git@github.com:databricks/app-templates.git", + ]: + try: + subprocess.run( + ["git", "clone", "--filter=blob:none", "--sparse", url, "temp-app-templates"], + check=True, + capture_output=True, + ) + break + except subprocess.CalledProcessError: + continue + else: + print("ERROR: Failed to clone repository.") + print( + "Manually download from: https://download-directory.github.io/?url=https://github.com/databricks/app-templates/tree/main/e2e-chatbot-app-next" + ) + return False + + subprocess.run( + ["git", "sparse-checkout", "set", "e2e-chatbot-app-next"], + cwd="temp-app-templates", + check=True, + ) + Path("temp-app-templates/e2e-chatbot-app-next").rename("e2e-chatbot-app-next") + shutil.rmtree("temp-app-templates", ignore_errors=True) + self.apply_chat_ui_patches() + return True + + def start_process(self, cmd, name, log_file, patterns, cwd=None, env=None): + print(f"Starting {name}...") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + cwd=cwd, + env=env, + ) + + thread = threading.Thread( + target=self.monitor_process, args=(process, name, log_file, patterns), daemon=True + ) + thread.start() + return process + + def print_logs(self, log_path): + print(f"\nLast 50 lines of {log_path}:") + print("-" * 40) + try: + lines = Path(log_path).read_text().splitlines() + print("\n".join(lines[-50:])) + except FileNotFoundError: + print(f"(no {log_path} found)") + print("-" * 40) + + def cleanup(self): + print("\n" + "=" * 42) + print("Shutting down..." if self.no_ui else "Shutting down both processes...") + print("=" * 42) + + for proc in [self.backend_process, self.frontend_process]: + if proc: + try: + proc.terminate() + proc.wait(timeout=5) + except (subprocess.TimeoutExpired, Exception): + proc.kill() + + if self.backend_log: + self.backend_log.close() + if self.frontend_log: + self.frontend_log.close() + + def run(self, backend_args=None): + load_dotenv(dotenv_path=Path(__file__).parent.parent / ".env", override=True) + if not os.environ.get("DATABRICKS_APP_NAME"): + self.check_ports() + + if not self.no_ui: + if not self.clone_frontend_if_needed(): + print("WARNING: Failed to clone frontend. Continuing with backend only.") + self.no_ui = True + else: + # Set API_PROXY environment variable for frontend to connect to backend + os.environ["API_PROXY"] = f"http://localhost:{self.port}/invocations" + + # Open log files + self.backend_log = open("backend.log", "w", buffering=1) + if not self.no_ui: + self.frontend_log = open("frontend.log", "w", buffering=1) + + try: + # Build the frontend FIRST — `npm run build` runs `db:migrate` (Drizzle). + # Do this BEFORE the backend binds the platform port so a build/migration + # failure takes the WHOLE container down (nothing binds :8000) instead of + # leaving the backend serving while the frontend crash-loops behind it — + # which would keep the live `app_status` misleadingly RUNNING (GAP-17). + # NOTE: this does NOT change the deploy-time state: the platform marks the + # deployment SUCCEEDED when this command STARTS, not when the port binds, + # so `active_deployment.status.state` is green regardless — always judge + # health from the LIVE `app_status.state` (or the runtime logs), not the + # deployment status. The build talks to Lakebase directly (not the + # backend), so it has no ordering dependency on the backend being up. + frontend_dir = Path("e2e-chatbot-app-next") + if not self.no_ui: + for cmd, desc in [("npm install", "install"), ("npm run build", "build")]: + print(f"Running npm {desc}...") + result = subprocess.run( + cmd.split(), cwd=frontend_dir, capture_output=True, text=True + ) + if result.returncode != 0: + print(f"npm {desc} failed: {result.stderr}") + return 1 + + backend_cmd = backend_command(backend_args) + backend_env = os.environ.copy() + if self.no_ui: + backend_env["ENABLE_CHAT_PROXY"] = "false" + + # Start backend — binds the platform port only now, after a healthy build. + self.backend_process = self.start_process( + backend_cmd, "backend", self.backend_log, BACKEND_READY, env=backend_env + ) + + if not self.no_ui: + # Frontend already built above; just start its server. + self.frontend_process = self.start_process( + ["npm", "run", "start"], + "frontend", + self.frontend_log, + FRONTEND_READY, + cwd=frontend_dir, + ) + + print( + f"\nMonitoring processes (Backend PID: {self.backend_process.pid}, Frontend PID: {self.frontend_process.pid})\n" + ) + else: + print(f"\nMonitoring backend process (PID: {self.backend_process.pid})\n") + + # Wait for failure + while not self.failed.is_set(): + time.sleep(0.1) + if self.backend_process.poll() is not None: + self.failed.set() + break + if ( + not self.no_ui + and self.frontend_process + and self.frontend_process.poll() is not None + ): + self.failed.set() + break + + # Determine which failed + if self.no_ui or self.backend_process.poll() is not None: + failed_name = "backend" + failed_proc = self.backend_process + else: + failed_name = "frontend" + failed_proc = self.frontend_process + exit_code = failed_proc.returncode if failed_proc else 1 + + print( + f"\n{'=' * 42}\nERROR: {failed_name} process exited with code {exit_code}\n{'=' * 42}" + ) + self.print_logs("backend.log") + if not self.no_ui: + self.print_logs("frontend.log") + return exit_code + + except KeyboardInterrupt: + print("\nInterrupted") + return 0 + + finally: + self.cleanup() + + +def main(): + parser = argparse.ArgumentParser( + description="Start agent frontend and backend", + usage="%(prog)s [OPTIONS]\n\nAll options are passed through to start-server. " + "Use 'uv run start-server --help' for available options.", + ) + parser.add_argument( + "--no-ui", + action="store_true", + help="Run backend only, skip frontend UI", + ) + args, backend_args = parser.parse_known_args() + + # Extract port from backend_args if specified + port = 8000 + for i, arg in enumerate(backend_args): + if arg == "--port" and i + 1 < len(backend_args): + try: + port = int(backend_args[i + 1]) + except ValueError: + pass + break + + sys.exit(ProcessManager(port=port, no_ui=args.no_ui).run(backend_args)) + + +if __name__ == "__main__": + main() diff --git a/agent/scripts/vendor_wheels.sh b/agent/scripts/vendor_wheels.sh new file mode 100755 index 0000000..141f82f --- /dev/null +++ b/agent/scripts/vendor_wheels.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Pre-fetch linux cp311 wheels so the Databricks Apps build never depends on the +# build container's PyPI egress (dead .dev host / lagging .cloud mirror / no offline +# fallback). Produces vendor-wheels/ that the build installs from via UV_FIND_LINKS. +# +# Run from agent-build/ (after assemble_agent.sh) BEFORE `databricks bundle deploy`. +set -euo pipefail +cd "$(dirname "$0")/.." # agent-build root + +PY=3.11 # MUST match the Apps runtime (cp311), not 3.12 +MAXBYTES=10485760 # workspace-files per-file import cap (10 MB) + +echo "==> Locking + exporting requirements for Python ${PY}" +uv lock --python "${PY}" + +# Deterministic pins for the whole dependency graph (#64). The Apps build runs a plain +# `uv sync` (uv.lock is excluded from sync — GAP-15), which can otherwise re-resolve a +# transitive dep to a newer release with no Linux wheel (the greenlet 3.5.4 crash). +# UV_CONSTRAINT *bounds* versions without the exact-match rigidity that made +# `uv sync --locked` fail. agent/databricks.yml points UV_CONSTRAINT at this file. +# +# This lives here, not in assemble_agent.sh, because this is the first point at which +# agent-build/uv.lock exists: the template ships no lock and assemble starts by deleting +# the build directory. Exported from the universal lock with environment markers, which +# uv applies on the Linux/cp311 Apps host at install time. Non-fatal: absent file → uv +# ignores it, and the #63 greenlet override still guards the known crash. +if uv export --frozen --no-hashes --no-emit-project -o constraints.txt 2>/dev/null \ + && [[ -s constraints.txt ]]; then + echo "==> Wrote constraints.txt ($(grep -c '==' constraints.txt) pins) → UV_CONSTRAINT (#64)" +else + rm -f constraints.txt + echo "==> WARN: 'uv export' failed; relying on the greenlet override only (#63)." >&2 +fi + +uv export --python "${PY}" --no-hashes --format requirements-txt --no-emit-project \ + | grep -v "python_full_version >= '3.12'" \ + | grep -v "sys_platform == 'win32'" \ + | sed -E 's/[[:space:]]*;.*$//' | sort -u > .vendor-req.txt + +echo "==> Downloading linux cp311 wheels" +rm -rf vendor-wheels && mkdir -p vendor-wheels +python3 -m pip download -r .vendor-req.txt \ + --platform manylinux2014_x86_64 --platform manylinux_2_17_x86_64 \ + --platform manylinux_2_28_x86_64 --platform manylinux_2_27_x86_64 \ + --python-version "${PY}" --implementation cp --abi cp311 --abi none --abi abi3 \ + --only-binary=:all: --dest vendor-wheels + +# Workspace files reject files > 10 MB; keep only small wheels in the synced source. +# The few large compiled wheels (pyarrow/scipy/numpy/pandas/mlflow) install from the +# index at build time (stable versions, present on the build mirror). +echo "==> Dropping wheels > 10 MB (installed from index instead):" +# BSD/macOS + GNU find compatible: print then delete +find vendor-wheels -name '*.whl' -size +"${MAXBYTES}"c -print -delete | sed 's#vendor-wheels/# #' +echo "==> Vendored $(ls vendor-wheels | wc -l | tr -d ' ') wheels (<=10 MB) into vendor-wheels/" diff --git a/agent/tests/test_genie_backends.py b/agent/tests/test_genie_backends.py new file mode 100644 index 0000000..6b077bc --- /dev/null +++ b/agent/tests/test_genie_backends.py @@ -0,0 +1,213 @@ +"""The two Genie backends are not interchangeable, and nothing checked that. + +A real bootstrap run deployed space-scoped Genie (the default), the agent POSTed the +workspace-wide tool name `genie_ask` to the space endpoint, and the server answered +`-32602 BAD_REQUEST: Tool genie_ask does not exist`. The user saw `Genie ask failed: {}`. + +Nine consecutive end-to-end runs had passed before that. They asserted the *configuration* +-- mode=space, a real space id, CAN_RUN granted, tables seeded -- and never asked the agent +a question, so every probe was green while the one path a user exercises was broken. These +tests cover the seam those probes could not see: + + workspace-wide -> genie_ask(question=...), genie_poll_response(response_id=...) + space-scoped -> query_space_(query=...), poll_response_(message_id=...) + +Tool names, argument names, id casing, status vocabulary and answer location all differ. + +Run: cd agent && python3 -m pytest tests/test_genie_backends.py -q +""" + +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# genie_tools imports `function_tool` from the OpenAI Agents SDK, which is a runtime +# dependency of the deployed app and not needed to test this logic. Stub it so the suite +# runs on a clean checkout: requiring the SDK here would mean these tests only run where +# the app already builds, which is precisely where they are least needed. +if "agents" not in sys.modules: + _agents = type(sys)("agents") + _agents.function_tool = lambda fn=None, **kw: (fn if fn is not None else (lambda f: f)) + sys.modules["agents"] = _agents + +from agent_server import genie_mcp # noqa: E402 +from agent_server import genie_tools # noqa: E402 + +SPACE_ID = "01f18b4e96a61645bbf63e14eb416cb5" + + +def env(**kw): + """Env with the Genie vars cleared unless named, so a stray export cannot pass a test.""" + base = {k: v for k, v in os.environ.items() + if k not in ("GENIE_MCP_MODE", "GENIE_SPACE_ID", "GENIE_MCP_URL")} + base.update(kw) + return mock.patch.dict(os.environ, base, clear=True) + + +def _fake_ws(response): + """Patch the SDK client the tool calls through. + + The HTTP boundary is WorkspaceClient().api_client.do, reached via + _app_workspace_client() -- patching a hand-invented `_mcp_post` made two tests error + out on a name that never existed, which is its own small lesson about asserting + against a real seam instead of an imagined one. + """ + client = mock.MagicMock() + client.api_client.do.return_value = response + return mock.patch.object(genie_tools, "_app_workspace_client", return_value=client) + + +class ToolNames(unittest.TestCase): + def test_space_mode_uses_space_scoped_tools(self): + """The defect verbatim: space mode must NOT ask for genie_ask.""" + with env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=SPACE_ID): + ask, poll = genie_mcp.genie_tool_names() + self.assertEqual(ask, f"query_space_{SPACE_ID}") + self.assertEqual(poll, f"poll_response_{SPACE_ID}") + self.assertNotIn("genie_ask", (ask, poll)) + + def test_space_is_the_default(self): + """No GENIE_MCP_MODE at all must still resolve to the space tools.""" + with env(GENIE_SPACE_ID=SPACE_ID): + ask, _ = genie_mcp.genie_tool_names() + self.assertEqual(ask, f"query_space_{SPACE_ID}") + + def test_agent_alias_behaves_as_space(self): + with env(GENIE_MCP_MODE="agent", GENIE_SPACE_ID=SPACE_ID): + ask, _ = genie_mcp.genie_tool_names() + self.assertEqual(ask, f"query_space_{SPACE_ID}") + + def test_workspace_mode_uses_workspace_tools(self): + with env(GENIE_MCP_MODE="one"): + ask, poll = genie_mcp.genie_tool_names() + self.assertEqual((ask, poll), ("genie_ask", "genie_poll_response")) + + def test_tool_names_and_path_agree_on_the_backend(self): + """A path pointing at a space while the names say workspace-wide is the bug.""" + with env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=SPACE_ID): + path = genie_mcp.genie_mcp_path() + ask, _ = genie_mcp.genie_tool_names() + self.assertTrue(path.endswith(SPACE_ID), path) + self.assertIn(SPACE_ID, ask) + + def test_space_mode_without_id_raises_rather_than_falling_back(self): + for value in ("", "none", "NONE", "null"): + with self.subTest(space_id=value), env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=value): + with self.assertRaises(ValueError): + genie_mcp.genie_tool_names() + + +class ArgumentNames(unittest.TestCase): + """Right tool, wrong argument name still fails -- as an invalid-argument error that + mentions nothing about Genie. Assert the wire call, not just the tool name.""" + + def _capture(self, question="how many bookings?"): + calls = [] + + def fake(name, arguments): + calls.append((name, arguments)) + return {"error": "stop here"} # short-circuit; ask/poll shape is tested below + + with mock.patch.object(genie_tools, "_mcp_tool_call", side_effect=fake): + genie_tools.ask_genie(question) + return calls[0] + + def test_space_mode_sends_query(self): + with env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=SPACE_ID): + name, args = self._capture() + self.assertEqual(name, f"query_space_{SPACE_ID}") + self.assertIn("query", args) + self.assertNotIn("question", args) + + def test_workspace_mode_sends_question(self): + with env(GENIE_MCP_MODE="one"): + name, args = self._capture() + self.assertEqual(name, "genie_ask") + self.assertIn("question", args) + self.assertNotIn("query", args) + + +class ErrorPropagation(unittest.TestCase): + """`return {}` on a JSON-RPC error is what turned a precise server message into + 'Genie ask failed: {}' -- the same discarded-error pattern the runbook forbids.""" + + def test_jsonrpc_error_message_survives(self): + raw = {"error": {"code": -32602, + "message": "BAD_REQUEST: Tool genie_ask does not exist"}} + with env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=SPACE_ID), \ + _fake_ws(raw): + out = genie_tools._mcp_tool_call("query_space_x", {"query": "q"}) + self.assertIn("error", out) + self.assertIn("does not exist", out["error"]) + self.assertNotEqual(out, {}) + + def test_user_visible_failure_names_the_cause(self): + raw = {"error": {"code": -32602, + "message": "BAD_REQUEST: Tool genie_ask does not exist"}} + with env(GENIE_MCP_MODE="space", GENIE_SPACE_ID=SPACE_ID), \ + _fake_ws(raw): + answer = genie_tools.ask_genie("how many bookings?") + self.assertNotIn("{}", answer) + self.assertTrue(len(answer) > 20, f"uninformative failure text: {answer!r}") + + +class SpaceResponseShape(unittest.TestCase): + """Space mode has no final_answer: the answer is in content, ids are camelCase, and + statuses are uppercase. Reading it with the workspace-wide shape yields a silent blank.""" + + COMPLETED = { + "conversationId": "c-1", + "messageId": "m-1", + "status": "COMPLETED", + "content": { + "queryAttachments": [{ + "description": "Total bookings and users", + "query": "SELECT count(*) FROM bookings", + "statement_response": { + "manifest": {"schema": {"columns": [{"name": "bookings"}, + {"name": "users"}]}}, + "result": {"data_array": [["72247", "124509"]]}, + }, + }] + }, + } + + def test_completed_space_response_yields_an_answer(self): + out = genie_tools._normalize(self.COMPLETED) + self.assertEqual(out["status"], "completed") + self.assertEqual(out["conversation_id"], "c-1") + self.assertEqual(out["response_id"], "m-1") + self.assertIn("72247", out["final_answer"]) + self.assertIn("SELECT", out["final_answer"]) + + def test_json_array_row_shape_also_renders(self): + payload = dict(self.COMPLETED) + payload["content"] = {"queryAttachments": [{ + "query": "SELECT 1", + "statement_response": { + "manifest": {"schema": {"columns": [{"name": "n"}]}}, + "result": {"data_array": [{"values": [{"string_value": "7"}]}]}, + }, + }]} + self.assertIn("7", genie_tools._normalize(payload)["final_answer"]) + + def test_uppercase_states_map_correctly(self): + for raw, want in (("COMPLETED", "completed"), ("FAILED", "failed"), + ("CANCELLED", "failed"), ("QUERY_RESULT_EXPIRED", "failed"), + ("IN_PROGRESS", "in_progress"), ("EXECUTING_QUERY", "in_progress")): + with self.subTest(state=raw): + out = genie_tools._normalize({"conversationId": "c", "messageId": "m", + "status": raw}) + self.assertEqual(out["status"], want) + + def test_workspace_shape_passes_through_untouched(self): + ws = {"conversation_id": "c", "response_id": "r", "status": "completed", + "final_answer": "42"} + self.assertEqual(genie_tools._normalize(ws), ws) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/databricks-apps/guest-manager/uv.lock b/databricks-apps/guest-manager/uv.lock index 3957844..d66d3bd 100644 --- a/databricks-apps/guest-manager/uv.lock +++ b/databricks-apps/guest-manager/uv.lock @@ -13,7 +13,7 @@ resolution-markers = [ [[package]] name = "altair" version = "6.0.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "jinja2" }, { name = "jsonschema" }, @@ -29,7 +29,7 @@ wheels = [ [[package]] name = "attrs" version = "26.1.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, @@ -38,7 +38,7 @@ wheels = [ [[package]] name = "blinker" version = "1.9.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, @@ -47,7 +47,7 @@ wheels = [ [[package]] name = "cachetools" version = "7.0.5" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/af/dd/57fe3fdb6e65b25a5987fd2cdc7e22db0aef508b91634d2e57d22928d41b/cachetools-7.0.5.tar.gz", hash = "sha256:0cd042c24377200c1dcd225f8b7b12b0ca53cc2c961b43757e774ebe190fd990", size = 37367, upload-time = "2026-03-09T20:51:29.451Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/06/f3/39cf3367b8107baa44f861dc802cbf16263c945b62d8265d36034fc07bea/cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114", size = 13918, upload-time = "2026-03-09T20:51:27.33Z" }, @@ -56,7 +56,7 @@ wheels = [ [[package]] name = "certifi" version = "2026.2.25" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, @@ -65,7 +65,7 @@ wheels = [ [[package]] name = "charset-normalizer" version = "3.4.7" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, @@ -154,7 +154,7 @@ wheels = [ [[package]] name = "click" version = "8.3.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] @@ -166,7 +166,7 @@ wheels = [ [[package]] name = "colorama" version = "0.4.6" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "gitdb" version = "4.0.12" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "smmap" }, ] @@ -187,7 +187,7 @@ wheels = [ [[package]] name = "gitpython" version = "3.1.46" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "gitdb" }, ] @@ -216,7 +216,7 @@ requires-dist = [ [[package]] name = "idna" version = "3.11" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, @@ -225,7 +225,7 @@ wheels = [ [[package]] name = "jinja2" version = "3.1.6" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "markupsafe" }, ] @@ -237,7 +237,7 @@ wheels = [ [[package]] name = "jsonschema" version = "4.26.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "jsonschema-specifications" }, @@ -252,7 +252,7 @@ wheels = [ [[package]] name = "jsonschema-specifications" version = "2025.9.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "referencing" }, ] @@ -264,7 +264,7 @@ wheels = [ [[package]] name = "markupsafe" version = "3.0.3" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, @@ -338,7 +338,7 @@ wheels = [ [[package]] name = "narwhals" version = "2.18.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/59/96/45218c2fdec4c9f22178f905086e85ef1a6d63862dcc3cd68eb60f1867f5/narwhals-2.18.1.tar.gz", hash = "sha256:652a1fcc9d432bbf114846688884c215f17eb118aa640b7419295d2f910d2a8b", size = 620578, upload-time = "2026-03-24T15:11:25.456Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3f/c3/06490e98393dcb4d6ce2bf331a39335375c300afaef526897881fbeae6ab/narwhals-2.18.1-py3-none-any.whl", hash = "sha256:a0a8bb80205323851338888ba3a12b4f65d352362c8a94be591244faf36504ad", size = 444952, upload-time = "2026-03-24T15:11:23.801Z" }, @@ -347,7 +347,7 @@ wheels = [ [[package]] name = "numpy" version = "2.4.4" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ef/c6/4218570d8c8ecc9704b5157a3348e486e84ef4be0ed3e38218ab473c83d2/numpy-2.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db", size = 16976799, upload-time = "2026-03-29T13:18:15.438Z" }, @@ -426,7 +426,7 @@ wheels = [ [[package]] name = "packaging" version = "26.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, @@ -435,7 +435,7 @@ wheels = [ [[package]] name = "pandas" version = "3.0.2" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, @@ -495,7 +495,7 @@ wheels = [ [[package]] name = "pillow" version = "12.2.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, @@ -582,7 +582,7 @@ wheels = [ [[package]] name = "protobuf" version = "7.34.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/6b/6b/a0e95cad1ad7cc3f2c6821fcab91671bd5b78bd42afb357bb4765f29bc41/protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280", size = 454708, upload-time = "2026-03-20T17:34:47.036Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/11/3325d41e6ee15bf1125654301211247b042563bcc898784351252549a8ad/protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7", size = 429247, upload-time = "2026-03-20T17:34:37.024Z" }, @@ -597,7 +597,7 @@ wheels = [ [[package]] name = "pyarrow" version = "23.0.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, @@ -647,7 +647,7 @@ wheels = [ [[package]] name = "pydeck" version = "0.9.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "jinja2" }, { name = "numpy" }, @@ -660,7 +660,7 @@ wheels = [ [[package]] name = "python-dateutil" version = "2.9.0.post0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "six" }, ] @@ -672,7 +672,7 @@ wheels = [ [[package]] name = "python-dotenv" version = "1.2.2" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, @@ -681,7 +681,7 @@ wheels = [ [[package]] name = "referencing" version = "0.37.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, @@ -695,7 +695,7 @@ wheels = [ [[package]] name = "requests" version = "2.33.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, @@ -710,7 +710,7 @@ wheels = [ [[package]] name = "rpds-py" version = "0.30.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, @@ -818,7 +818,7 @@ wheels = [ [[package]] name = "six" version = "1.17.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, @@ -827,7 +827,7 @@ wheels = [ [[package]] name = "smmap" version = "5.0.3" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, @@ -836,7 +836,7 @@ wheels = [ [[package]] name = "streamlit" version = "1.56.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "altair" }, { name = "blinker" }, @@ -865,7 +865,7 @@ wheels = [ [[package]] name = "tenacity" version = "9.1.4" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, @@ -874,7 +874,7 @@ wheels = [ [[package]] name = "toml" version = "0.10.2" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, @@ -883,7 +883,7 @@ wheels = [ [[package]] name = "tornado" version = "6.5.5" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/f8/f1/3173dfa4a18db4a9b03e5d55325559dab51ee653763bb8745a75af491286/tornado-6.5.5.tar.gz", hash = "sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9", size = 516006, upload-time = "2026-03-10T21:31:02.067Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa", size = 445983, upload-time = "2026-03-10T21:30:44.28Z" }, @@ -900,7 +900,7 @@ wheels = [ [[package]] name = "typing-extensions" version = "4.15.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, @@ -909,7 +909,7 @@ wheels = [ [[package]] name = "tzdata" version = "2025.3" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, @@ -918,7 +918,7 @@ wheels = [ [[package]] name = "urllib3" version = "2.6.3" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, @@ -927,7 +927,7 @@ wheels = [ [[package]] name = "watchdog" version = "6.0.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, diff --git a/package.json b/package.json index 40ee19b..6835ede 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "databricks-but-not", "version": "0.1.0", "private": true, + "packageManager": "pnpm@10.34.5", "scripts": { "dev": "next dev --turbopack -p 3000 -H 0.0.0.0", "build": "next build --turbopack", @@ -61,7 +62,7 @@ "zustand": "^5.0.8" }, "pnpm": { - "onlyBuiltDependencies": ["@tailwindcss/oxide", "esbuild", "sharp", "unrs-resolver"] + "onlyBuiltDependencies": ["@tailwindcss/oxide", "esbuild", "msw", "sharp", "unrs-resolver"] }, "devDependencies": { "@eslint/eslintrc": "^3", diff --git a/public/solutions/agent/architecture.mermaid b/public/solutions/agent/architecture.mermaid new file mode 100644 index 0000000..fd8172c --- /dev/null +++ b/public/solutions/agent/architecture.mermaid @@ -0,0 +1,45 @@ +sequenceDiagram + participant User + participant Browser as Browser
(Agent panel iframe) + participant NextJS as Next.js Server
(/api/agent-proxy) + participant DB as Database + participant OIDC as Databricks OIDC + participant App as Agent App
(Databricks) + participant Genie as Genie One MCP + participant Memory as Managed Memory
(UC store) + + User->>Browser: Open Agent panel + Browser->>NextJS: GET /api/agent-proxy (same origin) + + rect rgb(240, 248, 255) + Note over NextJS,OIDC: SSO-SPN Token Acquisition (guest supported) + NextJS->>NextJS: Resolve session + active organization + NextJS->>DB: Lookup user's mapped SPN + org workspaceUrl + DB-->>NextJS: SPN credentials (clientId, secret) + NextJS->>OIDC: POST /oidc/v1/token (client_credentials) + OIDC-->>NextJS: Workspace bearer token + end + + rect rgb(255, 248, 240) + Note over NextJS: HTML rewrite (document only) + NextJS->>App: GET / with injected Bearer + App-->>NextJS: index.html + NextJS->>NextJS: Inject base tag + force light theme, no-store + end + + NextJS-->>Browser: Proxied chat UI (assets under /api/agent-proxy) + + rect rgb(240, 255, 240) + Note over Browser,App: Chat request (SSE streaming) + Browser->>NextJS: POST /api/agent-proxy/api/chat + NextJS->>App: Forward buffered body + Bearer + App->>App: Node chat server -> Python agent (/invocations) + App->>Genie: ask_genie_one (genie_ask + poll) + Genie-->>App: Workspace data answer + App->>Memory: read/write long-term memory + Memory-->>App: memory context + App-->>NextJS: SSE token stream + NextJS-->>Browser: SSE token stream + end + + Browser->>User: Streamed answer (Powered by Genie) diff --git a/scripts/assemble_agent.sh b/scripts/assemble_agent.sh new file mode 100755 index 0000000..4bde13c --- /dev/null +++ b/scripts/assemble_agent.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Merge vendor/app-templates (submodule) + agent/ overlay -> agent-build/ (deploy source). +# The submodule is a pristine upstream pin; all Firefly deltas live in agent/. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TEMPLATE="$ROOT/vendor/app-templates/agent-openai-agents-sdk" +CHAT="$ROOT/vendor/app-templates/e2e-chatbot-app-next" +OVERLAY="$ROOT/agent" +BUILD="$ROOT/agent-build" + +[[ -d "$TEMPLATE" ]] || { echo "Missing $TEMPLATE — run: git submodule update --init"; exit 1; } +[[ -d "$CHAT" ]] || { echo "Missing $CHAT — check sparse-checkout"; exit 1; } + +rm -rf "$BUILD" +mkdir -p "$BUILD" + +# 1) upstream agent template +cp -R "$TEMPLATE"/. "$BUILD"/ +# 2) pre-vendor the chat UI so start_app.py doesn't clone at runtime +cp -R "$CHAT" "$BUILD"/e2e-chatbot-app-next +# 3) overlay: our agent_server deltas (agent.py, utils.py, start_server.py, genie_tools.py, utils_memory.py) +cp -R "$OVERLAY"/agent_server/. "$BUILD"/agent_server/ +# 4) overlay: bundle config + startup/build scripts (start_app.py, vendor_wheels.sh, ...) +[[ -f "$OVERLAY/databricks.yml" ]] && cp "$OVERLAY/databricks.yml" "$BUILD"/ +[[ -d "$OVERLAY/scripts" ]] && cp -R "$OVERLAY"/scripts/. "$BUILD"/scripts/ +# 5) overlay: chat UI patches (Genie attribution, proxy-friendly tweaks) +if [[ -d "$OVERLAY/patches/e2e-chatbot-app-next" ]]; then + cp -R "$OVERLAY/patches/e2e-chatbot-app-next/." "$BUILD"/e2e-chatbot-app-next/ +fi + +# 5b) Pin the transitive `greenlet` to the vendored Linux wheel (3.5.3). The upstream +# template re-resolves deps at runtime (uv.lock is excluded from sync — GAP-15), so +# `uv run` can pick a macOS-only newest greenlet (e.g. 3.5.4) from the index that has no +# Linux wheel → the app crashes on the Linux Apps host. This override lives HERE (our +# assembly), not in the submodule template, so it re-applies on every template re-copy. +# Idempotent; handles a pre-existing [tool.uv] section. +python3 - "$BUILD/pyproject.toml" <<'PY' +import re, sys +p = sys.argv[1] +s = open(p).read() +PIN = "greenlet==3.5.3" +if PIN in s: + sys.exit(0) # already pinned — no-op +if re.search(r'^\[tool\.uv\]', s, re.M): + if re.search(r'^\s*override-dependencies\s*=\s*\[', s, re.M): + s = re.sub(r'(override-dependencies\s*=\s*\[)', r'\1"%s", ' % PIN, s, count=1) + else: + s = re.sub(r'(^\[tool\.uv\][^\n]*\n)', r'\1override-dependencies = ["%s"]\n' % PIN, s, count=1, flags=re.M) +else: + s = s.rstrip() + '\n\n[tool.uv]\noverride-dependencies = ["%s"]\n' % PIN +open(p, "w").write(s) +print("assemble: pinned %s in agent-build/pyproject.toml" % PIN) +PY + +# 5c) Deterministic dependency pins for the whole graph (#64). The Apps build runs a plain +# `uv sync` (uv.lock is excluded from sync — GAP-15), so without version pins it can +# re-resolve a transitive dep to a newer release that lacks a Linux wheel (the greenlet +# 3.5.4 crash). Export the lock to a CONSTRAINTS file — this BOUNDS versions without the +# exact-match rigidity that made `uv sync --locked` fail (GAP-15) — and sync it; the app env +# sets UV_CONSTRAINT to it (see agent/databricks.yml). Exported from the universal lock with +# environment markers, which uv applies on the Linux/cp311 Apps host at install time. +# Supersedes the one-off greenlet override above once validated. Non-fatal: if export fails, +# warn and continue (the greenlet override still guards the known crash). +# constraints.txt is generated by scripts/vendor_wheels.sh (Phase 3d), NOT here. +# It cannot be generated here: this script starts with `rm -rf "$BUILD"`, the upstream +# template ships no uv.lock, and agent-build/uv.lock is first written by vendor_wheels.sh's +# own `uv lock`. So on a fresh clone this branch could only ever take the WARN path, and +# re-running assemble later to "fix" that would delete quickstart's .env. Generating it +# next to the `uv lock` that produces the input is the only ordering that works. + +# 6) Give agent-build its own git boundary. The parent repo gitignores +# agent-build/, and `databricks bundle deploy` respects the enclosing repo's +# ignore rules — without this, sync finds zero files ("no files to sync") and +# ships an empty app. A local-only git init (never committed) scopes the bundle +# to agent-build so its files sync correctly. +if [[ ! -d "$BUILD/.git" ]]; then + git init -q "$BUILD" +fi + +echo "Assembled agent at $BUILD" diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..a6876a0 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,1365 @@ +#!/usr/bin/env bash +# bootstrap.sh — Firefly Genie-Agent interactive setup runner +# +# Usage: +# bash scripts/bootstrap.sh # live mode (executes every command) +# bash scripts/bootstrap.sh --dry-run # prints commands, no infra touched +# bash scripts/bootstrap.sh --dry-run --stop-after=3 # stop after Phase 3 +# bash scripts/bootstrap.sh --stop-after=1 # collect inputs + auth only +# bash scripts/bootstrap.sh --trust-proxy-ca # auto-trust a detected proxy CA +# +# Mirrors every phase in BOOTSTRAP.md exactly. +# Secrets persist in a gitignored, chmod-600 .firefly-bootstrap/state.env under the repo. + +set -euo pipefail + +# ─── flags ──────────────────────────────────────────────────────────────────── +DRY_RUN=false +STOP_AFTER="" +CHECK_PYPI_PROXY=false +# Non-interactive trust of an auto-detected intercepting-proxy root CA (CI/automation). +# Also honored via env FIREFLY_TRUST_PROXY_CA=1. Off by default → we prompt with the +# root CA's fingerprint before trusting anything. +TRUST_PROXY_CA=false +[[ "${FIREFLY_TRUST_PROXY_CA:-}" == "1" ]] && TRUST_PROXY_CA=true +# Branch of databrickslabs/firefly to clone in Phase 2 (app code). +# +# This default outlives itself: once genie-agent merges and is deleted, a pinned +# `--branch genie-agent` clone fails with "Remote branch not found in upstream origin". +# firefly_resolve_branch prefers it while it exists and falls back to the default branch +# afterwards, so the same script works either side of the merge. +FIREFLY_REPO="${FIREFLY_REPO:-https://github.com/databrickslabs/firefly.git}" +FIREFLY_BRANCH="${FIREFLY_BRANCH:-genie-agent}" + +# Echoes the `--branch X` argument to use, or nothing when the branch is gone. +firefly_resolve_branch() { + if git ls-remote --exit-code --heads "$FIREFLY_REPO" "$FIREFLY_BRANCH" >/dev/null 2>&1; then + printf -- '--branch %s' "$FIREFLY_BRANCH" + else + note "branch '$FIREFLY_BRANCH' not on the remote — cloning the default branch" >&2 + fi +} + +for arg in "$@"; do + case $arg in + --dry-run) DRY_RUN=true ;; + --stop-after=*) STOP_AFTER="${arg#*=}" ;; + --trust-proxy-ca) TRUST_PROXY_CA=true ;; + --check-pypi-proxy) CHECK_PYPI_PROXY=true ;; + *) echo "Unknown flag: $arg (use --dry-run, --stop-after=N, --trust-proxy-ca, --check-pypi-proxy)"; exit 1 ;; + esac +done + +# ─── color helpers ──────────────────────────────────────────────────────────── +bold=$(tput bold 2>/dev/null || echo "") +dim=$(tput dim 2>/dev/null || echo "") +reset=$(tput sgr0 2>/dev/null || echo "") +cyan=$(tput setaf 6 2>/dev/null || echo "") +yellow=$(tput setaf 3 2>/dev/null || echo "") +green=$(tput setaf 2 2>/dev/null || echo "") +red=$(tput setaf 1 2>/dev/null || echo "") + +header() { echo; echo "${bold}${cyan}=== $* ===${reset}"; echo; } +step() { echo "${bold}$*${reset}"; } +note() { echo " ${dim}$*${reset}"; } +ok() { echo " ${green}✓ $*${reset}"; } +warn() { echo " ${yellow}⚠ $*${reset}"; } +fail() { echo " ${red}✗ $*${reset}"; } + +BOOTSTRAP_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib/corp-network.sh +source "$BOOTSTRAP_SCRIPT_DIR/lib/corp-network.sh" +# shellcheck source=scripts/lib/runbook.sh +source "$BOOTSTRAP_SCRIPT_DIR/lib/runbook.sh" + +if [[ "$CHECK_PYPI_PROXY" == "true" ]]; then + check_pypi_proxy_state "${REPO_DIR:-$PWD}" + ok "PyPI proxy config and checked uv.lock files are safe" + exit 0 +fi + +# ─── run helper ─────────────────────────────────────────────────────────────── +# In dry-run mode: prints the command. In live mode: executes it. +run() { + if [[ "$DRY_RUN" == "true" ]]; then + echo " ${yellow}[DRY-RUN]${reset} $*" + else + eval "$@" + fi +} + +# Like run, but captures output (dry-run returns a placeholder). +capture() { + local varname="$1"; shift + if [[ "$DRY_RUN" == "true" ]]; then + echo " ${yellow}[DRY-RUN]${reset} $* → <$varname>" + eval "$varname='<$varname-placeholder>'" + else + eval "$varname=\$( $* )" + fi +} + +# ─── prompt helpers ─────────────────────────────────────────────────────────── +ask() { + local varname="$1" prompt="$2" default="${3:-}" + local val="" + # A cached answer (from inputs.env) takes precedence as the default. + local cached="${!varname:-}" + [[ -n "$cached" ]] && default="$cached" + # When reusing saved answers, accept the cached value without prompting. + if [[ "${REUSE_INPUTS:-0}" == "1" && -n "$cached" ]]; then + eval "$varname='$cached'" + ok "$varname = $cached ${dim}(saved)${reset}" + return + fi + if [[ -n "$default" ]]; then + read -rp " ${bold}$prompt${reset} [${dim}$default${reset}]: " val + val="${val:-$default}" + else + while [[ -z "$val" ]]; do + read -rp " ${bold}$prompt${reset}: " val + [[ -z "$val" ]] && warn "Required — cannot be empty." + done + fi + eval "$varname='$val'" + store_input "$varname" "$val" + ok "$varname = $val" +} + +# Secret storage lives in scripts/lib/runbook.sh (store_secret / read_secret / +# require_secret / load_secrets / init_state_dir), sourced above. BOOTSTRAP.md +# sources the same file, so the runbook and this runner cannot drift apart — +# which is exactly how the markdown ended up calling helpers that existed only +# here, with a signature its own call sites could not use. + +ask_secret() { # ask_secret VARNAME PROMPT + local varname="$1" prompt="$3" + local val="" + while [[ -z "$val" ]]; do + read -rsp " ${bold}$prompt${reset} (hidden): " val; echo + [[ -z "$val" ]] && warn "Required — cannot be empty." + done + store_secret "$varname" "$val" +} + +# ─── Input persistence + resume (#18) ───────────────────────────────────────── +# Non-secret answers persist in ~/.firefly-bootstrap/inputs.env so re-runs don't +# re-prompt. This lives in $HOME (not $REPO_DIR) because REPO_DIR is itself an +# answer we cache — it must survive before the repo is cloned. +INPUTS_DIR="$HOME/.firefly-bootstrap" +INPUTS_FILE="$INPUTS_DIR/inputs.env" +REUSE_INPUTS=0 + +init_inputs_dir() { mkdir -p "$INPUTS_DIR"; chmod 700 "$INPUTS_DIR"; } + +store_input() { # store_input KEY VALUE + local key="$1" val="$2" + init_inputs_dir + local tmp="${INPUTS_FILE}.tmp" + touch "$INPUTS_FILE" + grep -v "^export ${key}=" "$INPUTS_FILE" > "$tmp" 2>/dev/null || true + printf 'export %s=%q\n' "$key" "$val" >> "$tmp" + mv "$tmp" "$INPUTS_FILE" + chmod 600 "$INPUTS_FILE" +} + +load_inputs() { [[ -f "$INPUTS_FILE" ]] && source "$INPUTS_FILE" || true; } + +# Track completed phases as a set (handles non-numeric ids like "6b"). Used to reword a +# phase prompt as "Re-execute" on a resumed run, so a redeploy is a conscious choice. +phase_done() { case " ${COMPLETED_PHASES:-} " in *" $1 "*) return 0 ;; *) return 1 ;; esac; } +mark_phase_done() { + phase_done "$1" || COMPLETED_PHASES="${COMPLETED_PHASES:+$COMPLETED_PHASES }$1" + store_input COMPLETED_PHASES "$COMPLETED_PHASES" + store_input LAST_COMPLETED_PHASE "$1" # retained for backward-compatible state files +} + +reset_inputs() { rm -f "$INPUTS_FILE"; unset LAST_COMPLETED_PHASE COMPLETED_PHASES; } + +# If a prior run left cached answers, show them and ask whether to reuse. +maybe_reuse_inputs() { + [[ "$DRY_RUN" == "true" ]] && return 0 + [[ -f "$INPUTS_FILE" ]] || return 0 + load_inputs + echo + note "Found saved answers from a previous run (~/.firefly-bootstrap/inputs.env):" + local shown=0 + for k in DATABRICKS_HOST UC_CATALOG UC_SCHEMA DATABRICKS_ACCOUNT_ID REPO_DIR VERCEL_TEAM VERCEL_PROJECT NEON_PROJECT_NAME; do + local v="${!k:-}"; [[ -n "$v" ]] && { echo " ${dim}$k = $v${reset}"; shown=1; } + done + [[ -n "${COMPLETED_PHASES:-}" ]] && note "Previously completed phases: ${COMPLETED_PHASES} (re-running any is a redeploy)." + [[ "$shown" == "0" ]] && return 0 + echo + read -rp " ${bold}Reuse these saved answers?${reset} [Y/n]: " reuse_ + if [[ "$reuse_" =~ ^[Nn]$ ]]; then + reset_inputs + note "Starting fresh — you'll be asked for each value." + else + REUSE_INPUTS=1 + ok "Reusing saved answers (press Enter at any prompt to keep a value)." + fi +} + +# Returns 0 (true) when the user wants to EXECUTE the phase body, 1 (false) otherwise. +# Completed phases default to SKIP (Enter breezes past on a resumed run); pending phases +# default to EXECUTE. run_phase() interprets a false return as skip-vs-stop. +confirm_phase() { + local phase="$1" ok_ + if [[ "$DRY_RUN" == "true" ]]; then + echo " ${yellow}[DRY-RUN]${reset} Would execute Phase $phase" + return 0 + fi + echo + if phase_done "$phase"; then + note "Phase $phase already completed in a previous run — Enter SKIPS it (resume forward); 'y' re-executes (idempotent redeploy)." + read -rp " ${bold}Re-execute Phase $phase?${reset} [y/N]: " ok_ + [[ "$ok_" =~ ^[Yy]$ ]] + else + read -rp " Execute Phase $phase? [Y/n]: " ok_ + [[ ! "$ok_" =~ ^[Nn]$ ]] + fi +} + +# Phase gate with skip-forward resume (#18). Wrap each phase body as: +# if run_phase "N"; then +# +# fi +# stop_if_done "N" +# • execute (user accepts) → run body +# • completed + declined (default on re-run) → SKIP body, advance to next phase +# • pending + declined → deliberate STOP (exit; re-run resumes) +run_phase() { + local phase="$1" + if confirm_phase "$phase"; then + return 0 + fi + if phase_done "$phase"; then + note "⏭ Skipping Phase $phase (already completed) — resuming forward." + return 1 + fi + echo + echo "${yellow}Stopped at Phase $phase (declined). Re-run to resume from here.${reset}" + exit 0 +} + +stop_if_done() { + local phase="$1" + [[ "$DRY_RUN" == "false" ]] && mark_phase_done "$phase" + if [[ -n "$STOP_AFTER" && "$STOP_AFTER" == "$phase" ]]; then + echo + echo "${green}Stopped after Phase $phase.${reset}" + exit 0 + fi +} + +validate_url() { + local url="$1" + [[ "$url" =~ ^https?:// ]] || { fail "Must start with https://"; return 1; } +} + +# ─── banner ─────────────────────────────────────────────────────────────────── +echo +echo "${bold}╔══════════════════════════════════════════════════════════╗${reset}" +echo "${bold}║ Firefly Genie-Agent — Bootstrap Runner ║${reset}" +echo "${bold}║ Implements BOOTSTRAP.md phases 0–9 ║${reset}" +[[ "$DRY_RUN" == "true" ]] && \ +echo "${bold}║ ${yellow}Mode: DRY-RUN (no infra will be created)${reset}${bold} ║${reset}" || \ +echo "${bold}║ ${green}Mode: LIVE (will create real resources)${reset}${bold} ║${reset}" +[[ -n "$STOP_AFTER" ]] && \ +echo "${bold}║ Stopping after Phase $STOP_AFTER ║${reset}" +echo "${bold}╚══════════════════════════════════════════════════════════╝${reset}" + +# ─── Phase 0: Collect inputs ───────────────────────────────────────────────── +header "Phase 0 — Collect inputs" +note "Answers persist in ~/.firefly-bootstrap/inputs.env; secrets in state.env (chmod 600)." +maybe_reuse_inputs +echo + +ask DATABRICKS_HOST "Databricks workspace URL (https://dbc-xxxx.cloud.databricks.com)" +# Users often paste the full browser URL (…/?autoLogin=true&o=…&email=…). The Databricks +# SDK reads DATABRICKS_HOST with precedence over the profile, and a query/path on the host +# breaks host-metadata resolution ("Expecting value: line 1 column 1"). Keep only scheme://host. +DATABRICKS_HOST=$(printf '%s' "$DATABRICKS_HOST" | sed -E 's|^(https?://[^/?#]+).*|\1|') +store_input DATABRICKS_HOST "$DATABRICKS_HOST" +validate_url "$DATABRICKS_HOST" + +ask DB_PROFILE "Databricks CLI profile name" "firefly-deploy" +ask UC_CATALOG "Unity Catalog catalog" "workspace" +ask UC_SCHEMA "Unity Catalog schema" "default" + +# Phase 6c inputs. Asked here, with everything else, because Phase 0 is the only +# place the runbook is allowed to block on a question (#83). Seeding only ever +# acts on an EMPTY schema and never overwrites a table, which is what makes `yes` +# a safe default. +ask SEED_SAMPLE_DATA "Seed samples.wanderbricks if $UC_CATALOG.$UC_SCHEMA is empty? (yes/no)" "yes" +ask GENIE_SPACE_IDS "Existing Genie space id(s) to use, comma-separated (None = create one)" "None" +case "$(printf '%s' "${GENIE_SPACE_IDS:-None}" | tr 'A-Z' 'a-z')" in + none|null|"") + GENIE_SPACE_IDS="" + ask CREATE_GENIE_SPACE "Create a Genie space over $UC_CATALOG.$UC_SCHEMA? (yes/no)" "yes" + GRANT_GUEST_SPACE_ACCESS="no" # nothing of the user's to grant against + ;; + *) + # Only meaningful when the user named spaces: these are the one set of spaces + # bootstrap is permitted to touch, so granting on them is an explicit choice. + CREATE_GENIE_SPACE="no" + ask GRANT_GUEST_SPACE_ACCESS "Grant the guest SP CAN_RUN on those spaces + SELECT on their tables? (yes/no)" "yes" + ;; +esac + +ask AGENT_APP_NAME "Databricks App name" "firefly-openai-managed-mem-v2" +ask LAKEBASE_NAME "Lakebase instance name" "firefly-lb" +ask DATABRICKS_ACCOUNT_ID "Databricks account ID (from accounts.cloud.databricks.com URL)" +ask REPO_DIR "Local clone directory (created if missing; must be new/empty, NOT your home dir)" "$HOME/firefly" +ask VERCEL_TEAM "Vercel team slug (e.g. acme-corp — from vercel.com//...)" +ask NEON_PROJECT_NAME "Neon project name" "firefly-genie" +ask VERCEL_PROJECT "Vercel project name" "firefly-genie" + +echo +# uv ships its OWN bundled cert store (rustls/webpki) and ignores the OS keychain by +# default, so it rejects an intercepting proxy's cert (UnknownIssuer) even when Node +# and keychain-backed tools succeed. UV_SYSTEM_CERTS=1 makes uv trust the platform +# store — harmless off-proxy (the store already trusts public CAs), required on-proxy. +export UV_SYSTEM_CERTS="${UV_SYSTEM_CERTS:-1}" + +# The gh / databricks / uv installers drop binaries into these user-local bin dirs (Phase 1). +# Export them EVERY run at top level (not inside the Phase 1 body): with skip-forward resume +# (#18), a run that skips Phase 1 must still find `databricks`/`uv`/`gh` in later phases — +# otherwise Phase 8's `databricks apps get` fails with "command not found". +# Create them up front: they are on PATH from here on, and the Phase 1 installers assume +# the target directory already exists. +mkdir -p "$HOME/bin" "$HOME/.local/bin" 2>/dev/null || true +export PATH="$HOME/bin:$HOME/.local/bin:$PATH" + +# Corporate-network handling (TLS proxy CA, uv←pip index bridge, corepack←npm registry +# bridge, npm reachability preflight) lives in a shared library that BOTH this runner and +# the Phase 0 block in BOOTSTRAP.md source. Keeping one implementation is what prevents the +# runbook and the runner from drifting — see ENV-0, where the bridges existed only here and +# the runbook merely described them, so following the doc skipped them entirely. +# +# TLS precedence (unchanged): +# 1. TLS_PEM_PATH given → use it verbatim. +# 2. Auto-detect a MITM by probing PyPI against the system trust store; if a PRIVATE +# root CA is presented, extract it, show CN + SHA-256, confirm (or --trust-proxy-ca), +# and build a LOCAL bundle (system roots + proxy CAs). System trust is never touched. +# 3. No MITM detected → nothing to do. +# Corporate-network helpers (TLS/CA, uv + corepack bridges, PyPI proxy policy) are sourced +# once near the top of this script, before flag handling. + +if [[ -n "${TLS_PEM_PATH:-}" && -f "${TLS_PEM_PATH}" ]]; then + apply_tls_bundle "$TLS_PEM_PATH" + ok "TLS trust from TLS_PEM_PATH → SSL_CERT_FILE / REQUESTS_CA_BUNDLE / NODE_EXTRA_CA_CERTS" +elif [[ "$DRY_RUN" == "true" ]]; then + run "# detect intercepting proxy; if a private CA is presented, confirm + build a local bundle" +elif derive_proxy_ca_bundle; then + warn "Intercepting HTTPS proxy detected — its chain terminates in a PRIVATE root CA." + note " Root CN: ${PROXY_CA_ROOT_CN:-}" + note " SHA-256: ${PROXY_CA_ROOT_FP:-}" + note " Copied to: $PROXY_CA_BUNDLE" + note " (system roots + ${PROXY_CA_ADDED} proxy CA cert(s); your system trust store is NOT modified)" + if [[ "$TRUST_PROXY_CA" == "true" ]]; then + apply_tls_bundle "$PROXY_CA_BUNDLE" + ok "--trust-proxy-ca set → using the copied CA (SSL_CERT_FILE / REQUESTS_CA_BUNDLE / NODE_EXTRA_CA_CERTS / CURL_CA_BUNDLE + UV_SYSTEM_CERTS)." + else + echo " ${bold}Only trust the copied CA if the SHA-256 above matches your organization's known root CA.${reset}" + echo " ${bold}1)${reset} Trust this CA for this setup — use the copied bundle above" + echo " ${bold}2)${reset} Use my own PEM — enter the path to a combined bundle" + echo " ${bold}3)${reset} Don't use a cert — proceed untrusted (installs may fail on-proxy)" + read -rp " ${bold}Choose [1/2/3] (default 1):${reset} " TLS_CHOICE + case "${TLS_CHOICE:-1}" in + 2) + read -rp " Path to your combined PEM: " TLS_OWN_PEM + if [[ -n "$TLS_OWN_PEM" && -f "$TLS_OWN_PEM" ]]; then + apply_tls_bundle "$TLS_OWN_PEM" + ok "TLS trust from your PEM → SSL_CERT_FILE / REQUESTS_CA_BUNDLE / NODE_EXTRA_CA_CERTS / CURL_CA_BUNDLE" + else + warn "No readable PEM at '${TLS_OWN_PEM:-}' — proceeding untrusted. Re-run with TLS_PEM_PATH= if installs fail." + fi + ;; + 3) + warn "Proceeding without a proxy CA. If installs fail with cert errors, re-run and pick 1, or set TLS_PEM_PATH=." + ;; + *) + apply_tls_bundle "$PROXY_CA_BUNDLE" + ok "TLS trust → SSL_CERT_FILE / REQUESTS_CA_BUNDLE / NODE_EXTRA_CA_CERTS / CURL_CA_BUNDLE (+ UV_SYSTEM_CERTS)" + ;; + esac + fi +else + ok "No intercepting proxy detected (TLS to PyPI validates against the system store)." +fi + +# Bridge the user's existing pip index mirror into uv (uv ignores pip.conf). +# Flag the unsanctioned .dev PyPI proxy before any uv lock/sync work (warn by default, +# fatal with FIREFLY_STRICT_PYPI_PROXY=1). +if [[ "$DRY_RUN" == "true" ]]; then + run "# if pip is configured with a custom index-url, export UV_DEFAULT_INDEX to match" + run "# refuse pypi-proxy.dev.databricks.com in env / pip / uv.toml" +else + reject_dead_pypi_proxy_config + bridge_pip_index_to_uv + if [[ -n "${PIP_BRIDGED_INDEX:-}" ]]; then + ok "uv index bridged from pip config → UV_DEFAULT_INDEX=$PIP_BRIDGED_INDEX" + elif [[ -n "${UV_DEFAULT_INDEX:-}${UV_INDEX_URL:-}" ]]; then + note "uv index already set via env — leaving as-is." + fi +fi + +# Bridge the user's existing npm registry mirror into corepack. Phase 1a installs pnpm via +# npm (which already honors that registry), so this is not required for the supported path +# — it is kept for parity with the uv bridge and for anyone who opts into corepack manually. +if [[ "$DRY_RUN" == "true" ]]; then + run "# if npm has a custom registry, export COREPACK_NPM_REGISTRY to match (+ disable download prompt)" +else + bridge_npm_registry_to_corepack + if [[ -n "${NPM_BRIDGED_REGISTRY:-}" ]]; then + ok "corepack registry bridged from npm config → COREPACK_NPM_REGISTRY=$NPM_BRIDGED_REGISTRY" + elif [[ -n "${COREPACK_NPM_REGISTRY:-}" ]]; then + note "corepack registry already set via env — leaving as-is." + fi +fi + +note "All inputs collected. Summary:" +note " DATABRICKS_HOST = $DATABRICKS_HOST" +note " DB_PROFILE = $DB_PROFILE" +note " UC_CATALOG = $UC_CATALOG / $UC_SCHEMA" +note " AGENT_APP_NAME = $AGENT_APP_NAME" +note " LAKEBASE_NAME = $LAKEBASE_NAME" +note " DATABRICKS_ACCOUNT_ID = $DATABRICKS_ACCOUNT_ID" +note " REPO_DIR = $REPO_DIR" +note " VERCEL_TEAM = $VERCEL_TEAM" +note " NEON_PROJECT_NAME = $NEON_PROJECT_NAME" +note " VERCEL_PROJECT = $VERCEL_PROJECT" + +stop_if_done "0" + +# ─── Phase 1: Auth ──────────────────────────────────────────────────────────── +header "Phase 1 — Auth" +# Steps here are deliberately unlettered. This runner installs in a different order than +# BOOTSTRAP.md documents, so numbering both 1a..1f made "1b" mean the Vercel CLI in one +# file and the Databricks CLI in the other. The runbook owns the letters. +if run_phase "1"; then + +echo +step "pnpm (pinned install via npm — deliberately NOT corepack; see ENV-0)" +# Two constraints, both load-bearing: +# 1. pnpm's npm "latest" dist-tag has shipped a 12.x ALPHA that ignores +# onlyBuiltDependencies (→ ERR_PNPM_IGNORED_BUILDS), so the version must be pinned. +# 2. corepack fetches its package manager from registry.npmjs.org and ignores the npm +# registry setting, so it hard-fails wherever public npm is blocked (ENV-0). +# `npm install -g pnpm@` satisfies both: npm honors the user's OWN configured +# registry, so the supported path needs no COREPACK_NPM_REGISTRY bridge at all. +if [[ "$DRY_RUN" != "true" ]] && ! firefly_preflight_npm_registry; then + fail "Cannot install pnpm until the npm registry above is reachable." + exit 1 +fi +if command -v pnpm &>/dev/null && [[ "$(pnpm --version 2>/dev/null)" == "$PNPM_VERSION" ]]; then + ok "pnpm already installed at the pinned version: $PNPM_VERSION" +else + # An enabled corepack puts its own pnpm shim ahead of the npm-global binary on PATH, + # which would shadow the version we just pinned. Drop the shim before installing. + run "corepack disable >/dev/null 2>&1 || true" + run "npm install -g pnpm@$PNPM_VERSION" + note "pnpm pinned to $PNPM_VERSION (matches the repo's packageManager field)." +fi + +echo +step "Vercel CLI OAuth (opens browser)" +# Best-effort: silence the CLI's update/telemetry check, which reaches public npm +# and fails behind a corp proxy, printing an "Error:" line before commands that +# then succeed. UNVERIFIED — reproduces only on a proxied network. +export VERCEL_TELEMETRY_DISABLED=1 +note "the vercel dist-tags 'Error:' line is its update check, not a failure" +# Pin to a Tart-tested floor — do not chase npm `latest` (CLI deploy semantics move). +# Override with VERCEL_CLI_VERSION=x.y.z if you need to bump deliberately. +VERCEL_CLI_VERSION="${VERCEL_CLI_VERSION:-56.3.1}" +vercel_needs_install=true +if command -v vercel &>/dev/null; then + VERCEL_CURRENT=$(vercel --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + # sort -C -V: already-sorted means VERCEL_CLI_VERSION <= VERCEL_CURRENT. + if [[ -n "$VERCEL_CURRENT" ]] && printf '%s\n%s\n' "$VERCEL_CLI_VERSION" "$VERCEL_CURRENT" | sort -C -V; then + ok "vercel already installed: $VERCEL_CURRENT (>= $VERCEL_CLI_VERSION)" + vercel_needs_install=false + else + note "vercel ${VERCEL_CURRENT:-unknown} is below required $VERCEL_CLI_VERSION — installing pin" + fi +fi +if [[ "$vercel_needs_install" == "true" ]]; then + # Silent while it works, and the duration is not predictable: ~40s on one run, ~2 minutes on + # another. Quoting a number is what made the first version of this note misleading. + note "installing the Vercel CLI: silent while it works and can take minutes; not stalled" + note " wait for the process, not for a clock, and do not run a second install alongside it" + run "npm install -g vercel@${VERCEL_CLI_VERSION}" +fi +if [[ "$DRY_RUN" == "false" ]] && vercel whoami &>/dev/null; then + ok "vercel already authenticated: $(vercel whoami 2>/dev/null | tail -1)" +else + run "vercel login" +fi +run "vercel whoami" + +echo +step "GitHub CLI" +if command -v gh &>/dev/null; then + ok "gh already installed: $(gh --version 2>/dev/null | head -1)" +elif [[ "$DRY_RUN" == "true" ]]; then + run "# download + install GitHub CLI (official release) to \$HOME/bin" +else + firefly_install_gh || exit 1 +fi +if gh auth status &>/dev/null; then + ok "gh already authenticated: $(gh auth status 2>&1 | head -1)" +else + run "gh auth login" +fi + +echo +step "Neon CLI OAuth (opens browser)" +if command -v neonctl &>/dev/null; then + ok "neonctl already installed: $(neonctl --version 2>/dev/null || echo '?')" +else + run "npm install -g neonctl" +fi +if [[ "$DRY_RUN" == "false" ]] && neonctl me &>/dev/null; then + ok "neonctl already authenticated" +else + run "neonctl auth" +fi +run "neonctl me" + +echo +step "Databricks CLI OAuth (opens browser)" +if command -v databricks &>/dev/null; then + ok "databricks already installed: $(databricks --version 2>/dev/null | head -1)" +elif [[ "$DRY_RUN" == "true" ]]; then + run "# download + install Databricks CLI (official release) to \$HOME/bin" +else + firefly_install_databricks_cli || exit 1 +fi +run "databricks auth login --host '$DATABRICKS_HOST' --profile '$DB_PROFILE'" +run "databricks workspace list / --profile '$DB_PROFILE' 2>&1 | head -3" + +echo +step "Python uv (used by the agent build in Phases 4–5)" +if command -v uv &>/dev/null; then + ok "uv already installed: $(uv --version 2>/dev/null || echo '?')" +elif [[ "$DRY_RUN" == "true" ]]; then + run "# install uv via astral.sh installer to \$HOME/.local/bin" +else + note "Installing uv (astral.sh installer; no Homebrew required)..." + # Stock macOS has no sha256sum, so the installer would skip verifying its own + # download and install an unverified binary. Restore the check first. + firefly_ensure_sha256sum + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" + ok "uv installed to \$HOME/.local/bin ($(uv --version 2>/dev/null || echo '?'))" +fi + +fi +stop_if_done "1" + +# ─── Phase 2: Clone and assemble ───────────────────────────────────────────── +header "Phase 2 — Clone and assemble" +if run_phase "2"; then + +step "Clone the app repo (idempotent — safe to re-run)" +if [[ "$DRY_RUN" == "true" ]]; then + run "git clone $(firefly_resolve_branch) '$FIREFLY_REPO' '$REPO_DIR'" +elif [[ -d "$REPO_DIR/.git" ]]; then + ok "Repo already present at $REPO_DIR — reusing it (skipping clone)." +elif [[ -e "$REPO_DIR" && -n "$(ls -A "$REPO_DIR" 2>/dev/null)" ]]; then + fail "$REPO_DIR exists and is non-empty but is not a git repo." + note "Pick a new REPO_DIR (re-run and decline reuse), or remove that directory, then re-run." + exit 1 +else + run "git clone $(firefly_resolve_branch) '$FIREFLY_REPO' '$REPO_DIR'" +fi +run "cd '$REPO_DIR'" + +# NOTE: no GitHub fork push. The frontend is deployed with the `vercel deploy` CLI (Phase 8), +# which uploads the local build directly — it does NOT use Vercel's Git integration, so a +# user-owned GitHub repo is unnecessary. (This also drops a `gh` auth dependency and a +# failure surface.) To enable push-to-deploy later, connect a repo in the Vercel dashboard: +# Project → Settings → Git. + +echo +step "Submodule init (must run before assemble_agent.sh)" +run "git -C '$REPO_DIR' submodule update --init" + +echo +step "First assemble (before quickstart)" +run "bash '$REPO_DIR/scripts/assemble_agent.sh'" + +fi +stop_if_done "2" + +# ─── Phase 3: Provision Databricks resources ────────────────────────────────── +header "Phase 3 — Provision Databricks resources" +if run_phase "3"; then + +echo +step "3a. quickstart — MLflow experiment + Lakebase (--python 3.12 required)" +# Gate on index reachability first. Everything below this line is uv-driven, and an +# unreachable index otherwise surfaces as a raw uv stack trace with no cause named. +if [[ "$DRY_RUN" == "false" ]]; then + firefly_preflight_pypi_index || exit 1 +fi +# Lakebase create-vs-reuse: --lakebase-create-new is non-idempotent (fails with +# "project slug already exists" on re-run) AND it disables quickstart's own .env +# reuse path. quickstart names resources deterministically, so if the project's +# primary endpoint already exists, reuse it instead of trying to re-create. +LB_ENDPOINT_PATH="projects/${LAKEBASE_NAME}/branches/${LAKEBASE_NAME}-branch/endpoints/primary" +if [[ "$DRY_RUN" == "false" ]] \ + && databricks api get "/api/2.0/postgres/${LB_ENDPOINT_PATH}" --profile "$DB_PROFILE" &>/dev/null; then + ok "Lakebase project '${LAKEBASE_NAME}' exists — reusing endpoint (idempotent re-run)." + LB_ARG="--lakebase-autoscaling-endpoint '${LB_ENDPOINT_PATH}'" +else + LB_ARG="--lakebase-create-new '${LAKEBASE_NAME}'" +fi +note "Lakebase provisioning takes several minutes; let this finish before Phase 4." +[[ "$DRY_RUN" == "false" ]] && firefly_warn_existing_app_wins "$AGENT_APP_NAME" "$DB_PROFILE" + +run "cd '$REPO_DIR/agent-build' && uv run --python 3.12 python scripts/quickstart.py \ + --profile '$DB_PROFILE' ${LB_ARG} --app-name '$AGENT_APP_NAME'" + +# An existing --app-name wins over --lakebase-create-new, so the requested project +# may never be created while the summary still names it. Let reality win. +[[ "$DRY_RUN" == "false" ]] && firefly_reconcile_lakebase "$REPO_DIR/agent-build" +# Completion test, not a formality: quickstart rewrites experiment_id in the bundle, so +# this is the one observable that distinguishes "finished" from "still running" or +# "exited early". A wrapper that backgrounds long commands returns 0 either way. +if [[ "$DRY_RUN" == "false" ]]; then + assert_bundle_quickstart_ran "$REPO_DIR/agent-build/databricks.yml" || exit 1 +fi + +echo +step "3b. Catalog/schema → bundle vars (applied at deploy; no yml edit)" +note "HOST/WORKSPACE_ID are injected by quickstart (no GENIE_ONE_URL — the attribution link was removed)." +note "catalog=$UC_CATALOG schema=$UC_SCHEMA are passed via --var at deploy (Phase 4);" +note "DATABRICKS_MEMORY_STORE resolves to $UC_CATALOG.$UC_SCHEMA.firefly_managed_memory." + +echo +step "3c. Create UC wheels volume" +# The schema is assumed to exist; on a fresh catalog it does not (the catalog can +# hold nothing but information_schema), and the volume create then fails with a +# message about the volume rather than the missing schema. +if [[ "$DRY_RUN" == "false" ]]; then + databricks schemas create "$UC_SCHEMA" "$UC_CATALOG" --profile "$DB_PROFILE" &>/dev/null \ + && ok "created schema $UC_CATALOG.$UC_SCHEMA" \ + || note "schema $UC_CATALOG.$UC_SCHEMA already exists — continuing" +fi + +if [[ "$DRY_RUN" == "false" ]] \ + && databricks volumes read "${UC_CATALOG}.${UC_SCHEMA}.firefly_wheels" --profile "$DB_PROFILE" &>/dev/null; then + ok "UC volume ${UC_CATALOG}.${UC_SCHEMA}.firefly_wheels already exists — skipping create." +else + run "databricks volumes create '$UC_CATALOG' '$UC_SCHEMA' firefly_wheels MANAGED \ + --profile '$DB_PROFILE'" +fi + +echo +step "3d. Vendor cp311 wheels (required for offline build)" +run "cd '$REPO_DIR/agent-build' && bash scripts/vendor_wheels.sh" + +echo +step "3e. Verify sync.exclude rules" +# Was three bare greps over the whole file, which matched the explanatory comment +# ("NOTE: pyproject.toml and vendor-wheels/ MUST sync") and reported two failures +# on a correct config. check_sync_exclude_rules parses the exclude list itself. +if [[ "$DRY_RUN" == "false" ]]; then + check_sync_exclude_rules "$REPO_DIR/agent/databricks.yml" || exit 1 +else + run "# check_sync_exclude_rules '$REPO_DIR/agent/databricks.yml'" +fi + +fi +stop_if_done "3" + +# ─── Phase 4: Deploy agent app ──────────────────────────────────────────────── +header "Phase 4 — Deploy agent app" +step "Preflight: uv.lock must not stamp the unsanctioned PyPI proxy (.dev)" +if [[ "$DRY_RUN" == "true" ]]; then + run "# assert no ${FIREFLY_UNSANCTIONED_PYPI_HOST} in agent-build/guest-manager uv.lock" +else + assert_uv_locks_not_dead_pypi_proxy \ + "$REPO_DIR/agent-build/uv.lock" \ + "$REPO_DIR/databricks-apps/guest-manager/uv.lock" + ok "No ${FIREFLY_UNSANCTIONED_PYPI_HOST} in checked uv.lock files" + # Deploying a bundle whose resource bindings quickstart never rewrote fails with + # a 404 that names only the stale id. Say which phase to go back to instead. + assert_bundle_quickstart_ran "$REPO_DIR/agent-build/databricks.yml" || exit 1 +fi +# ---- Phase 3f: seed + create the Genie space BEFORE the app is deployed --------------- +# The app used to deploy workspace-wide (no space existed yet) and get corrected by a +# Phase 6c redeploy. Any failure between the two left it answering from workspace-wide +# Genie, which guest users cannot query at all, and nothing said so. Creating the space +# first removes that window: the app is born scoped to it. Only the GRANTS need the +# service principals, so those stay in 6c. +header "Phase 3f — Seed the data and create the Genie space" +if run_phase "3f"; then + +step "Resolve the SQL warehouse (seeding, space creation and Phase 6 grants all need it)" +if [[ "$DRY_RUN" == "false" ]]; then + WAREHOUSE_ID=$(databricks warehouses list -o json --profile "$DB_PROFILE" 2>/dev/null \ + | python3 -c "import sys,json +d=json.load(sys.stdin) or [] +ws=d if isinstance(d,list) else (d.get('warehouses') or []) +print(ws[0].get('id','') if ws else '')" 2>/dev/null || echo "") + [[ -n "${WAREHOUSE_ID:-}" ]] && firefly_store_input WAREHOUSE_ID "$WAREHOUSE_ID" + note "warehouse=${WAREHOUSE_ID:-none}" +fi + +step "Seed sample data and create/resolve the Genie space (no SPs exist yet)" +if [[ "$DRY_RUN" == "true" ]]; then + note "[DRY-RUN] genie-data-setup.sh --seed ... --create-space ... (grants happen in 6c)" + GENIE_MCP_MODE="one"; GENIE_SPACE_ID="" +else + GENIE_SETUP_OUT="$(bash "$BOOTSTRAP_SCRIPT_DIR/genie-data-setup.sh" \ + --phase 3f \ + --catalog "$UC_CATALOG" --schema "$UC_SCHEMA" --profile "$DB_PROFILE" \ + --warehouse-id "${WAREHOUSE_ID:-}" \ + --seed "${SEED_SAMPLE_DATA:-yes}" \ + --space-ids "${GENIE_SPACE_IDS:-}" \ + --defer-grants \ + --create-space "${CREATE_GENIE_SPACE:-yes}")" \ + || warn "Phase 3f setup reported a problem (continuing)" + eval "$GENIE_SETUP_OUT" + note "seed=${SEED_STATUS:-?} tables=${SEED_TABLE_COUNT:-?} mode=${GENIE_MCP_MODE:-one} space=${GENIE_SPACE_ID:-none}" + [[ -n "${GENIE_SPACE_ID:-}" ]] && store_secret GENIE_SPACE_ID "$GENIE_SPACE_ID" + firefly_store_input GENIE_MCP_MODE "${GENIE_MCP_MODE:-one}" + if [[ "${GENIE_MCP_MODE:-one}" != "space" ]]; then + warn "no Genie space resolved — the app deploys workspace-wide (mode=one)." + warn " Guest users will reach it and be unable to ask data questions." + fi +fi + +fi + + +if run_phase "4"; then + +step "Bundle deploy + run (from agent-build/; do NOT re-run assemble_agent.sh)" +note "assemble_agent.sh already ran in Phase 2; re-running would wipe quickstart's .env and wheels." +# The Genie vars ship with the FIRST deploy, so there is no workspace-wide window and no +# redeploy to correct it. Recovered, not assumed: Phase 3f may have run in an earlier shell. +: "${GENIE_SPACE_ID:=$(read_secret GENIE_SPACE_ID 2>/dev/null || true)}" +: "${GENIE_MCP_MODE:=$(firefly_read_input GENIE_MCP_MODE 2>/dev/null || echo one)}" +if [[ "${GENIE_MCP_MODE:-one}" == "space" && -z "${GENIE_SPACE_ID:-}" ]]; then + fail "mode=space with no GENIE_SPACE_ID — re-run Phase 3f, or set GENIE_MCP_MODE=one" + exit 1 +fi +BUNDLE_VARS="--var catalog=$UC_CATALOG --var schema=$UC_SCHEMA --var genie_mcp_mode=${GENIE_MCP_MODE:-one} --var genie_space_id=${GENIE_SPACE_ID:-none}" +run "cd '$REPO_DIR/agent-build' && databricks bundle deploy --profile '$DB_PROFILE' -t dev $BUNDLE_VARS" +run "cd '$REPO_DIR/agent-build' && databricks bundle run agent_openai_agents_sdk --profile '$DB_PROFILE' -t dev $BUNDLE_VARS" + +echo +step "Poll until app_status.state = RUNNING (deployment state leads by ~44s)" +if [[ "$DRY_RUN" == "false" ]]; then + for i in $(seq 1 24); do + STATE=$(databricks apps get "$AGENT_APP_NAME" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); \ + print(d['app_status']['state'])" 2>/dev/null || echo "UNKNOWN") + echo " [$i/24] app_status.state = $STATE" + [[ "$STATE" == "RUNNING" ]] && { ok "App is RUNNING"; break; } + [[ "$STATE" == "CRASHED" || "$STATE" == "UNAVAILABLE" ]] && \ + { fail "App $STATE — check: databricks apps logs $AGENT_APP_NAME --profile $DB_PROFILE"; exit 1; } + sleep 30 + done +else + run "databricks apps get '$AGENT_APP_NAME' -o json --profile '$DB_PROFILE' \ + | python3 -c \"import sys,json; d=json.load(sys.stdin); print(d['app_status']['state'])\"" +fi + + +# The deploy's exit code is not evidence. CLI v1.9.0 can panic on bundle deploy +# and still exit 0, so a crashed deploy reads as success and Phases 5/6/9 then +# fail opaquely. Assert the app exists. +if [[ "$DRY_RUN" == "false" ]]; then + if databricks apps get "$AGENT_APP_NAME" --profile "$DB_PROFILE" &>/dev/null; then + ok "Phase 4 created $AGENT_APP_NAME" + else + fail "Phase 4 did not create the app $AGENT_APP_NAME (deploy exit code notwithstanding)" + note "Check the deploy output for a panic or an env-var rejection." + note "Stale bundle state also causes this: if the app was deleted but" + note "/Workspace/Users//.bundle/firefly_openai_managed_mem survives, the CLI" + note "diffs against an app that no longer exists. Delete that path and redeploy." + fi +fi +fi +stop_if_done "4" + +# ─── Phase 5: UC managed memory ─────────────────────────────────────────────── +header "Phase 5 — UC managed memory" +if run_phase "5"; then + +capture SP_CLIENT_ID \ + "databricks apps get '$AGENT_APP_NAME' -o json --profile '$DB_PROFILE' \ + | python3 -c \"import sys,json; d=json.load(sys.stdin); print(d['service_principal_client_id'])\"" +note "App service principal: $SP_CLIENT_ID" + +# The preview is enabled per-workspace by Databricks; without it this phase fails +# with NotImplemented and there is no local remedy. It was the most-reported gap +# in E2E runs (9 of 12) purely because the runbook charged ahead and failed +# opaquely. Say so up front, and let the rest of the bootstrap finish — only +# cross-session memory is lost. +# Attempt, then classify. An earlier version probed /api/2.0/memory-stores and +# read a clean response as "preview on" — that path returns `Error: Not Found`, +# which matched none of its patterns, so it declared the preview ENABLED on a +# workspace where setup then failed with NotImplemented. A preflight that cannot +# detect the state it exists to detect is worse than none. The operation itself +# is the only reliable signal. +if [[ "$DRY_RUN" == "true" ]]; then + note "[DRY-RUN] setup_memory_store.py $SP_CLIENT_ID" +else + MEM_LOG="$(mktemp)" + if (cd "$REPO_DIR/agent-build" && \ + uv run --python 3.12 python scripts/setup_memory_store.py "$SP_CLIENT_ID" \ + --memory-store "$UC_CATALOG.$UC_SCHEMA.firefly_managed_memory" \ + --profile "$DB_PROFILE") >"$MEM_LOG" 2>&1; then + ok "UC managed memory store configured" + elif grep -qiE 'not enabled|NotImplemented|preview' "$MEM_LOG"; then + warn "Managed Memory for Agents preview is NOT enabled on this workspace." + note "Skipping Phase 5 — the preview is enabled per-workspace by Databricks." + note "The agent still runs; it just has no cross-session memory." + else + fail "Phase 5 failed for a reason other than the preview:" + sed 's/^/ /' "$MEM_LOG" >&2 + fi + rm -f "$MEM_LOG" +fi + +fi +stop_if_done "5" + +# ─── Phase 6: Grant SP data access ──────────────────────────────────────────── +header "Phase 6 — Grant agent SP access to data" +if run_phase "6"; then + +note "Granting USE CATALOG on $UC_CATALOG..." +run "databricks api patch '/api/2.1/unity-catalog/permissions/catalog/$UC_CATALOG' \ + --profile '$DB_PROFILE' \ + --json '{\"changes\":[{\"principal\":\"$SP_CLIENT_ID\",\"add\":[\"USE CATALOG\"]}]}'" + +capture WAREHOUSE_ID \ + "databricks warehouses list -o json --profile '$DB_PROFILE' \ + | python3 -c \"import sys,json; ws=json.load(sys.stdin); print(ws[0]['id'] if ws else '')\"" + +if [[ -n "$WAREHOUSE_ID" && "$WAREHOUSE_ID" != "" ]]; then + note "Granting CAN_USE on warehouse $WAREHOUSE_ID..." + run "databricks api patch \ + \"/api/2.0/permissions/warehouses/$WAREHOUSE_ID\" \ + --profile '$DB_PROFILE' \ + --json '{\"access_control_list\":[{\"service_principal_name\":\"$SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}'" +else + warn "No warehouse found. Create one and grant CAN_USE manually." +fi + +# Executed, not described. These were `note` lines telling the operator to paste +# SQL by hand; the backquoted principal cannot survive `--json "..."`, so in +# practice the grants were skipped and Genie could not read the data. +if [[ -n "$WAREHOUSE_ID" && "$WAREHOUSE_ID" != "" && "$DRY_RUN" == "false" ]]; then + firefly_sql "$WAREHOUSE_ID" "GRANT USE SCHEMA ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$SP_CLIENT_ID\`" >/dev/null \ + && ok "granted USE SCHEMA to agent SP" || warn "could not grant USE SCHEMA to agent SP" + firefly_sql "$WAREHOUSE_ID" "GRANT SELECT ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$SP_CLIENT_ID\`" >/dev/null \ + && ok "granted SELECT to agent SP" || warn "could not grant SELECT to agent SP" +fi +note "Grant USE SCHEMA + SELECT on your data schemas via a SQL warehouse:" +note " GRANT USE SCHEMA, SELECT ON SCHEMA $UC_CATALOG.your_schema TO \`$SP_CLIENT_ID\`;" + +fi +stop_if_done "6" + +# ─── Phase 6b: Create guest SP with M2M credentials ────────────────────────── +header "Phase 6b — Create guest service principal" +if run_phase "6b"; then + +step "Create workspace SP" +if [[ "$DRY_RUN" == "false" ]]; then + # SCIM display names are NOT unique: `service-principals create` on a re-run + # makes a DUPLICATE SP (new client id + secret) and orphans the old one. + # Reuse the existing firefly-guest-sp if one is already present. + GUEST_SP_RESP=$(databricks service-principals list \ + --filter 'displayName eq "firefly-guest-sp"' -o json --profile "$DB_PROFILE" 2>/dev/null \ + | python3 -c "import sys,json +l=json.load(sys.stdin) or [] +m=[s for s in l if s.get('displayName')=='firefly-guest-sp'] +print(json.dumps(m[0]) if m else '')" 2>/dev/null || echo "") + if [[ -n "$GUEST_SP_RESP" ]]; then + ok "Guest SP 'firefly-guest-sp' already exists — reusing (idempotent re-run)." + else + GUEST_SP_RESP=$(databricks service-principals create \ + --display-name "firefly-guest-sp" \ + -o json \ + --profile "$DB_PROFILE") + fi + # CLI returns SCIM camelCase: applicationId, not application_id + GUEST_SP_CLIENT_ID=$(echo "$GUEST_SP_RESP" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['applicationId'])") + GUEST_SP_NUM_ID=$(echo "$GUEST_SP_RESP" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + ok "Guest SP client ID: $GUEST_SP_CLIENT_ID" + ok "Guest SP numeric ID: $GUEST_SP_NUM_ID" +else + run "databricks service-principals create --display-name 'firefly-guest-sp' -o json --profile '$DB_PROFILE'" + GUEST_SP_CLIENT_ID="" + GUEST_SP_NUM_ID="" +fi + +echo +step "Generate OAuth M2M secret at workspace level (no account console needed)" +if [[ "$DRY_RUN" == "false" ]]; then + GUEST_SP_SECRET=$(databricks service-principal-secrets-proxy create \ + "$GUEST_SP_NUM_ID" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['secret'])") + + store_secret GUEST_SP_CLIENT_ID "$GUEST_SP_CLIENT_ID" + store_secret GUEST_SP_SECRET "$GUEST_SP_SECRET" +else + run "databricks service-principal-secrets-proxy create '\$GUEST_SP_NUM_ID' -o json --profile '$DB_PROFILE'" + run "store_secret GUEST_SP_CLIENT_ID && store_secret GUEST_SP_SECRET # → state.env" +fi + +echo +step "Grant guest SP warehouse + agent app CAN_USE" +if [[ -n "$WAREHOUSE_ID" && "$WAREHOUSE_ID" != "" ]]; then + note "Granting CAN_USE on warehouse $WAREHOUSE_ID for guest SP..." + run "databricks api patch \ + \"/api/2.0/permissions/warehouses/$WAREHOUSE_ID\" \ + --profile '$DB_PROFILE' \ + --json '{\"access_control_list\":[{\"service_principal_name\":\"$GUEST_SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}'" +else + warn "WAREHOUSE_ID not set (run Phase 6 first). Grant guest SP warehouse CAN_USE manually." +fi + +note "Granting CAN_USE on agent app $AGENT_APP_NAME for guest SP..." +run "databricks api patch \ + \"/api/2.0/permissions/apps/$AGENT_APP_NAME\" \ + --profile '$DB_PROFILE' \ + --json '{\"access_control_list\":[{\"service_principal_name\":\"$GUEST_SP_CLIENT_ID\", \ + \"permission_level\":\"CAN_USE\"}]}'" + +if [[ -n "$WAREHOUSE_ID" && "$WAREHOUSE_ID" != "" && "$DRY_RUN" == "false" ]]; then + firefly_sql "$WAREHOUSE_ID" "GRANT USE CATALOG ON CATALOG \`$UC_CATALOG\` TO \`$GUEST_SP_CLIENT_ID\`" >/dev/null \ + && ok "granted USE CATALOG to guest SP" || warn "could not grant USE CATALOG to guest SP" + firefly_sql "$WAREHOUSE_ID" "GRANT USE SCHEMA ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$GUEST_SP_CLIENT_ID\`" >/dev/null \ + && ok "granted USE SCHEMA to guest SP" || warn "could not grant USE SCHEMA to guest SP" + firefly_sql "$WAREHOUSE_ID" "GRANT SELECT ON SCHEMA \`$UC_CATALOG\`.\`$UC_SCHEMA\` TO \`$GUEST_SP_CLIENT_ID\`" >/dev/null \ + && ok "granted SELECT to guest SP" || warn "could not grant SELECT to guest SP" +fi +note "Grant USE CATALOG / USE SCHEMA / SELECT via SQL warehouse if not already done:" +note " GRANT USE CATALOG ON CATALOG $UC_CATALOG TO \`$GUEST_SP_CLIENT_ID\`;" +note " GRANT USE SCHEMA, SELECT ON SCHEMA $UC_CATALOG.$UC_SCHEMA TO \`$GUEST_SP_CLIENT_ID\`;" + +fi +stop_if_done "6b" + +# ─── Phase 6c: data + Genie space ───────────────────────────────────────────── +# The runner had no 6c at all, so a fresh workspace finished "successfully" with +# an empty schema and a Genie that could not answer anything (#83). One shared +# script does the work, called exactly as BOOTSTRAP.md calls it. +header "Phase 6c — Grant the service principals on the Genie space" +if run_phase "6c"; then + +# Phase 3f seeded the data and created the space before the app was deployed, so the app has +# been scoped to it since it started. What is left needs the SPs to exist: CAN_RUN on the +# space and SELECT on its tables. There is NO redeploy here -- that redeploy existed only +# because the app used to be born in the wrong mode. +step "Grant agent + guest SP on the Genie space" +firefly_restore_phase6_context "$DB_PROFILE" +: "${GENIE_SPACE_ID:=$(read_secret GENIE_SPACE_ID 2>/dev/null || true)}" +: "${GENIE_MCP_MODE:=$(firefly_read_input GENIE_MCP_MODE 2>/dev/null || echo one)}" + +if [[ "${GENIE_MCP_MODE:-one}" != "space" || -z "${GENIE_SPACE_ID:-}" ]]; then + warn "no Genie space to grant on (mode=${GENIE_MCP_MODE:-unset})." + note "Guest users will reach the app and be unable to ask data questions." + note "Re-run Phase 3f if you expected a space to exist." +elif [[ "$DRY_RUN" == "true" ]]; then + note "[DRY-RUN] genie-data-setup.sh --seed skip --create-space no --space-ids $GENIE_SPACE_ID" +else + GENIE_GRANT_OUT="$(bash "$BOOTSTRAP_SCRIPT_DIR/genie-data-setup.sh" \ + --catalog "$UC_CATALOG" --schema "$UC_SCHEMA" --profile "$DB_PROFILE" \ + --warehouse-id "${WAREHOUSE_ID:-}" \ + --phase 6c \ + --seed skip --create-space no \ + --space-ids "$GENIE_SPACE_ID" \ + --grant-guest "${GRANT_GUEST_SPACE_ACCESS:-yes}" \ + --guest-sp "${GUEST_SP_CLIENT_ID:-}" \ + --agent-sp "${SP_CLIENT_ID:-}")" \ + || warn "Phase 6c grants reported a problem (continuing)" + eval "$GENIE_GRANT_OUT" + ok "granted on Genie space $GENIE_SPACE_ID" +fi + +fi + +header "Phase 7 — Neon database" +if run_phase "7"; then + +note "Using neonctl credentials from Phase 1b (no API key needed)" + +note "Fetching Neon org ID..." +if [[ "$DRY_RUN" == "false" ]]; then + ORG_ID=$(neonctl orgs list --output json 2>/dev/null \ + | python3 -c "import sys,json; orgs=json.load(sys.stdin); print(orgs[0]['id'] if orgs else '')" \ + || echo "") + note "Org ID: ${ORG_ID:-}" +else + run "neonctl orgs list --output json" + ORG_ID="" +fi + +note "Creating Neon project..." +if [[ "$DRY_RUN" == "false" ]]; then + ORG_FLAG=(); [[ -n "$ORG_ID" ]] && ORG_FLAG=(--org-id "$ORG_ID") + # Neon project names are NOT unique (id-based): `projects create` on a re-run + # makes a SECOND project, orphans the first, and can trip the project quota. + # Reuse an existing project with the same name if present. + PROJECT_ID=$(firefly_neon_project_id "${ORG_FLAG[@]}") + if [[ -n "$PROJECT_ID" ]]; then + ok "Neon project '$NEON_PROJECT_NAME' exists — reusing ($PROJECT_ID)." + else + # Resolve the id by re-listing rather than by parsing the create response. + # `create` succeeds server-side before any parse of its output can fail, so + # a parse bug there silently orphans a real project — which is how two + # `firefly-genie` projects appeared on 2026-07-25. One lookup path, and a + # create whose response we mis-read is still discoverable. + neonctl projects create --name "$NEON_PROJECT_NAME" "${ORG_FLAG[@]}" --output json >/dev/null + PROJECT_ID=$(firefly_neon_project_id "${ORG_FLAG[@]}") + ok "Project created: $PROJECT_ID" + fi + if [[ -z "$PROJECT_ID" ]]; then + fail "could not resolve a Neon project id for '$NEON_PROJECT_NAME'." + note "An empty id makes the next call ambiguous ('Multiple projects found')." + note "Check: neonctl projects list ${ORG_FLAG[*]}" + exit 1 + fi + + DB_URL=$(neonctl connection-string --project-id "$PROJECT_ID" --pooled) + store_secret DATABASE_URL "$DB_URL" +else + run "neonctl projects create --name '$NEON_PROJECT_NAME' --output json" + run "neonctl connection-string --project-id '' --pooled" +fi + +echo +step "Drizzle migrations" +if [[ "$DRY_RUN" == "false" ]]; then + run "cd '$REPO_DIR' && pnpm install" + require_secret DB_URL "DATABASE_URL" || exit 1 + run "cd '$REPO_DIR' && DATABASE_URL='$DB_URL' node_modules/.bin/drizzle-kit push" +else + run "cd '$REPO_DIR' && pnpm install" + run "cd '$REPO_DIR' && source .firefly-bootstrap/state.env && node_modules/.bin/drizzle-kit push" +fi + +fi +stop_if_done "7" + +# ─── Phase 8: Vercel frontend ───────────────────────────────────────────────── +header "Phase 8 — Vercel frontend" +if run_phase "8"; then + +step "8a. Create + link Vercel project (no Git integration)" +# `vercel link` on a NON-EXISTENT project CREATES it and then tries to wire up Git +# auto-deploy — detecting the clone's `origin` remote (upstream databrickslabs/firefly), +# prompting "which remote?", and calling Vercel's Git connect, which needs a Vercel↔GitHub +# Login Connection many accounts lack (→ HTTP 400 "add a Login Connection"). We deploy via +# the `vercel deploy` CLI and need NO Git integration, so PRE-CREATE the project +# (idempotent) — then `vercel link` just ATTACHES to an existing project and never touches +# Git. (Enable push-to-deploy later from the dashboard: Project → Settings → Git.) +run "vercel project add '$VERCEL_PROJECT' --scope '$VERCEL_TEAM' 2>/dev/null || true" +run "cd '$REPO_DIR' && vercel link --project '$VERCEL_PROJECT' --scope '$VERCEL_TEAM' --yes --non-interactive" + +# A Vercel project created without a framework preset (framework:null) builds `next build` +# but serves the output as STATIC → every route 404s despite a "Ready" deployment. Force +# the Next.js preset on the linked project via the API (verified: flips all routes 200). +if [[ "$DRY_RUN" == "false" ]]; then + # Token sources in order: explicit env (CI / non-interactive), then the CLI's store on + # macOS, then its XDG location. This is load-bearing twice — the framework preset below + # (without it every route 404s) and the origin resolution in 8a-2 — so a single + # hardcoded path made it a silent 404 or a hard stop for anyone storing auth elsewhere. + V_TOKEN="${VERCEL_TOKEN:-}" + for _v_auth in "$HOME/Library/Application Support/com.vercel.cli/auth.json" \ + "$HOME/.local/share/com.vercel.cli/auth.json"; do + [[ -n "$V_TOKEN" ]] && break + [[ -f "$_v_auth" ]] || continue + V_TOKEN=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("token",""))' "$_v_auth" 2>/dev/null || echo "") + done + V_ORG=$(python3 -c "import json;print(json.load(open('$REPO_DIR/.vercel/project.json'))['orgId'])" 2>/dev/null || echo "") + V_PROJ=$(python3 -c "import json;print(json.load(open('$REPO_DIR/.vercel/project.json'))['projectId'])" 2>/dev/null || echo "") + if [[ -n "$V_TOKEN" && -n "$V_PROJ" ]]; then + curl -s -X PATCH "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" \ + -H "Authorization: Bearer $V_TOKEN" -H "Content-Type: application/json" \ + -d '{"framework":"nextjs"}' -o /dev/null \ + && ok "Vercel project framework → nextjs (else Next.js routes 404)" \ + || warn "Couldn't PATCH framework=nextjs; if routes 404, set Framework Preset=Next.js in the dashboard." + else + warn "Couldn't read Vercel token/project id to set framework=nextjs; set it in the dashboard if routes 404." + fi +else + run "# PATCH Vercel project framework=nextjs via API (else Next.js routes 404 despite 'Ready')" +fi + +echo +step "8a-2. Resolve the origin Vercel serves this project on (API, not the CLI's stdout)" +# `.vercel.app` is globally unique across all Vercel accounts; when the name is +# taken Vercel assigns a RANDOM suffix (`demo` -> `demo-zeta-seven-61`), so the host is +# unguessable. Read it instead — and read it HERE, because the domain is allocated at +# project-creation time and is therefore known before the first deploy. +# Resolving pre-deploy is also what makes this correct on a RE-RUN: discovering the URL +# from `vercel deploy` output only yields the production domain on a project's FIRST +# deploy; on any later run a bare deploy is a preview with a per-deployment host, which +# silently becomes BETTER_AUTH_URL and reproduces #19 while reporting success. +if [[ "$DRY_RUN" == "false" ]]; then + if [[ -z "$V_TOKEN" || -z "$V_PROJ" ]]; then + fail "Cannot read the Vercel API token / project id — refusing to guess the app origin." + note "Set VERCEL_TOKEN, or re-run 'vercel login' so the CLI writes its auth store." + exit 1 + fi + V_TMP=$(mktemp -d) + curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ/domains?teamId=$V_ORG" > "$V_TMP/domains.json" || true + curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" > "$V_TMP/project.json" || true + APP_ORIGIN=$(python3 - "$V_TMP/domains.json" "$V_TMP/project.json" <<'PY' || true +import json, sys +try: + domains = json.load(open(sys.argv[1])); project = json.load(open(sys.argv[2])) +except Exception: + raise SystemExit +# Where production actually serves. Populated once a production deployment exists, and it +# can disagree with /domains, so it wins when present. +alias = [a for a in (((project.get("targets") or {}).get("production") or {}).get("alias") or []) if a] +if alias: + print("https://" + alias[0]); raise SystemExit +# Pre-deploy: the single verified .vercel.app assigned when the project was created. +hosts = [d["name"] for d in (domains.get("domains") or []) + if d.get("verified") and not d.get("gitBranch") and not d.get("redirect") + and str(d.get("name", "")).endswith(".vercel.app")] +if len(hosts) == 1: + print("https://" + hosts[0]) +PY +) + rm -rf "$V_TMP" + if [[ -z "$APP_ORIGIN" ]]; then + fail "No verified .vercel.app domain for '$VERCEL_PROJECT' — refusing to guess one (#19)." + note "Inspect with: vercel project inspect '$VERCEL_PROJECT' --scope '$VERCEL_TEAM'" + exit 1 + fi + ok "App origin: $APP_ORIGIN" + if [[ "$APP_ORIGIN" != "https://$VERCEL_PROJECT.vercel.app" ]]; then # fence-ok: detects the collision + note "'$VERCEL_PROJECT.vercel.app' belongs to another account — using the assigned domain." # fence-ok: diagnostic + fi + store_secret APP_ORIGIN "$APP_ORIGIN" +else + run "# GET /v9/projects//domains -> APP_ORIGIN (real serving host, never guessed)" + APP_ORIGIN="" +fi + +echo +step "8b. Generate secrets" +if [[ "$DRY_RUN" == "false" ]]; then + BETTER_AUTH_SECRET=$(openssl rand -base64 32) + ENCRYPTION_KEY=$(openssl rand -hex 32) + GUEST_API_SECRET=$(openssl rand -hex 64) + AGENT_APP_URL=$(databricks apps get "$AGENT_APP_NAME" -o json --profile "$DB_PROFILE" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('url',''))") + require_secret DB_URL "DATABASE_URL" || exit 1 +else + BETTER_AUTH_SECRET="" + ENCRYPTION_KEY="" + GUEST_API_SECRET="" + AGENT_APP_URL="" + DB_URL="" +fi + +step "8b-2. Clear stale JWKS (reused-DB safety for the freshly-minted BETTER_AUTH_SECRET)" +# Phase 7 REUSES a same-named Neon project, but we mint a NEW BETTER_AUTH_SECRET above on +# every run. Better Auth's jwt plugin stores a JWKS whose private key is encrypted under that +# secret; a jwks row left over from an earlier run (different secret) fails to decrypt, so +# EVERY GET /api/auth/get-session 500s — which silently bounces guest logins to the +# /sso-spn-login dead end. Clearing jwks makes Better Auth regenerate it under the current +# secret (its own recommended remediation). No-op on a fresh DB. Uses @neondatabase/serverless +# (already installed by Phase 7's pnpm install) so no psql dependency. +if [[ "$DRY_RUN" == "false" ]]; then + ( cd "$REPO_DIR" && DATABASE_URL="$DB_URL" node --input-type=module -e \ + 'import {neon} from "@neondatabase/serverless"; const sql=neon(process.env.DATABASE_URL); await sql.query("DELETE FROM jwks"); console.log("jwks cleared");' ) \ + && ok "Stale JWKS cleared (will regenerate under current BETTER_AUTH_SECRET)" \ + || warn "Could not clear jwks — if guest login 500s on /api/auth/get-session, run: DELETE FROM jwks;" +else + run "cd '$REPO_DIR' && DATABASE_URL='' node --input-type=module -e ''" +fi + +step "8b. Tier 1 env vars — required for guest login" +note "DO NOT set NEXT_PUBLIC_BETTER_AUTH_URL — causes CORS failures on preview deployments" +note "Omit SPN_AUTH_OKTA_* entirely — the plugin is conditional; absent vars are skipped" + +for SCOPE in preview production; do + note "Setting tier-1 vars for scope: $SCOPE" + # Empty means Phase 4 never created the app. The env add succeeds anyway and + # the frontend deploys pointing at nothing, which reads as a frontend bug. + if [[ -z "${AGENT_APP_URL:-}" && "$DRY_RUN" == "false" ]]; then + warn "AGENT_APP_URL is empty — Phase 4 produced no running app; the agent panel will not work." + fi + run "vercel env add DATABRICKS_AGENT_APP_URL $SCOPE --value '$AGENT_APP_URL' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add DATABASE_URL $SCOPE --value '$DB_URL' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add BETTER_AUTH_SECRET $SCOPE --value '$BETTER_AUTH_SECRET' --force --non-interactive --scope '$VERCEL_TEAM'" + # Production is the bootstrap's serving target and its origin is already known from + # 8a-2, so set it now — one deploy, no second pass. Preview is deliberately left unset: + # preview URLs are per-deployment, so pointing preview auth at production is wrong. + [[ "$SCOPE" == "production" ]] && \ + run "vercel env add BETTER_AUTH_URL $SCOPE --value '$APP_ORIGIN' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add ENCRYPTION_KEY $SCOPE --value '$ENCRYPTION_KEY' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add NEXT_PUBLIC_AGENT_ENABLED $SCOPE --value 'true' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add GUEST_API_SECRET $SCOPE --value '$GUEST_API_SECRET' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add SPN_AUTH_DATABRICKS_ACCOUNTS_URL $SCOPE --value 'https://accounts.cloud.databricks.com' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add SPN_AUTH_DATABRICKS_WORKSPACE_URL $SCOPE --value '$DATABRICKS_HOST' --force --non-interactive --scope '$VERCEL_TEAM'" + # Guest Catalog Explorer only lists catalogs whose name matches an allowed prefix + # (app default "firefly"). Set it to the catalog the user chose in Phase 0 ($UC_CATALOG) + # so guests can BROWSE the data provisioned there (#20) — the app's memory store lives in + # $UC_CATALOG too, so there's no separate "firefly" catalog to keep. Browse-only; data + # access is still governed by the guest SP's UC grants. + run "vercel env add GUEST_ALLOWED_CATALOG_PREFIXES $SCOPE --value '$UC_CATALOG' --force --non-interactive --scope '$VERCEL_TEAM'" +done + +echo +step "8b. Tier 2 env vars — admin Databricks OAuth (placeholder is safe for guest-only)" +note "genericOAuth receives these as a plain object — undefined does NOT crash the build." +note "Admin login will 404 at runtime with placeholders, but guest flow is unaffected." +for SCOPE in preview production; do + run "vercel env add DATABRICKS_U2M_CLIENT_ID $SCOPE --value 'placeholder' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add DATABRICKS_U2M_CLIENT_SECRET $SCOPE --value 'placeholder' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add DATABRICKS_ACCOUNT_ID $SCOPE --value '$DATABRICKS_ACCOUNT_ID' --force --non-interactive --scope '$VERCEL_TEAM'" + run "vercel env add SPN_AUTH_DATABRICKS_ACCOUNT_ID $SCOPE --value '$DATABRICKS_ACCOUNT_ID' --force --non-interactive --scope '$VERCEL_TEAM'" +done +note "To enable admin login later: replace 'placeholder' with real OAuth app credentials" +note "from: accounts.cloud.databricks.com → App connections → Register an app" + +echo +step "8c. Disable preview SSO protection" +run "vercel project protection disable '$VERCEL_PROJECT' --sso --scope '$VERCEL_TEAM'" + +echo +step "8d. Deploy (single pass — production env already points at the real origin)" +# Always --prod. A bare `vercel deploy` is production only on a project's FIRST deploy; +# on a re-run it produces a preview with a per-deployment host, which is how a +# discover-from-stdout flow silently sets BETTER_AUTH_URL to a dead origin (#19). +if [[ "$DRY_RUN" == "false" ]]; then + DEPLOY_URL=$(vercel deploy --prod --scope "$VERCEL_TEAM" 2>&1 | grep -oE 'https://[^ ]*\.vercel\.app' | tail -1) + ok "Deployed: ${DEPLOY_URL:-}" + : "${APP_ORIGIN:=$(read_secret APP_ORIGIN 2>/dev/null || true)}" + PREVIEW_URL="$APP_ORIGIN" # Phase 9 guest-entry URL (historical var name) + store_secret APP_ORIGIN "$APP_ORIGIN" + store_secret PREVIEW_URL "$PREVIEW_URL" + store_secret GUEST_API_SECRET "$GUEST_API_SECRET" +else + run "vercel deploy --prod --scope '$VERCEL_TEAM'" + PREVIEW_URL="$APP_ORIGIN" +fi + +echo +step "8e. Verify production serves the origin BETTER_AUTH_URL points at" +# #19 reports success at every earlier step and only surfaces later as "Invalid token" +# on the guest link, so the match is asserted here rather than assumed. +if [[ "$DRY_RUN" == "false" ]]; then + SERVING=$(curl -sf -H "Authorization: Bearer $V_TOKEN" \ + "https://api.vercel.com/v9/projects/$V_PROJ?teamId=$V_ORG" \ + | python3 -c 'import json,sys; t=(json.load(sys.stdin).get("targets") or {}).get("production") or {}; print("\n".join(t.get("alias") or []))' 2>/dev/null || true) + if grep -qxF "${APP_ORIGIN#https://}" <<<"$SERVING"; then + ok "Production serves $APP_ORIGIN" + else + fail "Production does NOT serve $APP_ORIGIN (aliases: $(tr '\n' ' ' <<<"$SERVING"))" + exit 1 + fi +else + run "# assert targets.production.alias contains APP_ORIGIN" +fi + +fi +stop_if_done "8" + +# ─── Phase 9: Verify ───────────────────────────────────────────────────────── +header "Phase 9 — Verify" +if run_phase "9"; then + +if [[ "$DRY_RUN" == "false" ]]; then + require_secret PREVIEW_URL "PREVIEW_URL" 2>/dev/null || { + read -rp " Paste the Vercel production URL: " PREVIEW_URL + } +fi + +step "App health check" +run "databricks apps get '$AGENT_APP_NAME' -o json --profile '$DB_PROFILE' \ + | python3 -c \"import sys,json; d=json.load(sys.stdin); \ + state=d['app_status']['state']; \ + print('app_status:', state); \ + exit(0 if state=='RUNNING' else 1)\"" + +echo +step "Guest login smoke test" +if [[ "$DRY_RUN" == "false" ]]; then + require_secret GUEST_API_SECRET_ "GUEST_API_SECRET" 2>/dev/null || \ + GUEST_API_SECRET_="$GUEST_API_SECRET" + require_secret GUEST_SP_CLIENT_ID "GUEST_SP_CLIENT_ID" || exit 1 + require_secret GUEST_SP_SECRET_VAL "GUEST_SP_SECRET" || exit 1 + WS=$(curl -sf -X POST "$PREVIEW_URL/api/guest/workspaces" \ + -H "X-API-Key: $GUEST_API_SECRET_" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"Bootstrap test\",\"workspaceUrl\":\"$DATABRICKS_HOST\"}") + WS_ID=$(echo "$WS" | python3 -c "import sys,json; print(json.load(sys.stdin)['workspace']['id'])") + ok "Workspace record created: $WS_ID" + + SPN=$(curl -sf -X POST "$PREVIEW_URL/api/guest/spns" \ + -H "X-API-Key: $GUEST_API_SECRET_" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"Bootstrap SPN\",\"clientId\":\"$GUEST_SP_CLIENT_ID\", \ + \"clientSecret\":\"$GUEST_SP_SECRET_VAL\",\"guestWorkspaceId\":\"$WS_ID\"}") + SPN_ID=$(echo "$SPN" | python3 -c "import sys,json; print(json.load(sys.stdin)['spn']['id'])") + ok "SPN record created: $SPN_ID" + + GU=$(curl -sf -X POST "$PREVIEW_URL/api/guest/users" \ + -H "X-API-Key: $GUEST_API_SECRET_" \ + -H "Content-Type: application/json" \ + -d "{\"orgName\":\"Bootstrap Org\",\"spnId\":\"$SPN_ID\"}") + LOGIN_URL=$(echo "$GU" | python3 -c "import sys,json; print(json.load(sys.stdin)['guestUser']['loginUrl'])") + ok "Guest login URL: $LOGIN_URL" + echo + note "Open this URL in a browser, sign in, open the Agent panel, and ask a question over your data." + note "Then start a new chat and ask the same fact back — to verify cross-session memory recall." +else + run "curl -s -X POST '$PREVIEW_URL/api/guest/workspaces' \ + -H 'X-API-Key: \$GUEST_API_SECRET' -H 'Content-Type: application/json' \ + -d '{\"name\":\"Test\",\"workspaceUrl\":\"$DATABRICKS_HOST\"}'" + run "# → SPN → user → loginUrl" +fi + +fi +stop_if_done "9" + +# ─── Done ───────────────────────────────────────────────────────────────────── +echo +echo "${bold}${green}╔══════════════════════════════════════════════════════════╗${reset}" +echo "${bold}${green}║ Bootstrap complete. ║${reset}" +echo "${bold}${green}╚══════════════════════════════════════════════════════════╝${reset}" +echo +if [[ "$DRY_RUN" == "false" ]]; then + # Lead the summary with the guest login URL — it's the one thing the user needs to open + # the app, so surface it first (not buried under the cleanup list). One-time, ~10 min. + echo " ${bold}▶ OPEN THE APP — guest login (one-time link, valid ~10 min):${reset}" + echo " ${bold}${LOGIN_URL:-}${reset}" + echo + note "Frontend (production): ${PREVIEW_URL:-}" + note "Agent app: $AGENT_APP_NAME (RUNNING)" + note "UC memory store: $UC_CATALOG.$UC_SCHEMA.firefly_managed_memory" + echo + note "Link expired/used? Re-run and re-execute Phase 9 (Enter-skip through 0–8) to mint a" + note "fresh one — it reads PREVIEW_URL / GUEST_API_SECRET / guest-SP creds from state.env." + echo + note "Resources to clean up when done:" + note " databricks apps delete $AGENT_APP_NAME --profile $DB_PROFILE" + note " databricks bundle destroy --profile $DB_PROFILE -t dev" + note " vercel remove $VERCEL_PROJECT --scope $VERCEL_TEAM" + note " Neon project: console.neon.tech → delete project" +fi diff --git a/scripts/check-no-dev-pypi-proxy.sh b/scripts/check-no-dev-pypi-proxy.sh new file mode 100755 index 0000000..178715d --- /dev/null +++ b/scripts/check-no-dev-pypi-proxy.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Check every tracked uv.lock for the known-dead Databricks PyPI proxy. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/corp-network.sh +source "$ROOT/scripts/lib/corp-network.sh" + +locks=() +while IFS= read -r lock; do + locks+=("$ROOT/$lock") +done < <(git -C "$ROOT" ls-files '*uv.lock') + +assert_uv_locks_not_dead_pypi_proxy "${locks[@]}" +printf 'OK: %s tracked uv.lock file(s) avoid %s\n' "${#locks[@]}" "$FIREFLY_UNSANCTIONED_PYPI_HOST" diff --git a/scripts/check-runbook-invariants.sh b/scripts/check-runbook-invariants.sh new file mode 100755 index 0000000..f3e7b85 --- /dev/null +++ b/scripts/check-runbook-invariants.sh @@ -0,0 +1,1687 @@ +#!/usr/bin/env bash +# check-runbook-invariants.sh — guard the invariants that ENV-0 taught us. +# +# Run locally: bash scripts/check-runbook-invariants.sh +# Run in CI: .github/workflows/runbook-invariants.yml +# +# BACKGROUND +# ENV-0 (corporate networks block public npm, so corepack cannot fetch pnpm) was found on +# 2026-07-11 and fixed by installing pnpm via npm. Because it was classified +# "environment/not-repo" it was never filed as an issue, its BOOTSTRAP.md warning was later +# deleted, and the runbook was then changed back to requiring corepack — reintroducing the +# original failure for the next person on a locked-down network. These checks make that +# specific regression, and the doc/runner drift that hid it, fail loudly instead. + +set -uo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +FAILED=0 +pass() { echo " ✓ $*"; } +bad() { echo " ✗ $*"; FAILED=1; } + +echo "== ENV-0 / runbook-parity invariants ==" + +# ── 1. The runbook must not require corepack for pnpm (ENV-0). ──────────────── +# corepack fetches its package manager from registry.npmjs.org and ignores the npm +# registry setting, so it hard-fails wherever public npm is blocked. +if grep -qE '^[[:space:]]*corepack[[:space:]]+(enable|prepare)' BOOTSTRAP.md; then + bad "BOOTSTRAP.md requires 'corepack enable/prepare' — blocked on corporate networks (ENV-0)." + echo " Use: npm install -g pnpm@ (npm honors the user's registry)" + grep -nE '^[[:space:]]*corepack[[:space:]]+(enable|prepare)' BOOTSTRAP.md | sed 's/^/ /' +else + pass "BOOTSTRAP.md does not require corepack for pnpm." +fi + +if grep -qE '^[[:space:]]*run "corepack[[:space:]]+(enable|prepare)' scripts/bootstrap.sh; then + bad "bootstrap.sh runs 'corepack enable/prepare' — blocked on corporate networks (ENV-0)." +else + pass "bootstrap.sh does not run corepack for pnpm." +fi + +# ── 2. Corporate-network handling must be reachable from the runbook. ───────── +# The original defect was that Phase 0 of BOOTSTRAP.md had zero runnable commands while all +# the bridges lived in bootstrap.sh. Anyone following the doc (the "Open in Cursor" path) +# therefore ran installs with no bridge set. +LIB="scripts/lib/corp-network.sh" +if [[ ! -f "$LIB" ]]; then + bad "$LIB is missing — the shared corporate-network library is the single source of truth." +else + pass "$LIB exists." + for consumer in BOOTSTRAP.md scripts/bootstrap.sh; do + if grep -qE 'lib/corp-network\.sh' "$consumer"; then + pass "$consumer references $LIB (no drift)." + else + bad "$consumer does not reference $LIB — runbook and runner can drift again." + fi + done + if grep -qE 'firefly_bridge_corp_network' BOOTSTRAP.md; then + pass "BOOTSTRAP.md Phase 0 invokes firefly_bridge_corp_network." + else + bad "BOOTSTRAP.md never invokes firefly_bridge_corp_network — Phase 0 would be prose-only again." + fi +fi + +# ── 3. Phase 0 of the runbook must contain at least one runnable command. ───── +# Phase 0 is everything before the '## Phase 1' heading. +PHASE0_FENCES=$(awk '/^## Phase 1/{exit} /^```bash/{n++} END{print n+0}' BOOTSTRAP.md) +if [[ "$PHASE0_FENCES" -gt 0 ]]; then + pass "BOOTSTRAP.md Phase 0 has $PHASE0_FENCES runnable bash block(s)." +else + bad "BOOTSTRAP.md Phase 0 has no runnable bash blocks — corporate-network setup is prose only." +fi + +# ── 4. The pnpm pin must agree everywhere. ─────────────────────────────────── +LIB_PIN=$(sed -nE 's/.*PNPM_VERSION:=([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' "$LIB" 2>/dev/null | head -1) +PKG_PIN=$(python3 -c 'import json;print(json.load(open("package.json")).get("packageManager","").split("@")[-1])' 2>/dev/null) +DOC_PIN=$(sed -nE 's/.*npm install -g pnpm@([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' BOOTSTRAP.md 2>/dev/null | head -1) +if [[ -n "$LIB_PIN" && "$LIB_PIN" == "$PKG_PIN" && "$LIB_PIN" == "$DOC_PIN" ]]; then + pass "pnpm pin consistent across lib / package.json / BOOTSTRAP.md ($LIB_PIN)." +else + bad "pnpm pin mismatch — lib='$LIB_PIN' package.json='$PKG_PIN' BOOTSTRAP.md='$DOC_PIN'." + echo " An unpinned pnpm resolves the 'latest' dist-tag, which has shipped a 12.x alpha" + echo " that ignores onlyBuiltDependencies (ERR_PNPM_IGNORED_BUILDS)." +fi + +# ── 5. Every library the runbook sources must work under bash AND zsh. ─────── +# BOOTSTRAP.md Phase 0a tells the reader to `source scripts/lib/*.sh` from their own shell, +# and zsh is the macOS default. Bash-only idioms silently break there: `declare -F name` is +# a function test in bash but declares a FLOAT in zsh (returns 0), so a +# `declare -F x || x() {…}` guard skips the definition and every call dies with +# "command not found". Caught on a live corp VM; assert both shells here. +# +# Derived from the runbook rather than hardcoded to one path: this check used to name +# corp-network.sh only, so a second library could be added with no portability coverage. +# Built with a read loop, not `mapfile`: macOS ships bash 3.2, where mapfile does not exist. +SOURCED_LIBS=() +while IFS= read -r _lib; do + [[ -n "$_lib" ]] && SOURCED_LIBS+=("$_lib") +done < <(sed -nE 's|^[[:space:]]*source[[:space:]]+(scripts/lib/[A-Za-z0-9_-]+\.sh).*|\1|p' BOOTSTRAP.md | sort -u) +if [[ "${#SOURCED_LIBS[@]}" -eq 0 ]]; then + bad "BOOTSTRAP.md sources no scripts/lib/*.sh — the shared-implementation path is gone." +else + pass "BOOTSTRAP.md sources ${#SOURCED_LIBS[@]} shared librar$([[ ${#SOURCED_LIBS[@]} -eq 1 ]] && echo y || echo ies): ${SOURCED_LIBS[*]}" +fi +for lib in "${SOURCED_LIBS[@]}"; do + if [[ ! -f "$lib" ]]; then + bad "BOOTSTRAP.md sources $lib, which does not exist." + continue + fi + for sh in bash zsh; do + if ! command -v "$sh" >/dev/null 2>&1; then + pass "$sh not installed — skipping portability check for $lib." + continue + fi + if "$sh" -n "$lib" 2>/dev/null && \ + "$sh" -c "source '$lib'; ok x >/dev/null && note x >/dev/null && warn x >/dev/null && fail x >/dev/null" >/dev/null 2>&1; then + pass "$lib sources cleanly under $sh (helpers resolve)." + else + bad "$lib is not usable under $sh — Phase 0a tells users to source it from their shell." + echo " Avoid bash-only idioms ('declare -F' is not a function test in zsh;" + echo " \${!var} indirect expansion does not exist there either)." + fi + done +done + +# ── 5a. Every function the runbook calls must actually be defined. ─────────── +# BOOTSTRAP.md called store_secret / read_secret for four days while both existed only in +# bootstrap.sh — and bootstrap.sh's read_secret took (VARNAME, _, KEY) while every runbook +# call site used $(read_secret KEY), so even copying it across did not work. Resolve every +# firefly-namespaced call against the libraries the runbook itself sources. +if [[ "${#SOURCED_LIBS[@]}" -gt 0 ]] && command -v bash >/dev/null 2>&1; then + # Command position only, inside ```bash fences only. `firefly_wheels` (a UC volume) and + # `firefly_managed_memory` (a table) are arguments, not calls; prose like + # `firefly_install_*` is not code at all. + CALLED=$(python3 - <<'PY' +import re +text = open("BOOTSTRAP.md").read() +NAME = r'(?:firefly_[a-z0-9_]+|store_secret|read_secret|require_secret|' \ + r'assert_bundle_quickstart_ran|check_sync_exclude_rules)' +found = set() +for m in re.finditer(r'^```bash\n(.*?)^```', text, re.S | re.M): + for raw in m.group(1).splitlines(): + line = re.sub(r'(^|\s)#.*$', '', raw).strip() + if not line: + continue + # start of line, after a separator, or inside $( ... ) + for cm in re.finditer(r'(?:^|[;&|]\s*|\$\(\s*|\bif\s+|\bthen\s+|\belse\s+|!\s*)(' + NAME + r')\b', line): + found.add(cm.group(1)) +print("\n".join(sorted(found))) +PY +) + MISSING="" + for fn in $CALLED; do + bash -c "$(printf 'source %q; ' "${SOURCED_LIBS[@]}") declare -F $fn >/dev/null" 2>/dev/null \ + || MISSING="$MISSING $fn" + done + if [[ -z "$MISSING" ]]; then + pass "every helper BOOTSTRAP.md calls is defined by a library it sources." + else + bad "BOOTSTRAP.md calls helpers that no sourced library defines:$MISSING" + echo " A reader following the runbook hits 'command not found'. Define them in" + echo " scripts/lib/ and source that file from Phase 0a." + fi +fi + +# ── 5b. Runnable blocks must parse under bash AND zsh. ─────────────────────── +# The reader pastes these into their own shell. A bash-only construct is a runbook bug, +# not a style issue: ${!key} produced '(eval):1: bad substitution' on a real run. +if command -v zsh >/dev/null 2>&1; then + BLOCK_ERRS=$(python3 - <<'PY' +import re, subprocess, sys, tempfile, os +text = open("BOOTSTRAP.md").read() +errs = [] +for i, m in enumerate(re.finditer(r'^```bash\n(.*?)^```', text, re.S | re.M), 1): + body = m.group(1) + line = text[:m.start()].count("\n") + 1 + with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False) as fh: + fh.write(body); path = fh.name + for sh in ("bash", "zsh"): + r = subprocess.run([sh, "-n", path], capture_output=True, text=True) + if r.returncode != 0: + errs.append(f"BOOTSTRAP.md:{line} block {i} does not parse under {sh}: " + + r.stderr.strip().splitlines()[-1 if r.stderr.strip() else 0]) + os.unlink(path) +print("\n".join(errs)) +PY +) + if [[ -z "$BLOCK_ERRS" ]]; then + pass "every \`\`\`bash block in BOOTSTRAP.md parses under bash and zsh." + else + bad "BOOTSTRAP.md has shell blocks that do not parse:" + echo "$BLOCK_ERRS" | sed 's/^/ /' + fi +fi + +# ── 5c. No runnable block may be a stub. ───────────────────────────────────── +# Phase 1b's Databricks CLI install and Phase 1e's gh install were both shipped as +# `command -v X || { }` / `{ : ; }`. They read as installers and do +# nothing, which is invisible on any machine that already has the tool. +STUBS=$(python3 - <<'PY' +import re +text = open("BOOTSTRAP.md").read() +out = [] +for m in re.finditer(r'^```bash\n(.*?)^```', text, re.S | re.M): + base = text[:m.start()].count("\n") + 1 + # Only command groups. `${#VAR}` / `${VAR}` are parameter expansions, and the + # bare-brace regex read `${#GUEST_API_SECRET}` as a group whose sole content + # was the comment `#GUEST_API_SECRET` - flagging correct shell as a stub. + for bm in re.finditer(r'(?/dev/null) +done + +if [[ -z "$GENIE_COUPLING_BAD" ]]; then + pass "Genie mode and space id are declared together and always passed together." +else + bad "GENIE_MCP_MODE=space can be set without GENIE_SPACE_ID — the app fails to boot (#83):$GENIE_COUPLING_BAD" +fi + +# ── 13. Seeding must be the user's choice, and offered (#83). ──────────────── +# The runbook used to forbid seeding while never asking, so "unless the user +# explicitly asks" was unreachable: no Phase 0 input mentioned it. An empty schema +# means Genie answers nothing, so the fix is an ask — not a silent default either way. +SEED_ASK_BAD="" +grep -qE '\*\*\[ASK — REQUIRED, BLOCKING\]\*\* `SEED_SAMPLE_DATA`' BOOTSTRAP.md \ + || SEED_ASK_BAD="$SEED_ASK_BAD BOOTSTRAP.md(no-blocking-ask)" +grep -qE '\*\*\[ASK — REQUIRED, BLOCKING\]\*\* `GENIE_SPACE_IDS`' BOOTSTRAP.md \ + || SEED_ASK_BAD="$SEED_ASK_BAD BOOTSTRAP.md(no-space-ids-ask)" +grep -qE '^ask SEED_SAMPLE_DATA' scripts/bootstrap.sh \ + || SEED_ASK_BAD="$SEED_ASK_BAD bootstrap.sh(no-ask)" +grep -qE '^ask GENIE_SPACE_IDS' scripts/bootstrap.sh \ + || SEED_ASK_BAD="$SEED_ASK_BAD bootstrap.sh(no-space-ids-ask)" +# The old prohibition must be gone: it contradicts the ask it predates. +grep -qE 'Do not auto-create seed tables during bootstrap unless the user explicitly asks' BOOTSTRAP.md \ + && SEED_ASK_BAD="$SEED_ASK_BAD BOOTSTRAP.md(stale-prohibition)" + +if [[ -z "$SEED_ASK_BAD" ]]; then + pass "seeding and Genie space setup are offered as Phase 0 blocking asks, in both surfaces." +else + bad "seeding is not a real user choice — the runbook and runner disagree (#83):$SEED_ASK_BAD" +fi + +# ── 14. Phase 6c must exist in BOTH the runbook and the runner (#83). ──────── +# bootstrap.sh had no 6c at all, so the automated path finished "successfully" +# with an empty schema. This is the same drift class as 8 and 9: one surface +# implements a phase and the other only describes it. +SIXC_BAD="" +grep -qE '^## Phase 6c' BOOTSTRAP.md || SIXC_BAD="$SIXC_BAD BOOTSTRAP.md(no-phase)" +grep -qE 'run_phase "6c"' scripts/bootstrap.sh || SIXC_BAD="$SIXC_BAD bootstrap.sh(no-phase)" +[[ -x scripts/genie-data-setup.sh ]] || SIXC_BAD="$SIXC_BAD genie-data-setup.sh(missing-or-not-executable)" +# Both surfaces must call the SAME implementation, or they drift again. +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + grep -qE 'genie-data-setup\.sh' "$f" || SIXC_BAD="$SIXC_BAD $f(does-not-call-shared-script)" +done + +if [[ -z "$SIXC_BAD" ]]; then + pass "Phase 6c exists in both surfaces and both call scripts/genie-data-setup.sh." +else + bad "Phase 6c is not implemented consistently — a fresh workspace can finish with no data (#83):$SIXC_BAD" +fi + +# ── 16. Point-of-use defaults, not source-time-only defaults. ─────────────── +# Two separate bugs, one cause: `: "${VAR:=default}"` runs once when the file is +# sourced, but the variable is READ later, inside a function. A caller that +# exports it empty in between gets the empty value. +# * INPUTS_DIR empty -> CA bundle written to /proxy-ca-bundle.pem (5 runs) +# * FIREFLY_PLACEHOLDER_EXPERIMENT_ID empty -> `grep -q ""` matches every line, +# so a HEALTHY bundle fails assert_bundle_quickstart_ran (3 runs) +POU_BAD="" +sed -n '/init_inputs_dir()/,/}/p' scripts/lib/corp-network.sh | grep -q 'INPUTS_DIR="\$HOME/.firefly-bootstrap"' || POU_BAD="$POU_BAD init_inputs_dir(no-point-of-use-default)" +sed -n '/assert_bundle_quickstart_ran()/,/^}/p' scripts/lib/runbook.sh | grep -q 'FIREFLY_PLACEHOLDER_EXPERIMENT_ID=123237888438046' || POU_BAD="$POU_BAD assert_bundle_quickstart_ran(no-point-of-use-default)" +if [[ -z "$POU_BAD" ]]; then + pass "helpers re-apply defaults at point of use, not only at source time." +else + bad "defaults that are applied only at source time break when a caller exports them empty:$POU_BAD" +fi + +# ── 17. A crashed deploy must not read as success. ────────────────────────── +# Databricks CLI v1.9.0 can panic on `bundle deploy` and still exit 0. Both +# surfaces must assert the app exists rather than trusting the exit code. +DEPLOY_ASSERT_BAD="" +grep -q 'did not create' BOOTSTRAP.md || DEPLOY_ASSERT_BAD="$DEPLOY_ASSERT_BAD BOOTSTRAP.md" +grep -q 'did not create the app' scripts/bootstrap.sh || DEPLOY_ASSERT_BAD="$DEPLOY_ASSERT_BAD bootstrap.sh" +if [[ -z "$DEPLOY_ASSERT_BAD" ]]; then + pass "Phase 4 asserts the app exists instead of trusting the deploy exit code." +else + bad "no post-deploy assertion — a panicking CLI that exits 0 reads as success:$DEPLOY_ASSERT_BAD" +fi + +# ── 18. Phase 5 must say the preview is missing, not fail opaquely. ───────── +# The most-reported gap across E2E runs (9 of 12). +# The first attempt at this was a PROBE of /api/2.0/memory-stores. That path +# returns "Error: Not Found", which matched none of its patterns, so it reported +# the preview as ENABLED on a workspace where setup then failed. Both surfaces +# must run the operation and classify the failure, and must not reintroduce the +# probe. +MEM_BAD="" +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + grep -qE 'not enabled\|NotImplemented\|preview' "$f" \ + || MEM_BAD="$MEM_BAD $f(does-not-classify-the-failure)" + grep -qE 'api get /api/2\.0/memory-stores' "$f" \ + && MEM_BAD="$MEM_BAD $f(broken-probe-is-back)" +done +if [[ -z "$MEM_BAD" ]]; then + pass "Phase 5 attempts the setup and classifies the failure (no unreliable probe)." +else + bad "Phase 5 cannot tell a missing preview from a real error:$MEM_BAD" +fi + +# ── 19. The SQL grants must be executable, not prose. ─────────────────────── +# They were commented-out SQL telling the reader to paste into a warehouse +# session. A backquoted principal cannot survive `--json "..."`, so in practice +# the grants were skipped and Genie could not read the data. +GRANT_BAD="" +grep -qE '^\s*#\s*GRANT (USE|SELECT)' BOOTSTRAP.md && GRANT_BAD="$GRANT_BAD BOOTSTRAP.md(grants-still-commented-out)" +grep -q 'firefly_sql' BOOTSTRAP.md || GRANT_BAD="$GRANT_BAD BOOTSTRAP.md(no-executed-grants)" +grep -q 'firefly_sql' scripts/bootstrap.sh || GRANT_BAD="$GRANT_BAD bootstrap.sh(no-executed-grants)" +if [[ -z "$GRANT_BAD" ]]; then + pass "the UC grants are executed via firefly_sql, not left as un-pasteable comments." +else + bad "the UC grants cannot actually be run as written:$GRANT_BAD" +fi + +# ── 20. A failed seed must say WHICH failure it was. ──────────────────────── +# One status, "source-unavailable", was reported for four different causes: the +# schema not existing yet, existing but empty, a real permission denial, and a +# warehouse error — with stderr discarded via 2>/dev/null so the server's own +# message was thrown away. On a fresh workspace the cause is almost always a +# race (samples is provisioned asynchronously; Phase 6c probed 40s early on an +# observed run), but it read as a permanent entitlement problem. +SEED_DIAG_BAD="" +grep -qE 'source-not-ready' scripts/genie-data-setup.sh \ + || SEED_DIAG_BAD="$SEED_DIAG_BAD genie-data-setup.sh(no-cause-classification)" +grep -qE 'source-denied' scripts/genie-data-setup.sh \ + || SEED_DIAG_BAD="$SEED_DIAG_BAD genie-data-setup.sh(cannot-report-a-denial)" +grep -qE 'FIREFLY_SEED_SOURCE_WAIT' scripts/genie-data-setup.sh \ + || SEED_DIAG_BAD="$SEED_DIAG_BAD genie-data-setup.sh(no-wait-for-async-samples)" +# The discarded-stderr pattern must not come back on the source probe. +grep -qE 'SHOW TABLES IN .*SRC_SCHEMA.*2>/dev/null' scripts/genie-data-setup.sh \ + && SEED_DIAG_BAD="$SEED_DIAG_BAD genie-data-setup.sh(stderr-discarded-again)" +grep -qE 'source-unavailable' BOOTSTRAP.md \ + && SEED_DIAG_BAD="$SEED_DIAG_BAD BOOTSTRAP.md(still-documents-the-catch-all)" +if [[ -z "$SEED_DIAG_BAD" ]]; then + pass "a failed seed reports which cause it was, and waits out async samples provisioning." +else + bad "the seed probe would collapse four causes into one misleading status:$SEED_DIAG_BAD" +fi + +# ── 21. No workspace attribution link in the guest panel. ─────────────────── +# The panel's audience is GUEST users, who have no Databricks workspace access, +# so a workspace link is dead for every one of them. It also named "Genie Agent" +# while GENIE_MCP_MODE defaults to a space — the wrong backend. Attribution is +# plain text; the link must not come back, and neither must the env var that fed +# it. +ATTRIB_LINK_BAD="" +APP_SRC="agent/patches/e2e-chatbot-app-next" +if [[ -d "$APP_SRC" ]]; then + grep -rqE 'genieOneUrl|buildGenieOneUrl' "$APP_SRC/client/src" "$APP_SRC/server/src" 2>/dev/null \ + && ATTRIB_LINK_BAD="$ATTRIB_LINK_BAD app(genieOneUrl-is-back)" + # A bare "/one?o=" means someone rebuilt the workspace-wide link by hand. + grep -rqE '/one\?o=' "$APP_SRC/client/src" "$APP_SRC/server/src" 2>/dev/null \ + && ATTRIB_LINK_BAD="$ATTRIB_LINK_BAD app(workspace-link-rebuilt)" +fi +# The env var must not be reintroduced as a real setting (prose explaining its +# removal is fine, so only an actual `name:`/export counts). +grep -qE '^\s*(- name: GENIE_ONE_URL|export GENIE_ONE_URL|GENIE_ONE_URL=)' agent/databricks.yml scripts/bootstrap.sh BOOTSTRAP.md 2>/dev/null \ + && ATTRIB_LINK_BAD="$ATTRIB_LINK_BAD GENIE_ONE_URL(set-again)" +if [[ -z "$ATTRIB_LINK_BAD" ]]; then + pass "the guest panel carries no workspace attribution link (dead for guests, wrong backend)." +else + bad "a workspace attribution link is back in a guest-facing panel:$ATTRIB_LINK_BAD" +fi + +# ── 22. Installers must be able to verify their downloads. ────────────────── +# The astral.sh uv installer checks its download with `sha256sum`, which does not +# exist on macOS 14 or earlier (the tool there is `shasum`). Without a shim it +# prints "skipping sha256 checksum verification" and installs an UNVERIFIED +# binary — the default path on a clean VM, with the message buried in a long +# install. Reported by the E2E agent once the gap prompt asked about detours as +# well as workarounds. +SHA_SHIM_BAD="" +grep -q 'firefly_ensure_sha256sum()' scripts/lib/runbook.sh \ + || SHA_SHIM_BAD="$SHA_SHIM_BAD runbook.sh(no-shim-helper)" +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + # The shim has to run BEFORE the installer, or it changes nothing. + python3 - "$f" <<'PYCHK' || SHA_SHIM_BAD="$SHA_SHIM_BAD $f(shim-not-before-uv-install)" +import sys +t = open(sys.argv[1]).read() +shim = t.find('firefly_ensure_sha256sum') +inst = t.find('astral.sh/uv/install.sh') +sys.exit(0 if (shim != -1 and inst != -1 and shim < inst) else 1) +PYCHK +done +if [[ -z "$SHA_SHIM_BAD" ]]; then + pass "the uv installer can verify its own download (sha256sum shim precedes it)." +else + bad "uv would install an unverified binary on macOS 14 and earlier:$SHA_SHIM_BAD" +fi + +# ── 23. The reported Lakebase must be the one that exists. ────────────────── +# Passing --app-name for an app that already exists makes quickstart bind Lakebase +# from that app and ignore --lakebase-create-new. The requested project is never +# created, Phase 3a still reports PASS, and the summary names a resource that was +# never created. Observed across two passes of one run: created +# firefly-lb-0727083127, reported firefly-lb-0727090103. +LB_TRUTH_BAD="" +grep -q 'firefly_reconcile_lakebase()' scripts/lib/runbook.sh \ + || LB_TRUTH_BAD="$LB_TRUTH_BAD runbook.sh(no-reconcile-helper)" +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + grep -q 'firefly_reconcile_lakebase' "$f" || LB_TRUTH_BAD="$LB_TRUTH_BAD $f(does-not-reconcile)" + # The reconcile must run AFTER quickstart, or there is nothing to read. + python3 - "$f" <<'PYCHK' || LB_TRUTH_BAD="$LB_TRUTH_BAD $f(reconcile-before-quickstart)" +import sys +t = open(sys.argv[1]).read() +qs = t.find('quickstart.py') +rc = t.find('firefly_reconcile_lakebase') +sys.exit(0 if (qs != -1 and rc != -1 and qs < rc) else 1) +PYCHK +done +# The reconciled name must PERSIST. An in-shell export is lost when a resumed run +# re-sources inputs.env, which resurrects the requested-but-never-created name. +grep -q 'firefly_store_input LAKEBASE_NAME' scripts/lib/runbook.sh \ + || LB_TRUTH_BAD="$LB_TRUTH_BAD runbook.sh(reconciled-name-not-persisted)" +# And the create path must not read as though it will provision the name, right up +# until a post-hoc warning. Say it in advance when the app already exists. +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + grep -q 'firefly_warn_existing_app_wins' "$f" \ + || LB_TRUTH_BAD="$LB_TRUTH_BAD $f(no-advance-warning)" +done +if [[ -z "$LB_TRUTH_BAD" ]]; then + pass "the reported Lakebase name is reconciled, persisted, and pre-announced." +else + bad "the summary could name a Lakebase project that was never created:$LB_TRUTH_BAD" +fi + +# ── 24. A tool must not advertise the version the pin forbids. ────────────── +# pnpm prints "Update available! 10.34.5 -> 12.0.0-alpha.16" — the exact alpha the +# pin exists to avoid, because it ignores onlyBuiltDependencies and fails with +# ERR_PNPM_IGNORED_BUILDS. A reader who follows the tool's own advice breaks the +# build, so the banner is suppressed and the runbook says to ignore it if it slips +# through. +PIN_ADVERT_BAD="" +grep -q 'NPM_CONFIG_UPDATE_NOTIFIER' scripts/lib/corp-network.sh \ + || PIN_ADVERT_BAD="$PIN_ADVERT_BAD corp-network.sh(notifier-not-suppressed)" +grep -qE 'Update available|IGNORE IT' BOOTSTRAP.md \ + || PIN_ADVERT_BAD="$PIN_ADVERT_BAD BOOTSTRAP.md(banner-not-documented)" +if [[ -z "$PIN_ADVERT_BAD" ]]; then + pass "pnpm's update banner cannot lead a reader onto the version the pin forbids." +else + bad "a tool advertises the one version that breaks the build:$PIN_ADVERT_BAD" +fi + +# ── 25. Recurring harmless noise must be documented where it appears. ─────── +# The Vercel CLI prints `Error: Failed to get package info ... dist-tags` on nearly +# every command behind a corp proxy. It is its own update check, it exits 0, and +# the command succeeds — but it is on stderr, prefixed "Error:", and printed before +# the real output, so it reads like an auth/install failure. SEVEN separate E2E +# agents flagged it, each rediscovering it, because the runbook never mentioned it. +# "Accepted" in a triage registry is not the same as telling the reader. +NOISE_DOC_BAD="" +grep -q 'dist-tags' BOOTSTRAP.md \ + || NOISE_DOC_BAD="$NOISE_DOC_BAD BOOTSTRAP.md(noise-undocumented)" +if [[ -z "$NOISE_DOC_BAD" ]]; then + pass "the recurring vercel 'Error:' line is documented as harmless where it appears." +else + bad "output that reads like a failure but is not is left unexplained:$NOISE_DOC_BAD" +fi + +# ── 26. No command may run before the tool it needs is installed. ─────────── +# I added an IP-allowlist check to Phase 0 so it would be seen early, and it called +# `databricks` — which Phase 1b installs. It could not run as written; the E2E agent +# had to defer it. Early placement is worthless if the step cannot execute there. +ORDER_BAD="" +python3 - <<'PYCHK' || ORDER_BAD="$ORDER_BAD BOOTSTRAP.md(databricks-used-before-install)" +import re, sys +t = open('BOOTSTRAP.md').read() +install = t.find('firefly_install_databricks_cli') +if install == -1: + sys.exit(0) # nothing to order against +# Any `databricks ...` invocation inside a bash block before the install is a bug. +for m in re.finditer(r'^\s*databricks\s+\S', t[:install], re.M): + sys.exit(1) +sys.exit(0) +PYCHK +if [[ -z "$ORDER_BAD" ]]; then + pass "no runbook step invokes the databricks CLI before Phase 1b installs it." +else + bad "a step runs before the tool that runs it exists:$ORDER_BAD" +fi + +# ── 27. Never pipe a discarded-stderr call into a parser. ─────────────────── +# Written three separate times today, and each time the failure mode was the same: +# `cmd 2>/dev/null | python3 -c "json.load(...)"` turns ANY failure — endpoint 404, +# expired auth, tool not installed — into `JSONDecodeError: Expecting value`, which +# reads like a bad API response and hides the real cause. The seed probe had it, the +# Phase 5 preflight had it, and the IP-allowlist check had it. +# +# Two shapes evaded the first version of this check, and the second one shipped a +# false all-clear on the IP allowlist: +# +# 1. Assignment, then parse on a LATER line: +# ACL=$(databricks api get ... 2>/dev/null) +# ENABLED=$(printf '%s' "$ACL" | python3 -c "...json.load...") +# The discard and the parser are separate statements, so a same-line regex +# never sees them together. +# +# 2. An `except` that swallows instead of speaking: +# except Exception: raise SystemExit +# This satisfies "has a guard" while printing NOTHING — and the caller read +# that empty string as "no allowlist is enabled" and printed +# "ok: ... the app can reach it". A handler that exits silently is not a +# guard; it manufactures a confident wrong answer. +PIPE_BLIND_BAD="$(python3 - <<'PYCHK' +import re +t = open('BOOTSTRAP.md').read() +bad = [] + +def guarded(seg): + """A parser is guarded only if every failure path SAYS something.""" + if 'json.load' not in seg: + return True + for h in re.finditer(r'except[^\n:]*:\s*([^\n]*)\n((?:[ \t]+[^\n]*\n)*)', seg): + body = (h.group(1) + ' ' + h.group(2)).strip() + if not body: + return False + # Swallowing silently is as misleading as crashing: no print, no raise + # of anything a human reads. + if not re.search(r'print|sys\.stderr|write|echo', body): + # An empty fallback is legitimate when it feeds a self-correcting + # ACTION (create the missing thing, omit an optional flag) rather + # than a CLAIM about state. That difference is not mechanically + # detectable, so it must be declared and reviewed: mark the handler + # `# SAFE-EMPTY: `. Undeclared silence stays a failure — that + # is precisely how "ok: no enabled IP allowlist" got printed for a + # check that never ran. + if 'SAFE-EMPTY' not in body: + return False + return bool(re.search(r'except|if not raw', seg)) + +# Shape 1: discarded stderr piped straight into a parser. +for m in re.finditer(r'2>/dev/null[^\n]*\\?\n?[^\n]*\|[^\n]*python3[^\n]*', t): + seg = t[m.start():m.start() + 400] + if 'json.load' in seg and not guarded(seg): + bad.append('inline:' + m.group(0)[:52].replace('\n', ' ')) + +# Shape 2: stderr discarded into a variable that a parser later reads. +for m in re.finditer(r'(\w+)=\$\([^\n]*2>/dev/null\s*\)', t): + var, tail = m.group(1), t[m.end():m.end() + 900] + use = re.search(r'\$(?:\{)?' + re.escape(var) + r'\}?[^\n]*\|[^\n]*python3', tail) + if use and not guarded(tail[use.start():use.start() + 500]): + bad.append('via-$' + var) + +# Shape 3: any silent swallow, wherever it lives. +for m in re.finditer(r'except[^\n:]*:\s*(?:raise SystemExit|pass|sys\.exit\(\))\s*\n', t): + bad.append('silent-swallow:' + t[m.start():m.start() + 34].replace('\n', ' ')) + +print(' '.join(sorted(set(bad)))) +PYCHK +)" +if [[ -z "$PIPE_BLIND_BAD" ]]; then + pass "no runbook step pipes a discarded-stderr call into an unguarded parser." +else + bad "a parser will report JSONDecodeError instead of the real cause:$PIPE_BLIND_BAD" +fi + +# ── 15. The runner itself must parse, under bash AND zsh. ──────────────────── +# Invariant 5 checks every ```bash block in BOOTSTRAP.md and sources the shared +# libs, but nothing ever ran `bash -n` on scripts/bootstrap.sh. A stray edit left +# an unterminated `if` in Phase 3c and this suite still printed "All runbook +# invariants hold" — the runner is the one file guaranteed to be executed, and it +# was the one file not being checked. +SHELL_PARSE_BAD="" +for f in scripts/bootstrap.sh scripts/genie-data-setup.sh scripts/new-guest-link.sh; do + [[ -f "$f" ]] || continue + bash -n "$f" 2>/dev/null || SHELL_PARSE_BAD="$SHELL_PARSE_BAD $f(bash)" + command -v zsh >/dev/null 2>&1 && { zsh -n "$f" 2>/dev/null || SHELL_PARSE_BAD="$SHELL_PARSE_BAD $f(zsh)"; } +done +if [[ -z "$SHELL_PARSE_BAD" ]]; then + pass "bootstrap.sh and the helper scripts parse under bash and zsh." +else + bad "these scripts do not parse — the runner would die at that line:$SHELL_PARSE_BAD" +fi + +# ── 28. Phase 0a's confirmation command must list every var the bridge sets ─── +# The bridge exported CURL_CA_BUNDLE and REQUESTS_CA_BUNDLE, and Phase 9's smoke +# tests depend on CURL_CA_BUNDLE, but the documented `env | grep` omitted both -- +# so a reader following the runbook could not verify the trust var the later phases +# rely on. Derive the expected set from the bridge itself rather than restating it +# here, so adding an export cannot silently outrun the confirmation step. +CONFIRM_MISSING="$(python3 - <<'PYCHK' +import re +bridge = open('scripts/lib/corp-network.sh').read() +names = ('UV_DEFAULT_INDEX', 'UV_SYSTEM_CERTS', 'PIP_INDEX_URL', + 'COREPACK_NPM_REGISTRY', 'NODE_EXTRA_CA_CERTS', 'SSL_CERT_FILE', + 'CURL_CA_BUNDLE', 'REQUESTS_CA_BUNDLE') +want = [n for n in names if re.search(r'\b' + n + r'\b', bridge)] +# Every line of the runbook that documents an `env | grep` confirmation. +confirm = [l for l in open('BOOTSTRAP.md') if 'env |' in l and 'grep' in l] +blob = ' '.join(confirm) +print(' '.join(n for n in want if n not in blob)) +PYCHK +)" +if [[ -z "$CONFIRM_MISSING" ]]; then + pass "Phase 0a's confirmation lists every proxy/CA var the bridge sets." +else + bad "the bridge sets vars Phase 0a's confirmation cannot show:$CONFIRM_MISSING" +fi + +# ── 29. What Phase 0 CLAIMS the sourced helpers do, they must actually do ────── +# BOOTSTRAP.md said $HOME/bin and $HOME/.local/bin "are added to PATH in Phase 0", +# and Phase 0a promised that re-sourcing the helpers restores a fresh shell. Both +# were false: firefly_bridge_corp_network only mkdir'd the dirs and the export lived +# in scripts/bootstrap.sh, which a reader following the runbook by hand never runs. +# After Phase 1, databricks and uv were not findable and the reader had to invent an +# export. A claim that only holds for the RUNNER is a defect for every human reader, +# so prove it against the sourced helper -- in a fresh shell, the way a reader gets it. +PATH_CLAIM_BAD="" +if grep -qE 'added to .?PATH.? in Phase 0' BOOTSTRAP.md; then + for d in bin .local/bin; do + env -i HOME="$HOME" PATH="/usr/bin:/bin" bash -c " + cd '$PWD' + source scripts/lib/corp-network.sh >/dev/null 2>&1 || exit 1 + FIREFLY_TRUST_PROXY_CA=1 firefly_bridge_corp_network >/dev/null 2>&1 + case \":\$PATH:\" in *\":\$HOME/$d:\"*) exit 0 ;; *) exit 1 ;; esac + " || PATH_CLAIM_BAD="$PATH_CLAIM_BAD \$HOME/$d" + done +fi +if [[ -z "$PATH_CLAIM_BAD" ]]; then + pass "the dirs Phase 0 claims to put on PATH are on PATH after sourcing the helpers." +else + bad "BOOTSTRAP.md claims Phase 0 adds these to PATH, but sourcing the helpers does not:$PATH_CLAIM_BAD" +fi + +# ── 30. quickstart's first-run "does not exist or is deleted" must be pre-empted ─ +# With --app-name, quickstart.py answers a not-yet-created app with +# Could not fetch app details: App with name '...' does not exist or is deleted +# and suggests `databricks bundle deployment bind` / `bundle deploy`. That is the +# normal first-run state -- Phase 4 creates the app -- but it reads as a recovery +# path for an app someone deleted, and a run reported chasing it. Technically +# accurate output that points at the wrong problem costs the same as an error. +FIRSTRUN_BAD="" +grep -q 'does not exist or is deleted' scripts/lib/runbook.sh \ + || FIRSTRUN_BAD="$FIRSTRUN_BAD runbook.sh(no-preempt)" +grep -q 'does not exist or is deleted' BOOTSTRAP.md \ + || FIRSTRUN_BAD="$FIRSTRUN_BAD BOOTSTRAP.md(undocumented)" +# The pre-empt is worthless if it only fires when the app EXISTS. +sed -n '/^firefly_warn_existing_app_wins()/,/^}/p' scripts/lib/runbook.sh | grep -q 'else' \ + || FIRSTRUN_BAD="$FIRSTRUN_BAD warn_existing_app_wins(no-absent-branch)" +# quickstart prints the bind suggestion on BOTH paths. Defusing only one of them leaves +# it reading as an actionable next step on the other, which is what a run reported after +# the absent-case fix shipped. Require both halves of the function to mention it. +BIND_HALVES="$(sed -n '/^firefly_warn_existing_app_wins()/,/^}/p' scripts/lib/runbook.sh \ + | awk 'BEGIN{half=0} + /^[[:space:]]*#/ {next} # a comment defuses nothing + /else/{half=1} + /deployment bind/ && /note|warn|echo/ {print half}' | sort -u | tr -d '\n')" +[[ "$BIND_HALVES" == "01" ]] \ + || FIRSTRUN_BAD="$FIRSTRUN_BAD warn_existing_app_wins(bind-not-defused-on-both-paths)" +if [[ -z "$FIRSTRUN_BAD" ]]; then + pass "quickstart's first-run 'does not exist or is deleted' noise is pre-empted." +else + bad "a first run will chase quickstart's bind suggestion:$FIRSTRUN_BAD" +fi + +# ── 31. Helpers must reach their own error reporting under `set -e` ──────────── +# firefly_ip_allowlist_status did `raw="$(databricks api get ...)"`. On a tier without +# the IP-allowlist feature the CLI exits 1, and an assignment from a failing command +# substitution ABORTS the shell under errexit in both bash and zsh. So the branch +# written to handle that tier gracefully instead killed Phase 1b and Phase 9 on exactly +# the workspaces that reach it -- worse than the wrong "ok" it replaced. firefly_sql had +# the same shape, with an `rc=$?` check immediately after that errexit never reached. +# +# Assert the DIAGNOSTIC, not merely that the shell survived: calling the helper as +# `fn || true` suppresses errexit for the whole function body, so a survival check +# passes even when the helper is broken. That was the first version of this invariant +# and it could never fail. What matters is that the helper still SPEAKS when its own +# CLI fails, so require the words it promises to emit. +ERREXIT_BAD="" +for _sh in bash zsh; do + # firefly_ip_allowlist_status must classify, not die. + _out="$("$_sh" -c " + set -e + cd '$PWD' + source scripts/lib/runbook.sh >/dev/null 2>&1 + databricks() { echo 'Error: IP access list is not available in the pricing tier' >&2; return 1; } + firefly_ip_allowlist_status myprof + " 2>/dev/null)" + case "$_out" in + unavailable:*|unknown:*|none|enabled:*) ;; + *) ERREXIT_BAD="$ERREXIT_BAD $_sh:firefly_ip_allowlist_status(silent)" ;; + esac + + # firefly_sql must print its submit-failed diagnostic, not vanish. + _out="$("$_sh" -c " + set -e + cd '$PWD' + source scripts/lib/runbook.sh >/dev/null 2>&1 + databricks() { echo 'Error: warehouse not found' >&2; return 1; } + firefly_sql 'SELECT 1' myprof mywh + " 2>&1 || true)" + case "$_out" in + *"statement submit failed"*) ;; + *) ERREXIT_BAD="$ERREXIT_BAD $_sh:firefly_sql(silent)" ;; + esac +done +if [[ -z "$ERREXIT_BAD" ]]; then + pass "helpers still report their own CLI failure under set -e (bash and zsh)." +else + bad "these helpers die before reporting, under set -e:$ERREXIT_BAD" +fi + +# ── 32. The app must be DEPLOYED in its final Genie mode, not corrected afterwards ── +# This used to assert that Phase 6c's redeploy waited for Phase 4's deployment to settle, +# because the app was deployed workspace-wide (no space existed yet) and redeployed into +# space mode later. That race is gone -- along with the redeploy -- now that Phase 3f +# creates the space BEFORE Phase 4. Retiring the old check rather than leaving it in place +# matters: it searched for the literal `genie_mcp_mode=space`, which no longer appears, so +# it had started passing vacuously. A check that cannot fail still reports success. +# +# What must hold now: the first deploy carries both Genie vars, and nothing redeploys to +# fix the mode afterwards. +DEPLOY_MODE_BAD="" +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + # The deploy that Phase 4 performs must pass both vars together. + python3 - "$f" <<'PYCHK' || DEPLOY_MODE_BAD="$DEPLOY_MODE_BAD $(basename "$f")(deploy-lacks-genie-vars)" +import re, sys +t = open(sys.argv[1]).read() +# Any bundle deploy line, plus the ~600 chars before it (where BUNDLE_VARS is built). +for m in re.finditer(r'bundle deploy', t): + seg = t[max(0, m.start() - 800):m.end() + 200] + if 'genie_mcp_mode' in seg and 'genie_space_id' in seg: + sys.exit(0) +sys.exit(1) +PYCHK + # And no bundle deploy may appear at or after Phase 6 -- that late deploy IS the + # correction this reordering removed. Matching the WORD "redeploy" flagged prose that + # denies one ("There is no redeploy here"), which is the third time a check here has been + # decided by documentation instead of by what executes. + python3 - "$f" <<'PYCHK' || DEPLOY_MODE_BAD="$DEPLOY_MODE_BAD $(basename "$f")(late-deploy-corrects-mode)" +import re, sys +t = open(sys.argv[1]).read() +m = re.search(r'(##|header ")\s*"?Phase 6', t) +if not m: + sys.exit(0) +tail = t[m.start():] +# A deploy command, not the word. Comment/prose lines are excluded. +for line in tail.split('\n'): + stripped = line.strip() + if stripped.startswith('#') or stripped.startswith('>'): + continue + if 'bundle deploy' in stripped: + sys.exit(1) +sys.exit(0) +PYCHK +done +if [[ -z "$DEPLOY_MODE_BAD" ]]; then + pass "the app is deployed in its final Genie mode; nothing redeploys to correct it." +else + bad "the app can still be born in the wrong Genie mode:$DEPLOY_MODE_BAD" +fi + +# ── 33. Value-returning helpers must keep stdout clean ──────────────────────── +# Callers use these as VALUE="$(fn ...)". warn/note/ok all echo to STDOUT, so any +# diagnostic emitted through them is captured INTO the value. genie-data-setup.sh +# shipped that defect once (progress folded into its key=value output), and +# read_secret nearly shipped it again with a duplicate-key warning. +STDOUT_BAD="" +# REPO_DIR, not FIREFLY_STATE_DIR: init_state_dir derives STATE_FILE from +# ${1:-${REPO_DIR:-$PWD}} and overwrites it unconditionally, so the first version of +# this check wrote its K=first/K=second probe lines straight into the repo's real +# .firefly-bootstrap/state.env -- every invocation of this suite mutating the operator's +# own state. A test must not write where the thing it tests writes. +_out="$(bash -c " + export REPO_DIR=\$(mktemp -d) + cd '$PWD' + source scripts/lib/corp-network.sh >/dev/null 2>&1 + source scripts/lib/runbook.sh >/dev/null 2>&1 + init_state_dir \"\$REPO_DIR\" >/dev/null 2>&1 + # Two assignments: whatever the helper wants to say about that must go to stderr. + printf 'export K=first\nexport K=second\n' >> \"\$STATE_FILE\" + read_secret K 2>/dev/null + rm -rf \"\$REPO_DIR\" +" 2>/dev/null)" +case "$_out" in + second) ;; + *) STDOUT_BAD="$STDOUT_BAD read_secret(stdout=$(printf '%s' "$_out" | tr '\n' '/' | cut -c1-40))" ;; +esac +if [[ -z "$STDOUT_BAD" ]]; then + pass "value-returning helpers keep diagnostics off stdout." +else + bad "a diagnostic is being captured as part of the value:$STDOUT_BAD" +fi + +# ── 34. firefly_store_inputs must cover every [ASK] key, and dodge ${!k} ─────── +# The runbook only ever said bootstrap.sh saves answers to inputs.env and showed a +# reader nothing, so an agent invented the loop -- reaching for `${!k}`, bash-only +# indirect expansion, which is `bad substitution` under zsh, macOS's default shell. +# The same hazard was already documented on read_secret, so leaving the safe form +# unwritten is what let it recur. Two things must hold: the helper's default list +# covers every [ASK] row, and nothing here teaches ${!k}. +ASK_DRIFT="$(python3 - <<'PYCHK' +import re +doc = open('BOOTSTRAP.md').read() +lib = open('scripts/lib/runbook.sh').read() +asked = re.findall(r'\*\*\[ASK[^\]]*\]\*\* `([A-Z_][A-Z0-9_]*)`', doc) +m = re.search(r'firefly_store_inputs\(\)\s*\{(.*?)\n\}', lib, re.S) +covered = set(re.findall(r'\b([A-Z_][A-Z0-9_]{2,})\b', m.group(1))) if m else set() +print(' '.join(sorted(set(asked) - covered))) +PYCHK +)" +INDIRECT_TAUGHT="" +# A ${!k} that is not immediately called out as the thing NOT to do. +python3 - <<'PYCHK' || INDIRECT_TAUGHT="BOOTSTRAP.md(teaches-\${!k})" +import re, sys +t = open('BOOTSTRAP.md').read() +for m in re.finditer(r'\$\{!\w+\}', t): + window = t[m.start():m.start() + 400] + if not re.search(r'bash-only|Do \*\*not\*\*|bad substitution', window): + sys.exit(1) +sys.exit(0) +PYCHK +if [[ -z "$ASK_DRIFT$INDIRECT_TAUGHT" ]]; then + pass "firefly_store_inputs covers every [ASK] key and no phase teaches \${!k}." +else + bad "headless Phase 0 persistence is incomplete:${ASK_DRIFT:+ uncovered:$ASK_DRIFT}$INDIRECT_TAUGHT" +fi + +# ── 35. A Genie space must not be abandoned on one failed read ───────────────── +# Phase 6c created a space, round-tripped 16 tables through it and granted CAN_RUN -- +# then a single GET failed, printed "not readable - staying on workspace-wide Genie", cleared +# GENIE_SPACE_ID and shipped the app on Genie Agent, which guest users cannot use. The +# same GET succeeded minutes later. Reads after a create are eventually consistent, so +# one failure means "not yet", not "not there", and `2>/dev/null` meant the only thing +# it could ever report was "not readable" whatever actually went wrong. +SPACE_READ_BAD="" +grep -q 'space_readable()' scripts/genie-data-setup.sh \ + || SPACE_READ_BAD="$SPACE_READ_BAD no-retry-helper" +# The decision points must use it rather than a bare single GET. +if grep -nE 'if get_space "\$[A-Za-z_]+" \| grep -q' scripts/genie-data-setup.sh >/dev/null 2>&1; then + SPACE_READ_BAD="$SPACE_READ_BAD single-GET-decides" +fi +# It has to actually loop, and it has to keep stderr, or it is the old check renamed. +sed -n '/^space_readable()/,/^}/p' scripts/genie-data-setup.sh | grep -q 'while' \ + || SPACE_READ_BAD="$SPACE_READ_BAD helper-does-not-retry" +sed -n '/^space_readable()/,/^}/p' scripts/genie-data-setup.sh | grep -q '2>&1' \ + || SPACE_READ_BAD="$SPACE_READ_BAD helper-discards-stderr" +# Falling back to workspace-wide Genie costs the guest experience, so it must name the space and +# where it came from -- not just say "not readable". +grep -q 'abandoning space' scripts/genie-data-setup.sh \ + || SPACE_READ_BAD="$SPACE_READ_BAD fallback-not-explained" +if [[ -z "$SPACE_READ_BAD" ]]; then + pass "a Genie space survives a transient read failure instead of falling back to workspace-wide Genie." +else + bad "Phase 6c can discard a space it just created:$SPACE_READ_BAD" +fi + +# ── 36. Phase 6 must not depend on a shell that may be gone ──────────────────── +# Phases 6, 6b and 6c read WAREHOUSE_ID, SP_CLIENT_ID, GUEST_SP_CLIENT_ID, +# GENIE_MCP_MODE and GENIE_SPACE_ID straight from the shell. A second pass in a fresh +# terminal lost them, and not one resulting error named the empty variable: +# WAREHOUSE_ID="" -> "No API found for 'PATCH /permissions/warehouses/'" +# GUEST_SP_CLIENT_ID="" -> "Principal: ServicePrincipalName() does not exist" +# GENIE_MCP_MODE="" -> the space-mode gate silently skipped, and the app never +# received genie_space_id at all +# The silent skip is the worst of the three: nothing failed, so nothing was investigated. +PHASE6_BAD="" +grep -q 'firefly_restore_phase6_context()' scripts/lib/runbook.sh \ + || PHASE6_BAD="$PHASE6_BAD no-restore-helper" +grep -q 'firefly_require()' scripts/lib/runbook.sh \ + || PHASE6_BAD="$PHASE6_BAD no-require-helper" +# Presence is not enough: BOOTSTRAP.md has several call sites, so requiring merely one +# passed even with a phase boundary left unguarded. Each place that CONSUMES the vars +# must have a restore ahead of it -- the same positional requirement as invariant 32. +PHASE6_POS="$(python3 - <<'PYCHK' +import re + +# Derive the consumers rather than listing them. The first version named two -- the +# warehouse PATCH and the space-mode gate -- and Phase 6's FIRST command is neither: it +# is the catalog PATCH, which read an empty $SP_CLIENT_ID and got back "UpdatePermissions +# Missing required field: principal". A hand-picked list of consumers has exactly the +# blind spot of whichever one nobody thought of, which is the bug it was meant to catch. +VARS = ('SP_CLIENT_ID', 'WAREHOUSE_ID', 'GUEST_SP_CLIENT_ID', + 'GENIE_MCP_MODE', 'GENIE_SPACE_ID') + +t = open('BOOTSTRAP.md').read() +# Phase 6 onward; earlier phases legitimately establish these values. +start = t.find('## Phase 6') +body = t[start:] if start != -1 else t + +bad = [] +for var in VARS: + for m in re.finditer(r'\$\{?' + var + r'\b', body): + line_start = body.rfind('\n', 0, m.start()) + 1 + line = body[line_start:body.find('\n', m.start())] + # Skip comments, the helper's own definition/mention, and assignments. + if re.match(r'\s*#', line) or 'restore_phase6_context' in line or 'firefly_require' in line: + continue + if re.search(r'\b' + var + r'=(?!=)', line) or ':=' in line: + continue + # Only calls that actually leave the machine can misattribute the cause -- but + # judge the LOGICAL command, not the physical line. $SP_CLIENT_ID appears on the + # `--json` continuation of a multi-line `databricks api patch`, so a per-line + # filter skipped the very call that shipped this bug. + cmd_start = line_start + while cmd_start > 0: + prev_end = body.rfind('\n', 0, cmd_start - 1) + prev = body[prev_end + 1:cmd_start - 1] + if prev.rstrip().endswith('\\'): + cmd_start = prev_end + 1 + else: + break + logical = body[cmd_start:body.find('\n', m.start())] + if not re.search(r'databricks |curl |firefly_sql', logical): + continue + before = body[:m.start()] + if 'firefly_restore_phase6_context' not in before: + ln = t[:start + m.start()].count('\n') + 1 + bad.append(f'{var}@{ln}') + break +print(' '.join(bad)) +PYCHK +)" +[[ -n "$PHASE6_POS" ]] && PHASE6_BAD="$PHASE6_BAD $PHASE6_POS" +grep -q 'firefly_restore_phase6_context' scripts/bootstrap.sh \ + || PHASE6_BAD="$PHASE6_BAD bootstrap.sh(no-restore)" +grep -q 'firefly_require WAREHOUSE_ID' BOOTSTRAP.md \ + || PHASE6_BAD="$PHASE6_BAD BOOTSTRAP.md(no-assert)" +# The space-mode gate must have an else that SAYS something. +python3 - <<'PYCHK' || PHASE6_BAD="$PHASE6_BAD space-gate-silent" +import re, sys +for path in ('BOOTSTRAP.md', 'scripts/bootstrap.sh'): + t = open(path).read() + # Anchor on the GATE, not on the first mention of the variable. + # + # The previous version searched from the first occurrence of GENIE_MCP_MODE, asked + # whether a gate appeared within 2000 chars, then looked for speech within 900 -- two + # spans that need not overlap. It duly landed on Phase 3f's firefly_store_input, found + # Phase 4's gate far downstream, and judged Phase 3f's neighbourhood for speech. Correct + # code failed. Find each gate and inspect that gate's own body. + gates = list(re.finditer(r'(?:if|elif)[^\n]*GENIE_MCP_MODE[^\n]*(?:=|!=)\s*.?space.?', t)) + if not gates: + sys.exit(0) # nothing gating on the mode here + # EVERY gate must speak, not merely one of them. Requiring "some gate speaks" let a + # silenced gate hide behind a talkative one -- verified by silencing Phase 6c's and + # watching this pass because Phase 4's was still loud. + for g in gates: + body = t[g.end():g.end() + 800] + if not re.search(r'(warn|note|echo|fail)\s', body): + sys.exit(1) +sys.exit(0) +PYCHK +if [[ -z "$PHASE6_BAD" ]]; then + pass "Phase 6 restores its context, asserts before use, and never skips in silence." +else + bad "Phase 6 breaks when the shell is fresh:$PHASE6_BAD" +fi + +# ── 37. The guest-grant answer must be asked everywhere and honoured exactly ─── +# GRANT_GUEST_SPACE_ACCESS used to be asked only when the operator SUPPLIED space ids, +# and defaulted to no -- so on the common path, where bootstrap creates the space, the +# guest SP got nothing and guest users could not query the space the app was pointed at. +# +# The first fix forced the grant for a bootstrap-created space. That was worse in a +# different way: Phase 0 presents this as a controlling input, so overriding it made the +# ask a lie, and two runs reported exactly that. The answer is now put on every path, +# defaults to yes so the guest flow works without anyone having to know, and is honoured +# as given -- with the cost of `no` stated where it takes effect. +GUEST_GRANT_BAD="" +grep -qE 'ask this on every path' BOOTSTRAP.md \ + || GUEST_GRANT_BAD="$GUEST_GRANT_BAD ask-still-conditional" +grep -qE 'GRANT_GUEST_SPACE_ACCESS:-no\b' BOOTSTRAP.md scripts/bootstrap.sh \ + && GUEST_GRANT_BAD="$GUEST_GRANT_BAD default-still-no" +# The override must be gone: no forcing of the grant based on who owns the space. +grep -q 'GUEST_GRANT_WANTED' scripts/genie-data-setup.sh \ + && GUEST_GRANT_BAD="$GUEST_GRANT_BAD answer-overridden" +grep -qE 'regardless of --grant-guest' scripts/genie-data-setup.sh \ + && GUEST_GRANT_BAD="$GUEST_GRANT_BAD answer-overridden-msg" +# And `no` must state what it costs, not pass in silence. +grep -q 'unable to ask' scripts/genie-data-setup.sh \ + || GUEST_GRANT_BAD="$GUEST_GRANT_BAD cost-of-no-not-stated" +if [[ -z "$GUEST_GRANT_BAD" ]]; then + pass "the guest-grant answer is asked on every path, honoured exactly, and its cost stated." +else + bad "the guest-grant ask does not match what happens:$GUEST_GRANT_BAD" +fi + +# ── 38. State must not depend on cwd, and an empty write must not erase a value ─ +# Two failures with one shape, both costing real time on a live run: +# * init_state_dir fell back to $PWD, so a phase run from anywhere but the repo read a +# DIFFERENT state.env. Phase 9 concluded GUEST_API_SECRET was "0 chars" and spent +# three minutes reminting and redeploying a secret that was already stored. +# * Phase 8d does store_secret PREVIEW_URL "$APP_ORIGIN"; in a fresh shell APP_ORIGIN +# was unset, so a good value was overwritten with "" and Phase 9's curls returned +# HTTP 000 with nothing to explain it. +STATE_BAD="" +# REPO_DIR must be recovered from inputs.env before any $PWD fallback. +python3 - <<'PYCHK' || STATE_BAD="$STATE_BAD cwd-dependent-state" +import re, sys +t = open('scripts/lib/runbook.sh').read() +m = re.search(r'init_state_dir\(\)\s*\{(.*?)\n\}', t, re.S) +if not m: + sys.exit(1) +# Strip comments before looking: the explanatory comment above this function mentions +# $PWD, and matching that made the check fail against correct code. Twice now a check +# here has been satisfied or broken by prose rather than by what executes. +body = '\n'.join(re.sub(r'#.*$', '', ln) for ln in m.group(1).split('\n')) +# The recovery has to come before the PWD fallback, or it never runs. +i_rec = body.find('inputs.env') +i_pwd = body.find('$PWD') +sys.exit(0 if (i_rec != -1 and i_pwd != -1 and i_rec < i_pwd) else 1) +PYCHK +sed -n '/^store_secret()/,/^}/p' scripts/lib/runbook.sh | grep -q 'refusing to overwrite' \ + || STATE_BAD="$STATE_BAD empty-write-erases" +# And the phase that hit it must recover rather than trust the shell. +grep -q 'APP_ORIGIN:=$(read_secret APP_ORIGIN' BOOTSTRAP.md \ + || STATE_BAD="$STATE_BAD phase8d-trusts-shell" +grep -q 'is NOT a domain mismatch' BOOTSTRAP.md \ + || STATE_BAD="$STATE_BAD phase8e-blames-domain" +if [[ -z "$STATE_BAD" ]]; then + pass "state resolves independently of cwd, and an empty write cannot erase a value." +else + bad "a lost shell variable can still destroy or misreport state:$STATE_BAD" +fi + +# ── 39. A re-store must recover the value first, or it saves nothing ─────────── +# Phase 8d ends with store_secret GUEST_API_SECRET "$GUEST_API_SECRET", re-storing a +# value minted back in 8b. After a shell boundary the variable is empty, so the line +# passes nothing. The empty-write guard (invariant 38) keeps the stored value, so no data +# is lost -- but the call is then a save that saves nothing, and APP_ORIGIN two lines +# above already recovered itself first. The asymmetry was the defect, and three keys had +# it. A store right after a mint is fine; only a RE-store needs the recovery. +RESTORE_BAD="$(python3 - <<'PYCHK' +import re +t = open('BOOTSTRAP.md').read() +bad = [] +for m in re.finditer(r'store_secret\s+([A-Z_][A-Z0-9_]*)\s+"\$\{?\1\}?"', t): + key = m.group(1) + window = t[max(0, m.start() - 700):m.start()] + # `eval "$(...)"` mints variables without the text `KEY=` appearing, so a purely + # textual "was it assigned nearby" test cannot see it. Phase 3f captures GENIE_SPACE_ID + # exactly that way and then stores it -- a mint-then-store, not a re-store -- and the + # first version of this check flagged it. + minted = (re.search(r'\b' + key + r'=(?!=)', window) + or re.search(r'eval "\$\(', window)) + recovered = re.search(r':\s*"\$\{' + key + r':=\$\(read_secret', window) + if not minted and not recovered: + bad.append(key) +print(' '.join(sorted(set(bad)))) +PYCHK +)" +if [[ -z "$RESTORE_BAD" ]]; then + pass "every re-store of a secret recovers the value before storing it." +else + bad "these re-stores pass an empty variable after a shell boundary:$RESTORE_BAD" +fi + +# ── 40. The shareable summary must not hand the recipient operator commands ───── +# The deployment summary is the one table an operator forwards to whoever will use the +# app. It carried `Expired or already used? | bash scripts/new-guest-link.sh --open`, +# which that person cannot run: the script needs a clone of this repo and reads +# PREVIEW_URL, GUEST_API_SECRET, GUEST_SP_CLIENT_ID and GUEST_SP_SECRET from +# .firefly-bootstrap/state.env, exiting if any is absent. So the row was useless to its +# reader, and the only way to make it work would be to hand over credentials. +SUMMARY_BAD="$(python3 - <<'PYCHK' +import re +t = open('BOOTSTRAP.md').read() +i = t.find('### Deployment summary') +if i == -1: + print('summary-section-missing'); raise SystemExit +# The table is the block of pipe-rows following the heading. +rows = [l for l in t[i:i + 3000].split('\n') if l.startswith('|')] +bad = [] +for r in rows: + # Anything the recipient would have to run locally, or any secret name. + if re.search(r'\b(bash|sh|databricks|vercel|neonctl|npm|pnpm|uv)\s+\S', r) \ + or re.search(r'scripts/\S+\.sh', r) \ + or re.search(r'GUEST_API_SECRET|GUEST_SP_SECRET|BETTER_AUTH_SECRET|ENCRYPTION_KEY', r): + bad.append(r.strip()[:60]) +print(' | '.join(bad)) +PYCHK +)" +if [[ -z "$SUMMARY_BAD" ]]; then + pass "the shareable summary asks nothing of its reader that needs the repo or a secret." +else + bad "the summary table hands the recipient something only the operator can do:$SUMMARY_BAD" +fi + +# ── 41. "Genie One" must not return as a current product name ─────────────────── +# The product is Genie Agent -- the app's own attribution component has returned +# 'Genie Agent' for a while, and the docs lagged it in 49 places. Three references +# survive on purpose, and only those three: passages explaining why a link LABELLED +# "Genie One" was removed. Renaming a quoted historical label makes its own explanation +# self-contradictory, so they are quoted and allowed. +# +# Note what this check does NOT touch: the mode VALUE is `one`, an identifier, and +# renaming it would be a breaking change to bundle variables for no reader's benefit. +# Where "Genie One" was doing the work of saying "workspace-wide", the fix was to say +# workspace-wide -- both modes are the Genie Agent, so swapping the name alone would have +# left `one` and `space` reading identically. +GENIE_NAME_BAD="$(python3 - <<'PYCHK' +import re, subprocess +files = subprocess.run( + ['grep','-rl','Genie One','--include=*.md','--include=*.tsx','--include=*.ts', + '--include=*.py','--include=*.sh','--include=*.yml','.'], + capture_output=True, text=True).stdout.split() +bad = [] +# Exclude this script: it must name the old label to explain and to search for it, and a +# check that flags its own definition is the third time a guard here has been decided by +# its own prose rather than by what it is checking. +SELF = 'check-runbook-invariants.sh' +for f in (x for x in files if 'node_modules' not in x and SELF not in x): + for i, line in enumerate(open(f, errors='replace'), 1): + if 'Genie One' not in line: + continue + # Allowed only as a quoted label being discussed, e.g. a "Genie One" link. + if re.search(r'["\u201c]Genie One["\u201d]', line): + continue + bad.append(f'{f}:{i}') +print(' '.join(bad)) +PYCHK +)" +if [[ -z "$GENIE_NAME_BAD" ]]; then + pass "\"Genie One\" appears only as a quoted historical label, never as the current name." +else + bad "\"Genie One\" is used as a current product name:$GENIE_NAME_BAD" +fi + +# ── 42. Nothing may pin the bootstrap branch, because it outlives itself ──────── +# genie-agent carries this runbook; the default branch does not. So today a clone must +# ask for genie-agent, and the day it merges and is deleted that same clone fails with +# "Remote branch genie-agent not found in upstream origin" -- at Phase 2, the first step +# a new reader takes. Dropping the branch breaks the opposite state. Neither fixed choice +# survives the transition, so every clone resolves the branch and falls back. +# +# This also covers the README's "Open in Cursor" deeplink, which cannot run shell: its +# prompt has to TELL the agent to check, since a URL cannot. +BRANCH_PIN_BAD="" +# A clone that names the branch literally rather than through the variable. +# +# Capture and test for emptiness rather than piping into `grep -q`: grep -q exits on its +# first match, SIGPIPEs the greps upstream, and under `set -o pipefail` that turns the +# whole pipeline non-zero -- so an `&& set the flag` after it never fired, and this arm +# was dead code that passed against a deliberately re-pinned clone. +# +# Also exclude this file: it must contain the pattern in order to search for it. +# --exclude-dir=.git as well: a commit message that QUOTES the old pin (explaining why it +# was removed) lives in .git/COMMIT_EDITMSG and is not source. Without this the check fails +# on its own commit message the moment the fix is described accurately. +# Filter paths explicitly rather than trusting grep's --include/--exclude-dir: on the BSD +# grep macOS ships, --exclude-dir=.git did not exclude and --include='*.md' still matched +# .git/COMMIT_EDITMSG, a file with no extension at all. So this check failed on its own +# commit message -- the one describing the fix -- which is a guard broken by being +# explained. Path filtering here is portable and does what it says. +PIN_HITS="$(grep -rn -- '--branch genie-agent' . 2>/dev/null \ + | grep -E '\.(md|sh):[0-9]+:' \ + | grep -v '/\.git/' \ + | grep -v node_modules \ + | grep -v 'check-runbook-invariants.sh' \ + | grep -vE ':[[:space:]]*#' || true)" +[[ -n "$PIN_HITS" ]] && BRANCH_PIN_BAD="$BRANCH_PIN_BAD literal-clone-pin($(printf '%s' "$PIN_HITS" | head -1 | cut -d: -f1-2))" +# Both clone paths must resolve rather than assume. +grep -q 'ls-remote --exit-code --heads' BOOTSTRAP.md \ + || BRANCH_PIN_BAD="$BRANCH_PIN_BAD runbook-does-not-resolve" +grep -q 'firefly_resolve_branch()' scripts/bootstrap.sh \ + || BRANCH_PIN_BAD="$BRANCH_PIN_BAD runner-does-not-resolve" +# And the clone must be proved right: assert the runbook is actually in the checkout. +grep -q 'BOOTSTRAP.md is not in this checkout' BOOTSTRAP.md \ + || BRANCH_PIN_BAD="$BRANCH_PIN_BAD no-post-clone-assert" +# The deeplink prompt must name the fallback, not just the branch. +python3 - <<'PYCHK' || BRANCH_PIN_BAD="$BRANCH_PIN_BAD deeplink-pins-branch" +import re, sys, urllib.parse +s = open('README.md').read() +m = re.search(r'\]\((https://cursor\.com/link/prompt\?text=[^)]+)\)', s) +if not m: + sys.exit(0) # no deeplink to guard +txt = urllib.parse.parse_qs(urllib.parse.urlparse(m.group(1)).query).get('text', [''])[0] +# Naming genie-agent is fine; naming it with no fallback is the defect. +sys.exit(0 if ('genie-agent' not in txt or 'default branch' in txt) else 1) +PYCHK +if [[ -z "$BRANCH_PIN_BAD" ]]; then + pass "no clone or deeplink pins the bootstrap branch without a fallback." +else + bad "this breaks the day genie-agent merges and is deleted:$BRANCH_PIN_BAD" +fi + +# ── 43. Components must not hand-build GitHub URLs ───────────────────────────── +# github-source-link.tsx derived its blob URL from a GITHUB_REPO constant and then built +# its RAW url from a second, separately hardcoded copy of the same owner/repo. Changing the +# constant would have moved one link and silently left the other pointing at the old +# repository -- a half-applied edit that stays invisible until a user clicks. Both now come +# from src/lib/repo-links.ts. +# +# This does NOT require the two components to share a repo or branch: the docs "edit" links +# and the source-view links legitimately differ, and forcing them together would be a +# behaviour change wearing a refactor's clothes. Only hand-rolled URL STRINGS are banned. +URLGEN_BAD="$(python3 - <<'PYCHK' +import os, re +bad = [] +for root, dirs, files in os.walk('src'): + dirs[:] = [d for d in dirs if d != 'node_modules'] + for fn in files: + if not fn.endswith(('.ts', '.tsx')): + continue + path = os.path.join(root, fn) + if path.endswith(os.path.join('lib', 'repo-links.ts')): + continue # the one module allowed to know the URL shapes + for i, line in enumerate(open(path, errors='replace'), 1): + if re.match(r'\s*(//|\*)', line): + continue # a comment naming a URL is documentation, not construction + if re.search(r'https://(github\.com|raw\.githubusercontent\.com)/\S*\$\{', line): + bad.append(f'{path}:{i}') +print(' '.join(bad)) +PYCHK +)" +if [[ -z "$URLGEN_BAD" ]]; then + pass "GitHub URLs are built only in src/lib/repo-links.ts." +else + bad "these hand-build a GitHub URL and can drift from their own constant:$URLGEN_BAD" +fi + +# --------------------------------------------------------------------------- +# invariant 44: a phase that CANNOT grant must not report that grants were REFUSED. +# +# Phase 3f creates the Genie space before any service principal exists, so it cannot grant +# and deliberately passes no --grant-guest. A missing flag defaulted to no, and the run told +# an operator who had answered yes that the guest SP would get NO access to the space -- +# while Phase 6c went on to grant it. Technically the flag was absent; what the reader took +# from it was that their Phase 0 answer had been discarded. A run reported it as a gap. +# +# So: 3f must pass --defer-grants, and the script must treat deferral as its own state rather +# than folding it into refusal. +GRANT_DEFER_BAD=() + +# Anchored on the invocation, not on a phase range. "Phase 4" is mentioned in prose and in +# comments well before Phase 3f's heading, so /Phase 3f/,/Phase 4/ collapsed to a three-line +# window and reported a file that did carry the flag -- the guard failing for its own reason +# rather than the runbook's, which is worse than not having it. +# +# The rule that actually matters: the invocation that CREATES the space (it passes +# --create-space with the Phase 0 answer) runs before the SPs exist, so it must defer. The +# grants invocation in 6c passes --create-space no and must NOT defer. +DEFER_SCAN="$(python3 - <<'PYCHK' +import re +bad = [] +for f in ('BOOTSTRAP.md', 'scripts/bootstrap.sh'): + text = open(f, errors='replace').read() + # each genie-data-setup.sh call, including its backslash-continued argument lines + for m in re.finditer(r'genie-data-setup\.sh((?:[^\n]*\\\n)*[^\n]*)', text): + args = m.group(1) + # A dry-run banner that NAMES the command is not the command. Counting + # note "[DRY-RUN] genie-data-setup.sh --create-space ..." as a call site made the + # guard demand a flag inside a string literal. + line_start = text.rfind('\n', 0, m.start()) + 1 + prefix = text[line_start:m.start()] + if re.search(r'\b(note|warn|echo|printf|step|ok)\b\s', prefix) or '[DRY-RUN]' in prefix: + continue + if '--create-space' not in args: + continue # not the creating call + if re.search(r'--create-space\s+(no|"no")', args): + continue # explicitly not creating + # Either answer is fine; SILENCE is not. A call made before the SPs exist defers; + # a recovery call made after they exist grants. What must never happen is creating a + # space with neither flag, where an absent --grant-guest is indistinguishable from + # a deliberate no from the operator -- which is how a documented recovery command + # came to drop guest access without saying so. (No apostrophes here: a lone one + # inside a heredoc nested in $( ) unbalances the bash 3.2 substitution scanner.) + if '--defer-grants' not in args and '--grant-guest' not in args: + line = text[:m.start()].count('\n') + 1 + bad.append(f'{f}:{line} creates the space with neither --defer-grants nor ' + f'--grant-guest, so an absent flag reads as a refusal') +print('\n'.join(bad)) +PYCHK +)" +[[ -n "$DEFER_SCAN" ]] && while IFS= read -r l; do + [[ -n "$l" ]] && GRANT_DEFER_BAD+=("$l") +done <<<"$DEFER_SCAN" + +# the script must branch on deferral BEFORE the refusal warning, and must not grant when deferring +SETUP=scripts/genie-data-setup.sh +grep -q 'DEFER_GRANTS' "$SETUP" \ + || GRANT_DEFER_BAD+=("$SETUP: no DEFER_GRANTS state, so 'cannot grant yet' and 'told not to grant' are the same thing") +DEFER_LINE="$(grep -n 'DEFER_GRANTS:-no' "$SETUP" | head -1 | cut -d: -f1)" +WARN_LINE="$(grep -n 'GRANT_GUEST_SPACE_ACCESS=no, so the guest SP gets NO access' "$SETUP" | head -1 | cut -d: -f1)" +if [[ -n "$DEFER_LINE" && -n "$WARN_LINE" && "$DEFER_LINE" -gt "$WARN_LINE" ]]; then + GRANT_DEFER_BAD+=("$SETUP: the refusal warning (line $WARN_LINE) is reached before the deferral check (line $DEFER_LINE)") +fi + +# Same defect, two more places a run found it: the grants-only call passed --seed no and was +# told "seeding declined at Phase 0" when Phase 0 said yes and Phase 3f had already seeded; +# and every message hardcoded "Phase 6c", so a reader in Phase 3f was pointed at a phase they +# had not reached. Both are a flag meaning "not here" being read back as a decision. +# Both of these were first written as a grep over the whole file, and both were vacuous: the +# dry-run banner mentions --seed skip, so the grep passed no matter what the real call did, +# and PHASE_LABEL appears in its own comment. Sabotaging each one changed nothing and the +# guard still said OK. Scope them to the invocation and the message. +SEED_SKIP_SCAN="$(python3 - <<'PYCHK' +import re +bad = [] +for f in ('BOOTSTRAP.md', 'scripts/bootstrap.sh'): + text = open(f, errors='replace').read() + for m in re.finditer(r'genie-data-setup\.sh((?:[^\n]*\\\n)*[^\n]*)', text): + args = m.group(1) + line_start = text.rfind('\n', 0, m.start()) + 1 + prefix = text[line_start:m.start()] + if re.search(r'\b(note|warn|echo|printf|step|ok)\b\s', prefix) or '[DRY-RUN]' in prefix: + continue # a banner naming the command is not the command + if not re.search(r'--create-space\s+"?no"?', args): + continue # only the grants-only call is in scope + if re.search(r'--seed\s+"?no"?', args): + line = text[:m.start()].count('\n') + 1 + bad.append(f'{f}:{line} grants-only call passes --seed no, so "not this phase" ' + f'is reported to the operator as "you declined seeding"') +print('\n'.join(bad)) +PYCHK +)" +[[ -n "$SEED_SKIP_SCAN" ]] && while IFS= read -r l; do + [[ -n "$l" ]] && GRANT_DEFER_BAD+=("$l") +done <<<"$SEED_SKIP_SCAN" + +# the warehouse line is the one a run actually complained about: it must carry the label +grep -qE 'note "Phase \$\{?PHASE_LABEL' "$SETUP" \ + || GRANT_DEFER_BAD+=("$SETUP: the warehouse message does not use PHASE_LABEL, so one caller sends its readers to the other caller phase") +for f in BOOTSTRAP.md scripts/bootstrap.sh; do + grep -q -- '--phase 3f' "$f" \ + || GRANT_DEFER_BAD+=("$f: Phase 3f does not tell the script which phase is running") +done + +if [[ "${#GRANT_DEFER_BAD[@]}" -eq 0 ]]; then + pass "a phase that cannot act yet says so, instead of reporting the answer as refused." +else + for b in "${GRANT_DEFER_BAD[@]}"; do bad "$b"; done +fi + +# --------------------------------------------------------------------------- +# invariant 45: a restore must look in every store a value is written to. +# +# store_secret writes state.env; firefly_store_input writes inputs.env. Phase 3f records +# GENIE_MCP_MODE with the latter, and firefly_restore_phase6_context read only the former -- +# so it never found the mode and announced "assuming space" about a value Phase 3f had already +# written down. Accurate about what the function could see, wrong about what was known, and a +# run reported it as a gap. +RESTORE_BAD=() +RESTORE_BODY="$(awk '/^firefly_restore_phase6_context\(\)/{f=1} f{print} f&&/^}/{exit}' scripts/lib/runbook.sh)" +if [[ -z "$RESTORE_BODY" ]]; then + RESTORE_BAD+=("scripts/lib/runbook.sh: firefly_restore_phase6_context not found - this guard is measuring nothing") +else + grep -q 'read_secret' <<<"$RESTORE_BODY" \ + || RESTORE_BAD+=("firefly_restore_phase6_context never reads state.env") + grep -q 'firefly_read_input' <<<"$RESTORE_BODY" \ + || RESTORE_BAD+=("firefly_restore_phase6_context never reads inputs.env, so any value written by firefly_store_input is invisible to it and gets 'assumed' instead") + # Comments are stripped first. The initial version matched the comment that EXPLAINS the old + # message and reported the fixed code as broken -- twice, because the awk range matched twice. + RESTORE_CODE="$(sed -E 's/[[:space:]]*#.*$//' <<<"$RESTORE_BODY")" + if grep -qE 'assuming space' <<<"$RESTORE_CODE"; then + RESTORE_BAD+=("firefly_restore_phase6_context still says 'assuming space' - name the stores it actually checked instead") + fi +fi + +if [[ "${#RESTORE_BAD[@]}" -eq 0 ]]; then + pass "a restore reads every store its values are written to before assuming one is unset." +else + for b in "${RESTORE_BAD[@]}"; do bad "$b"; done +fi + +# --------------------------------------------------------------------------- +# invariant 46: a grant loop must not report only its successes. +# +# grant_table_select sent every GRANT to /dev/null and counted the ones that returned 0, so it +# printed "granted SELECT on 15 table(s)" after Phase 3f had seeded 16 -- and a table the guest +# cannot read was indistinguishable from one it can. A run reported the discrepancy, which is +# the only reason anyone noticed. Failures must be named, and a coverage gap between the schema +# and the Genie space must be stated rather than left as a number the reader has to reconcile. +GRANT_SILENT_BAD=() +GTS="$(awk '/^grant_table_select\(\)/{f=1} f{print} f&&/^}/{exit}' scripts/genie-data-setup.sh)" +if [[ -z "$GTS" ]]; then + GRANT_SILENT_BAD+=("scripts/genie-data-setup.sh: grant_table_select not found - this guard measures nothing") +else + GTS_CODE="$(sed -E 's/[[:space:]]*#.*$//' <<<"$GTS")" + grep -qE 'failed=\$\(\(failed \+ 1\)\)' <<<"$GTS_CODE" \ + || GRANT_SILENT_BAD+=("grant_table_select does not count failures, so it can only report successes") + grep -qE 'FAILED' <<<"$GTS_CODE" \ + || GRANT_SILENT_BAD+=("grant_table_select never reports a failed grant, so an unreadable table looks readable") + grep -q 'first_err' <<<"$GTS_CODE" \ + || GRANT_SILENT_BAD+=("grant_table_select discards the grant error, leaving no way to know why one failed") + grep -q 'existing_tables' <<<"$GTS_CODE" \ + || GRANT_SILENT_BAD+=("grant_table_select never compares the space coverage with the schema, so a seeded-but-uncovered table is silent") +fi + +if [[ "${#GRANT_SILENT_BAD[@]}" -eq 0 ]]; then + pass "a grant loop names its failures and states any gap between the schema and the space." +else + for b in "${GRANT_SILENT_BAD[@]}"; do bad "$b"; done +fi + +# --------------------------------------------------------------------------- +# invariant 47: the Genie tool NAMES must be resolved per backend, like the path already is. +# +# Invariant 12 and the shared resolver fixed the PATH: one place decides whether this +# deployment talks to /api/2.0/mcp/genie or /api/2.0/mcp/genie/. The tool names were +# left hardcoded to the workspace-wide pair one layer down, and the two backends do not expose +# the same tools: +# +# /api/2.0/mcp/genie genie_ask, genie_poll_response +# /api/2.0/mcp/genie/ query_space_, poll_response_ +# +# Space is the default, so a real bootstrap run POSTed genie_ask to a space endpoint, got +# "-32602 BAD_REQUEST: Tool genie_ask does not exist", and the user saw "Genie ask failed: {}". +# Fixing the path while leaving the names is the same defect with a smaller blast radius. +GENIE_TOOLNAME_BAD=() +GT=agent/agent_server/genie_tools.py +GM=agent/agent_server/genie_mcp.py + +grep -q 'def genie_tool_names' "$GM" \ + || GENIE_TOOLNAME_BAD+=("$GM: no genie_tool_names() - the tool names cannot be resolved per backend") + +# genie_tools.py must not name a workspace-wide tool in a CALL. Mentioning it in a comment +# that explains the defect is fine and necessary. +GT_CODE="$(sed -E 's/[[:space:]]*#.*$//' "$GT" | grep -v '^[[:space:]]*"""')" +if grep -qE '_mcp_tool_call\([[:space:]]*"genie_(ask|poll_response)"' <<<"$GT_CODE"; then + GENIE_TOOLNAME_BAD+=("$GT: calls a hardcoded workspace-wide tool name; use genie_tool_names()") +fi +grep -q 'genie_tool_names' "$GT" \ + || GENIE_TOOLNAME_BAD+=("$GT: never calls genie_tool_names(), so it cannot be talking to the space backend") + +# A JSON-RPC error must reach the caller. `return {}` is what made a precise server message +# ("Tool genie_ask does not exist") surface as "Genie ask failed: {}". +# Scoped to the branch, not the file. The first version grepped the whole file for +# `return {"error"` and was satisfied by an unrelated one a few lines earlier, so replacing +# the JSON-RPC return with `return {}` -- the original bug, verbatim -- still passed. +if grep -qE 'raw\.get\("error"\)' "$GT"; then + ERR_BRANCH="$(awk '/raw\.get\("error"\)/{f=1} f{print; n++} n>8{exit}' "$GT")" + grep -qE 'return \{"error"' <<<"$ERR_BRANCH" \ + || GENIE_TOOLNAME_BAD+=("$GT: detects a JSON-RPC error but does not return it - the cause is discarded") +else + GENIE_TOOLNAME_BAD+=("$GT: does not check for a JSON-RPC error at all; a failed tool call reads as an empty result") +fi + +if [[ "${#GENIE_TOOLNAME_BAD[@]}" -eq 0 ]]; then + pass "Genie tool names are resolved per backend, and MCP errors reach the caller." +else + for b in "${GENIE_TOOLNAME_BAD[@]}"; do bad "$b"; done +fi + +# --------------------------------------------------------------------------- +# invariant 48: a lookup that CANNOT answer must not fall through to creating the resource. +# +# The Genie space lookup listed spaces with stderr sent to /dev/null and a parser that returned +# {} on any failure, so "the API did not answer" and "no such space exists" were one value -- +# and the branch on that value creates a space. One hiccup produced a second Genie space, and +# the evidence was destroyed by the same line that caused it. Three runs on one workspace took +# the count from 2 to 3. +# +# Creating on an unknown state is the specific mistake: reuse-or-create must have three +# outcomes, not two. +LOOKUP_BAD=() +GDS=scripts/genie-data-setup.sh +LOOKUP_BODY="$(awk '/WANT_TITLE="\$\(space_title_for/{f=1} f{print} f&&/GENIE_SPACE_SOURCE="created"/{exit}' "$GDS")" +if [[ -z "$LOOKUP_BODY" ]]; then + LOOKUP_BAD+=("$GDS: could not find the space lookup - this guard is measuring nothing") +else + grep -q 'SPACES_OK' <<<"$LOOKUP_BODY" \ + || LOOKUP_BAD+=("$GDS: the spaces list result is not checked, so a failed lookup reads as 'none exists' and creates one") + grep -qE 'Refusing to create' <<<"$LOOKUP_BODY" \ + || LOOKUP_BAD+=("$GDS: nothing refuses to create when the lookup could not answer") + grep -q 'next_page_token' <<<"$LOOKUP_BODY" \ + || LOOKUP_BAD+=("$GDS: the spaces list is not paged, so an existing space beyond page 1 is invisible and a duplicate gets created") + grep -q 'DUPES' <<<"$LOOKUP_BODY" \ + || LOOKUP_BAD+=("$GDS: duplicates are not counted, so which space gets reused is whatever the API returned first") + # the old shape, verbatim: a discarded-stderr list feeding a parser + grep -qE 'genie/spaces[^|]*2>/dev/null[[:space:]]*\|' <<<"$LOOKUP_BODY" \ + && LOOKUP_BAD+=("$GDS: the spaces list still discards stderr into a parser - the failure becomes an empty result") +fi + +if [[ "${#LOOKUP_BAD[@]}" -eq 0 ]]; then + pass "a reuse-or-create lookup refuses to create when it cannot tell, pages, and counts duplicates." +else + for b in "${LOOKUP_BAD[@]}"; do bad "$b"; done +fi + +# --------------------------------------------------------------------------- +# invariant 49: a decision about whether to CALL THE MODEL must not sit behind a persistence +# guard. +# +# chat.ts short-circuits when the user has DENIED an MCP tool call: no denial, no model call. +# That check was nested inside `if (dbAvailable && ...)` and inside its own +# `assistantMessages.length > 0` branch, so in ephemeral mode -- a documented, supported +# configuration where the database is disabled -- a continuation carrying a denial fell through +# to streamText and the model ran against a tool the user had just refused. Raised by a reviewer. +# +# Persistence and permission are unrelated concerns. Only saveMessages belongs behind the +# database check. +CHAT_TS=agent/patches/e2e-chatbot-app-next/server/src/routes/chat.ts +DENIAL_BAD=() +if [[ ! -f "$CHAT_TS" ]]; then + DENIAL_BAD+=("$CHAT_TS is missing - this guard is measuring nothing") +else + # A denied tool part stays in the client history forever, so judging "any denial anywhere" + # means the first denial stalls every later continuation: approving a subsequent tool silently + # drops it. Only the most recent tool interaction can decide this. + if grep -qE 'hasMcpDenial = requestBody\.previousMessages\?\.some' "$CHAT_TS"; then + DENIAL_BAD+=("$CHAT_TS: the denial check scans ALL previous messages, so one denial stalls every later continuation; judge the most recent tool part") + fi + grep -q 'lastToolPart' "$CHAT_TS" \ + || DENIAL_BAD+=("$CHAT_TS: nothing identifies the most recent tool part, so the denial check cannot be scoped to it") + + DENIAL_LINE="$(grep -n 'const hasMcpDenial' "$CHAT_TS" | head -1 | cut -d: -f1)" + DB_GUARD_LINE="$(grep -n 'if (dbAvailable && requestBody.previousMessages)' "$CHAT_TS" | head -1 | cut -d: -f1)" + if [[ -z "$DENIAL_LINE" ]]; then + DENIAL_BAD+=("$CHAT_TS: no hasMcpDenial check - a denied tool call would reach the model") + elif [[ -n "$DB_GUARD_LINE" ]]; then + # Closing brace of the dbAvailable block, found by brace depth from its opening line. + CLOSE_LINE="$(awk -v start="$DB_GUARD_LINE" ' + NR < start { next } + { for (i = 1; i <= length($0); i++) { + c = substr($0, i, 1) + if (c == "{") d++ + else if (c == "}") { d--; if (d == 0) { print NR; exit } } + } }' "$CHAT_TS")" + if [[ -n "$CLOSE_LINE" && "$DENIAL_LINE" -gt "$DB_GUARD_LINE" && "$DENIAL_LINE" -lt "$CLOSE_LINE" ]]; then + DENIAL_BAD+=("$CHAT_TS: the hasMcpDenial check (line $DENIAL_LINE) is inside the dbAvailable block (lines $DB_GUARD_LINE-$CLOSE_LINE), so a denial is ignored in ephemeral mode") + fi + fi +fi + +if [[ "${#DENIAL_BAD[@]}" -eq 0 ]]; then + pass "a denied tool call stops the model call in both persistence modes." +else + for b in "${DENIAL_BAD[@]}"; do bad "$b"; done +fi + +echo +if [[ "$FAILED" -eq 0 ]]; then + echo "All runbook invariants hold." +else + echo "One or more invariants FAILED (see ✗ above)." >&2 +fi +exit "$FAILED" diff --git a/scripts/genie-data-setup.sh b/scripts/genie-data-setup.sh new file mode 100755 index 0000000..191e2d3 --- /dev/null +++ b/scripts/genie-data-setup.sh @@ -0,0 +1,626 @@ +#!/usr/bin/env bash +# genie-data-setup.sh — Phase 6c: make sure Genie has data, and a space, to work with. +# +# WHY THIS EXISTS +# Bootstrap used to finish "successfully" on a fresh workspace while Genie could +# not answer a single data question, because $UC_CATALOG.$UC_SCHEMA was empty. +# Phase 6c detected that and the runbook ended with homework. The user was never +# offered a way to fix it during the run (#83). +# +# This script is invoked identically from BOOTSTRAP.md and scripts/bootstrap.sh, +# for the same reason scripts/lib/runbook.sh exists: one implementation, so the +# runbook and the automated runner cannot drift. +# +# SAFETY RULES (do not relax these) +# * Never clobber a table that already exists. CREATE TABLE IF NOT EXISTS only. +# * Never touch a Genie space this script did not create, even one covering the +# same schema. Create ours alongside it. The sole exception is a space the +# user named explicitly in --space-ids. +# * Never emit GENIE_MCP_MODE=space without a non-empty GENIE_SPACE_ID: +# agent.py raises ValueError on that combination and the app fails to boot. +# * Only switch to space mode after confirming the agent SP can actually run +# the space. A working agent on Genie Agent beats a scoped-but-broken one. +# +# Output: `KEY=value` lines on stdout for the caller to capture. Everything +# human-facing goes to stderr, so `eval "$(genie-data-setup.sh ...)"` is safe. +set -uo pipefail + +# Define the progress helpers BEFORE sourcing the lib, which only defines them if +# they are missing. The lib's versions print to stdout; here stdout is the machine +# contract, so every human-facing line has to go to stderr instead. +note() { echo " $*" >&2; } +ok() { echo " ✓ $*" >&2; } +warn() { echo " ⚠ $*" >&2; } +fail() { echo " ✗ $*" >&2; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/runbook.sh +. "$SCRIPT_DIR/lib/runbook.sh" + +SEED_SOURCE="${FIREFLY_SEED_SOURCE:-samples.wanderbricks}" +# Ownership marker. A space is "ours" only if the title matches this exactly, so a +# re-run reuses it instead of creating a duplicate, and a foreign space covering +# the same schema is never mistaken for ours. +space_title_for() { printf 'Firefly Genie Agent — %s.%s' "$1" "$2"; } + +CATALOG="" SCHEMA="" PROFILE="" WAREHOUSE_ID="" +SEED="yes" SPACE_IDS="" CREATE_SPACE="yes" GRANT_GUEST="no" +# This script is called by two phases. Hardcoding one of their names meant a reader in +# Phase 3f was told about "Phase 6c using warehouse ..." and sent looking for a phase they +# had not reached. The caller says who it is. +PHASE_LABEL="6c" +GUEST_SP="" AGENT_SP="" + +usage() { + cat >&2 <<'USAGE' +usage: genie-data-setup.sh --catalog C --schema S --profile P [options] + + --catalog C UC catalog holding the agent's data + --schema S UC schema within that catalog + --profile P Databricks CLI profile + --warehouse-id W SQL warehouse (default: first available) + --phase LABEL Phase name to use in messages (default 6c), so a reader is not + pointed at a phase they are not in. + --seed yes|no|skip Seed sample data when the schema is empty (default yes). "skip" + means seeding is not this call's concern, as distinct from "no", + which is a decision not to seed and is reported as one. + --space-ids a,b Use these existing Genie space ids instead of creating one + --create-space y|n Create a space when --space-ids is empty (default yes) + --defer-grants The SPs do not exist yet, so make no grants and say where + they happen. Distinct from --grant-guest no, which is a + decision to withhold access. + --grant-guest y|n Grant the guest SP CAN_RUN on the space(s) + SELECT on + the tables they reference (default no) + --guest-sp ID Guest service principal application (client) id + --agent-sp ID Agent app service principal application (client) id + +Emits: SEED_STATUS, SEED_TABLE_COUNT, GENIE_SPACE_ID, GENIE_SPACE_IDS, + GENIE_SPACE_SOURCE, GENIE_MCP_MODE +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --catalog) CATALOG="${2:-}"; shift 2 ;; + --schema) SCHEMA="${2:-}"; shift 2 ;; + --profile) PROFILE="${2:-}"; shift 2 ;; + --warehouse-id) WAREHOUSE_ID="${2:-}"; shift 2 ;; + --phase) PHASE_LABEL="${2:-6c}"; shift 2 ;; + --seed) SEED="${2:-yes}"; shift 2 ;; + --space-ids) SPACE_IDS="${2:-}"; shift 2 ;; + --create-space) CREATE_SPACE="${2:-yes}"; shift 2 ;; + --defer-grants) DEFER_GRANTS=yes; shift ;; + --grant-guest) GRANT_GUEST="${2:-no}"; shift 2 ;; + --guest-sp) GUEST_SP="${2:-}"; shift 2 ;; + --agent-sp) AGENT_SP="${2:-}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown flag: $1" >&2; usage; exit 2 ;; + esac +done + +[ -n "$CATALOG" ] && [ -n "$SCHEMA" ] && [ -n "$PROFILE" ] || { usage; exit 2; } + +# Normalize the yes/no flags once; "None"/"none" is how Phase 0 spells "unset". +yn() { case "$(printf '%s' "${1:-}" | tr 'A-Z' 'a-z')" in y|yes|true|1) echo yes ;; *) echo no ;; esac; } +[ "$SEED" = "skip" ] || SEED="$(yn "$SEED")"; CREATE_SPACE="$(yn "$CREATE_SPACE")"; GRANT_GUEST="$(yn "$GRANT_GUEST")" +case "$(printf '%s' "$SPACE_IDS" | tr 'A-Z' 'a-z')" in none|null|"") SPACE_IDS="" ;; esac + +dbx() { databricks "$@" --profile "$PROFILE"; } + +# ─── warehouse ──────────────────────────────────────────────────────────────── +if [ -z "$WAREHOUSE_ID" ]; then + WAREHOUSE_ID="$(dbx warehouses list -o json 2>/dev/null | python3 -c 'import sys,json +try: ws = json.load(sys.stdin) or [] +except Exception: ws = [] +ws = ws.get("warehouses", ws) if isinstance(ws, dict) else ws +# Prefer a warehouse that is already RUNNING: a cold start adds minutes to Phase 6c. +for want in ("RUNNING", None): + for w in ws: + if want is None or w.get("state") == want: + print(w.get("id","")); raise SystemExit +print("")' 2>/dev/null)" +fi +[ -n "$WAREHOUSE_ID" ] || { fail "no SQL warehouse available — Genie cannot run queries without one"; exit 1; } +note "Phase $PHASE_LABEL using warehouse $WAREHOUSE_ID" + +sql() { firefly_sql "$WAREHOUSE_ID" "$1" "$PROFILE"; } +sql_scalar() { firefly_sql_scalar "$WAREHOUSE_ID" "$1" "$PROFILE"; } + +# ─── 1. seed ────────────────────────────────────────────────────────────────── +# The decision to seed is gated on the schema being empty (or holding nothing but +# our own seed tables — that is how a half-finished seed self-heals on a re-run). +# A schema with any foreign table is the user's data: report and leave it alone. +existing_tables() { sql "SHOW TABLES IN \`$CATALOG\`.\`$SCHEMA\`" 2>/dev/null | cut -f2 | sed '/^$/d'; } + +SEED_STATUS="skipped" +SRC_CATALOG="${SEED_SOURCE%%.*}"; SRC_SCHEMA="${SEED_SOURCE#*.}" + +# Newline-delimited strings, not arrays: `mapfile` is bash 4+ and the target is +# stock macOS bash 3.2 (SHELL-PORTABILITY). +EXISTING="$(existing_tables)" +EXISTING_COUNT="$(printf '%s\n' "$EXISTING" | sed '/^$/d' | wc -l | tr -d ' ')" + +if [ "$SEED" != "yes" ]; then + if [ "$SEED" = "skip" ]; then + # The grants-only call passes --seed skip. It previously passed --seed no, and the run + # then told the operator that seeding was "declined at Phase 0" when Phase 0 had in fact + # answered yes and Phase 3f had already seeded 16 tables. Same defect as the grant + # deferral above: a flag meaning "not here" was read back as a decision. + SEED_STATUS="not-this-phase" + else + SEED_STATUS="declined" + note "seeding declined at Phase 0 — leaving ${CATALOG}.${SCHEMA} as-is" + fi +else + # Probe the source, keeping stderr. The previous version discarded it with + # 2>/dev/null and reported one message — "not readable from this workspace" — + # for four different causes: the schema not existing yet, existing but empty, a + # real permission denial, and a warehouse/SQL error. That reads as a permanent + # entitlement problem when it is usually neither permanent nor permissions. + # + # On a BRAND-NEW workspace the real cause is a race: `samples` is provisioned + # asynchronously and Phase 6c can run before it lands. Observed on a fresh + # workspace — Phase 4 finished 12:04:46Z, Phase 6c probed ~12:05:00Z, and the + # samples.wanderbricks schema was not created until 12:05:39Z. A re-run minutes + # later seeded all 16 tables. So wait for it rather than declaring defeat 40 + # seconds early. + SRC_TABLES="" + SRC_ERR="" + _src_out="$(mktemp)"; _src_err="$(mktemp)" + _deadline=$(( $(date +%s) + ${FIREFLY_SEED_SOURCE_WAIT:-180} )) + while :; do + : > "$_src_out"; : > "$_src_err" + sql "SHOW TABLES IN \`$SRC_CATALOG\`.\`$SRC_SCHEMA\`" >"$_src_out" 2>"$_src_err" + SRC_TABLES="$(cut -f2 "$_src_out" 2>/dev/null | sed '/^$/d')" + SRC_ERR="$(tr '\n' ' ' < "$_src_err")" + [ -n "$SRC_TABLES" ] && break + # A denial will not resolve by waiting; stop immediately and say so. + case "$SRC_ERR" in + *PERMISSION_DENIED*|*[Pp]ermission*|*[Ff]orbidden*|*UNAUTHORIZED*) break ;; + esac + [ "$(date +%s)" -ge "$_deadline" ] && break + note "$SEED_SOURCE not populated yet — waiting 15s (a fresh workspace provisions it asynchronously)" + sleep 15 + done + rm -f "$_src_out" "$_src_err" + + if [ -z "$SRC_TABLES" ]; then + # Say WHICH of the four it was, and show the server's own words. + case "$SRC_ERR" in + *PERMISSION_DENIED*|*[Pp]ermission*|*[Ff]orbidden*|*UNAUTHORIZED*) + SEED_STATUS="source-denied" + warn "no permission to read $SEED_SOURCE — grant SELECT on it, or set SEED_SAMPLE_DATA=no" ;; + *SCHEMA_NOT_FOUND*|*does\ not\ exist*|*NOT_FOUND*|*[Nn]ot\ [Ff]ound*) + SEED_STATUS="source-not-ready" + warn "$SEED_SOURCE does not exist yet after ${FIREFLY_SEED_SOURCE_WAIT:-180}s." + note "On a new workspace the samples catalog is provisioned asynchronously." + note "Re-run this phase in a few minutes; the data is not missing, just late:" + note " databricks tables list $SRC_CATALOG $SRC_SCHEMA --profile $PROFILE" ;; + "") + SEED_STATUS="source-empty" + warn "$SEED_SOURCE exists but reported no tables after ${FIREFLY_SEED_SOURCE_WAIT:-180}s." + note "Re-run this phase once it is populated." ;; + *) + SEED_STATUS="source-error" + warn "could not read $SEED_SOURCE: $SRC_ERR" ;; + esac + else + # Is every existing table one of ours? Then a partial seed can be completed. + foreign=0 + for t in $EXISTING; do + printf '%s\n' "$SRC_TABLES" | grep -qxF "$t" || { foreign=1; break; } + done + # Which source tables are still missing? Distinguishing "nothing to do" from + # "seeded 16" matters: a re-run that reports `seeded` looks like it rewrote + # the schema when it did nothing at all. + MISSING="" + for t in $SRC_TABLES; do + printf '%s\n' "$EXISTING" | grep -qxF "$t" || MISSING="${MISSING}${MISSING:+ }$t" + done + + if [ "$EXISTING_COUNT" -gt 0 ] && [ "$foreign" = "1" ]; then + SEED_STATUS="already-populated" + note "${CATALOG}.${SCHEMA} already holds tables that are not ours — not seeding" + elif [ -z "$MISSING" ]; then + SEED_STATUS="already-seeded" + ok "${CATALOG}.${SCHEMA} already holds all $EXISTING_COUNT seed table(s) — nothing to do" + else + [ "$EXISTING_COUNT" -gt 0 ] && note "completing a partial seed ($EXISTING_COUNT present, $(set -- $MISSING; echo $#) missing)" + made=0 failed=0 + for t in $MISSING; do + # IF NOT EXISTS is what makes this non-destructive and re-runnable. + if sql "CREATE TABLE IF NOT EXISTS \`$CATALOG\`.\`$SCHEMA\`.\`$t\` AS SELECT * FROM \`$SRC_CATALOG\`.\`$SRC_SCHEMA\`.\`$t\`" >/dev/null 2>&1; then + made=$((made + 1)) + else + failed=$((failed + 1)); warn "could not seed $t" + fi + done + SEED_STATUS="seeded" + [ "$failed" -gt 0 ] && SEED_STATUS="seeded-partial" + ok "seeded $made table(s) from $SEED_SOURCE into ${CATALOG}.${SCHEMA}" + fi + fi +fi + +SEED_TABLE_COUNT="$(existing_tables | wc -l | tr -d ' ')" + +# ─── 2. Genie space ─────────────────────────────────────────────────────────── +# Table identifiers the space will reference. Sorted, because the API stores +# data_sources.tables sorted by identifier and an unsorted payload round-trips +# differently than it was sent. +space_tables() { existing_tables | sed "s/^/${CATALOG}.${SCHEMA}./" | LC_ALL=C sort; } + +get_space() { dbx api get "/api/2.0/genie/spaces/$1?include_serialized_space=true" 2>/dev/null; } + +# A space is readable, or we retried long enough to mean it. A single GET here abandoned +# a space this script had just created, round-tripped 16 tables through, and granted +# CAN_RUN on: one transient failure printed "not readable - staying on workspace-wide Genie", +# cleared GENIE_SPACE_ID, and shipped the app on Genie Agent. The same GET succeeded +# minutes later. Reads after a create are eventually consistent, so treat one failure as +# "not yet" rather than "not there", and keep stderr so a real error can be named -- +# the discarded-stderr version could only ever say "not readable", whatever went wrong. +space_readable() { # space_readable [budget_s] + local sid="$1" budget="${2:-90}" waited=0 raw err="" + while :; do + raw="$(dbx api get "/api/2.0/genie/spaces/$sid?include_serialized_space=true" 2>&1 || true)" + case "$raw" in + *'"space_id"'*) return 0 ;; + esac + err="$(printf '%s' "$raw" | tr '\n' ' ' | cut -c1-160)" + [ "$waited" -ge "$budget" ] && break + [ "$waited" -eq 0 ] && note "space $sid not readable yet; retrying for up to ${budget}s" + sleep 10 + waited=$((waited + 10)) + done + warn "space $sid still unreadable after ${budget}s. The API said:" + warn " ${err:-(no output)}" + return 1 +} + +GENIE_SPACE_ID="" GENIE_SPACE_SOURCE="none" GENIE_MCP_MODE="one" +RESOLVED_IDS="" + +if [ -n "$SPACE_IDS" ]; then + # User-supplied ids: verify each is real and readable, then use them verbatim. + # These are the only spaces this script is permitted to touch. + for sid in $(printf '%s' "$SPACE_IDS" | tr ', ' '\n' | sed '/^$/d'); do + if space_readable "$sid" 30; then + RESOLVED_IDS="${RESOLVED_IDS}${RESOLVED_IDS:+ }$sid" + else + warn "skipping Genie space $sid — see the API message above" + fi + done + if [ -n "$RESOLVED_IDS" ]; then + GENIE_SPACE_ID="${RESOLVED_IDS%% *}" + GENIE_SPACE_SOURCE="user-supplied" + ok "using user-supplied Genie space(s): $RESOLVED_IDS" + else + warn "none of the supplied space ids were usable — falling back to workspace-wide Genie" + fi +elif [ "$CREATE_SPACE" = "yes" ]; then + WANT_TITLE="$(space_title_for "$CATALOG" "$SCHEMA")" + # Ours = exact title match AND covers this schema. Anything else is off-limits. + # + # Three defects lived in the previous version of this lookup and they compounded: + # + # 1. The list call sent stderr to /dev/null and its parser returned {} on any failure, so + # "the lookup failed" and "no such space exists" became the same value -- and the else + # branch on that value CREATES. One hiccup in one API call silently produced an extra + # space, with the evidence discarded by the same line that caused it. + # 2. It took the first title match and broke, so once two spaces shared a title, which one + # got reused was whatever order the API returned, and duplicates could not be detected. + # 3. It read one page and never followed next_page_token, so correctness depended on a + # space count nobody was watching. + # + # Now the call is checked, an inconclusive lookup REFUSES to create, every page is read, and + # all matches are counted so duplicates are stated rather than silently arbitrated. + SPACES_ERR="$(mktemp -t ffspaces)" + SPACES_ALL="$(mktemp -t ffspacesj)" + SPACES_OK=1 + : > "$SPACES_ALL" + _page_token="" + while : ; do + _url="/api/2.0/genie/spaces?page_size=100" + [ -n "$_page_token" ] && _url="$_url&page_token=$_page_token" + # COMPACTED to one line per page before appending. `databricks api get` pretty-prints, and + # the reader below parses line by line, so appending raw output gave it a stream of + # fragments -- every json.loads failed, every page counted zero, and the lookup reported + # "Genie spaces visible: 0" on a workspace holding several. I introduced that when adding + # pagination: the version before it piped the whole response into json.load, which does not + # care about newlines. The symptom looked like an API returning nothing, which is why I + # spent two commits explaining an empty list that was never empty. + if ! dbx api get "$_url" 2>"$SPACES_ERR" \ + | python3 -c 'import sys,json +try: sys.stdout.write(json.dumps(json.load(sys.stdin)) + "\n") +except Exception: sys.exit(3)' >>"$SPACES_ALL"; then + SPACES_OK=0 + break + fi + _page_token="$(python3 -c 'import sys,json +try: d = json.loads(open(sys.argv[1]).read().strip().split("\n")[-1]) or {} +except Exception: d = {} +print(d.get("next_page_token") or "")' "$SPACES_ALL" 2>/dev/null)" + [ -n "$_page_token" ] || break + done + + MINE="" + DUPES=0 + if [ "$SPACES_OK" = "1" ]; then + # PREFIX match, not equality. The Genie API appends a creation timestamp to the title, so a + # space created as "Firefly Genie Agent — cat.default" comes back as + # "Firefly Genie Agent — cat.default 2026-07-29 18:15:23". Equality therefore never matches + # anything this script created, every run concluded no space existed, and one workspace + # accumulated five of them. space_title_for adds no timestamp; the server does. + _found="$(FF_TITLE="$WANT_TITLE" python3 -c 'import sys,json,os,re +want = os.environ["FF_TITLE"] +# The server-added suffix is a timestamp, so accept exactly that and nothing else -- a bare +# prefix test would also swallow a DIFFERENT schema whose name extends ours. +suffix = re.compile(r"^\s*\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}") +def ours(title): + if title == want: + return True + if not title.startswith(want): + return False + return bool(suffix.match(title[len(want):])) +ids, seen = [], 0 +for line in open(sys.argv[1]): + line = line.strip() + if not line: + continue + try: d = json.loads(line) or {} + except Exception: continue + for s in d.get("spaces") or []: + seen += 1 + if ours(s.get("title") or ""): + ids.append(s.get("space_id") or "") +ids = sorted(i for i in ids if i) +print(len(ids)) +print(ids[0] if ids else "") +print(seen)' "$SPACES_ALL" 2>/dev/null)" + DUPES="$(printf '%s' "$_found" | sed -n 1p)" + MINE="$(printf '%s' "$_found" | sed -n 2p)" + _seen="$(printf '%s' "$_found" | sed -n 3p)" + note "Genie spaces visible: ${_seen:-0}; titled for this schema: ${DUPES:-0}" + else + warn "could not list Genie spaces, so whether one already exists is UNKNOWN:" + warn " $(head -1 "$SPACES_ERR" 2>/dev/null)" + warn " Refusing to create one - creating on an unknown state is how duplicates appear." + warn " Re-run Phase 3f once the API answers, or pass GENIE_SPACE_IDS explicitly." + fi + rm -f "$SPACES_ERR" "$SPACES_ALL" + + if [ "${DUPES:-0}" -gt 1 ] 2>/dev/null; then + warn "$DUPES Genie spaces share the title: $WANT_TITLE" + warn " Reusing $MINE (lowest id, so every run agrees). Delete the extras when convenient;" + warn " a duplicate means an earlier run created one while this lookup could not see it." + fi + + # An EMPTY but SUCCESSFUL list is the case the exit-status check misses, and it is the one that + # actually happens: a run on a workspace holding several spaces printed "Genie spaces visible: + # 0" and created another. Zero-with-exit-0 is indistinguishable from a genuinely empty + # workspace unless you already know a space exists -- and bootstrap does, because it wrote the + # id down. If the list cannot see a space we recorded, the LIST is wrong, not the record. + PRIOR_SPACE_ID="$(read_secret GENIE_SPACE_ID 2>/dev/null || true)" + case "$(printf '%s' "$PRIOR_SPACE_ID" | tr 'A-Z' 'a-z')" in none|null) PRIOR_SPACE_ID="" ;; esac + if [ "$SPACES_OK" = "1" ] && [ -z "$MINE" ] && [ -n "$PRIOR_SPACE_ID" ]; then + if space_readable "$PRIOR_SPACE_ID"; then + warn "the spaces list did not include $PRIOR_SPACE_ID, but that space is readable." + warn " The list is inconsistent, so this is NOT an empty workspace. Reusing the recorded" + warn " space instead of creating another - an empty list is how duplicates accumulate." + MINE="$PRIOR_SPACE_ID" + else + note "a space id was recorded ($PRIOR_SPACE_ID) but it is no longer readable; it was" + note " probably deleted, so creating a replacement is correct." + fi + fi + + if [ "$SPACES_OK" != "1" ]; then + GENIE_SPACE_SOURCE="lookup-failed" + elif [ -n "$MINE" ]; then + GENIE_SPACE_ID="$MINE"; GENIE_SPACE_SOURCE="reused-ours" + ok "reusing the Genie space this bootstrap created earlier ($MINE)" + else + TABLES="$(space_tables)" + if [ -z "$TABLES" ]; then + warn "no tables in ${CATALOG}.${SCHEMA} — not creating an empty Genie space" + else + REQ="$(mktemp -t ffgenie)" + FF_TABLES="$TABLES" FF_TITLE="$WANT_TITLE" FF_WH="$WAREHOUSE_ID" \ + FF_DESC="Firefly agent data in ${CATALOG}.${SCHEMA}. Created by BOOTSTRAP.md Phase 6c." \ + python3 -c 'import json, os, sys +tables = [t for t in os.environ["FF_TABLES"].split("\n") if t.strip()] +space = { + "version": 2, + "data_sources": {"tables": [{"identifier": t} for t in sorted(tables)]}, + "instructions": {}, +} +sys.stdout.write(json.dumps({ + "title": os.environ["FF_TITLE"], + "description": os.environ["FF_DESC"], + "warehouse_id": os.environ["FF_WH"], + "serialized_space": json.dumps(space), +}))' > "$REQ" + NEW="$(dbx api post /api/2.0/genie/spaces --json "@$REQ" 2>&1)" + rm -f "$REQ" + GENIE_SPACE_ID="$(printf '%s' "$NEW" | python3 -c 'import sys,json +try: print((json.load(sys.stdin) or {}).get("space_id") or "") +except Exception: print("")' 2>/dev/null)" + if [ -n "$GENIE_SPACE_ID" ]; then + GENIE_SPACE_SOURCE="created" + ok "created Genie space $GENIE_SPACE_ID over $(printf '%s\n' "$TABLES" | sed '/^$/d' | wc -l | tr -d ' ') table(s)" + # Round-trip verify: the stored space must hold the tables we sent. + got="$(get_space "$GENIE_SPACE_ID" | python3 -c 'import sys,json +try: d = json.load(sys.stdin) or {} +except Exception: raise SystemExit +ss = d.get("serialized_space") or "{}" +ss = json.loads(ss) if isinstance(ss, str) else ss +print(len((ss.get("data_sources") or {}).get("tables") or []))' 2>/dev/null)" + note "round-trip: space reports ${got:-0} table(s)" + else + warn "Genie space creation failed — staying on workspace-wide Genie" + printf '%s\n' "$NEW" | head -3 >&2 + fi + fi + fi +else + note "Genie space creation declined at Phase 0 — staying on workspace-wide Genie" +fi + +[ -n "$RESOLVED_IDS" ] || RESOLVED_IDS="$GENIE_SPACE_ID" + +# ─── 3. grants ──────────────────────────────────────────────────────────────── +# CAN_RUN is "view and ask questions" — the level an SP needs to serve the agent. +grant_space_run() { # grant_space_run + local sid="$1" sp="$2" req + [ -n "$sid" ] && [ -n "$sp" ] || return 0 + req="$(mktemp -t ffperm)" + FF_SP="$sp" python3 -c 'import json,os,sys +sys.stdout.write(json.dumps({"access_control_list":[ + {"service_principal_name": os.environ["FF_SP"], "permission_level": "CAN_RUN"}]}))' > "$req" + # PATCH adds to the ACL; PUT would replace it and drop the owner's CAN_MANAGE. + if dbx api patch "/api/2.0/permissions/genie/$sid" --json "@$req" >/dev/null 2>&1; then + ok "granted CAN_RUN on space $sid to $sp" + else + warn "could not grant CAN_RUN on space $sid to $sp" + fi + rm -f "$req" +} + +# The tables a space actually references — the set that needs SELECT. +space_table_identifiers() { # space_table_identifiers + get_space "$1" | python3 -c 'import sys,json +try: d = json.load(sys.stdin) or {} +except Exception: raise SystemExit +ss = d.get("serialized_space") or "{}" +ss = json.loads(ss) if isinstance(ss, str) else ss +for t in (ss.get("data_sources") or {}).get("tables") or []: + ident = t.get("identifier") + if ident: print(ident)' 2>/dev/null +} + +grant_table_select() { # grant_table_select + local sp="$1"; shift + [ -n "$sp" ] || return 0 + local sid ident cat sch seen="" n=0 failed=0 first_err="" _gt_err + local _covered _present _missing + for sid in "$@"; do + [ -n "$sid" ] || continue + for ident in $(space_table_identifiers "$sid"); do + cat="${ident%%.*}"; sch="${ident#*.}"; sch="${sch%%.*}" + # USE CATALOG / USE SCHEMA once per namespace, then SELECT per table. + case " $seen " in + *" $cat.$sch "*) ;; + *) + sql "GRANT USE CATALOG ON CATALOG \`$cat\` TO \`$sp\`" >/dev/null 2>&1 + sql "GRANT USE SCHEMA ON SCHEMA \`$cat\`.\`$sch\` TO \`$sp\`" >/dev/null 2>&1 + seen="$seen $cat.$sch" + ;; + esac + _gt_err="$(sql "GRANT SELECT ON TABLE $ident TO \`$sp\`" 2>&1 >/dev/null)" \ + && n=$((n + 1)) \ + || { failed=$((failed + 1)) + [ -z "$first_err" ] && first_err="$ident: $(printf '%s' "$_gt_err" | head -1)"; } + done + done + [ "$n" -gt 0 ] && ok "granted SELECT on $n table(s) to $sp" + # Failures used to go to /dev/null and only the success count was printed, so a table the + # guest cannot read was indistinguishable from one it can. Say so. + if [ "$failed" -gt 0 ]; then + warn "$failed SELECT grant(s) FAILED for $sp - those tables will refuse guest queries" + warn " first failure: $first_err" + fi + + # A count that disagrees with what was seeded is the reader's problem to explain unless we + # explain it. A run seeded 16 tables and this reported 15 with no reason given; whether the + # 16th belongs in the space is a real question, and it cannot be asked if nobody says which + # table is absent. Compare the space's coverage against the schema and name the difference. + _covered="$(for sid in "$@"; do [ -n "$sid" ] && space_table_identifiers "$sid"; done \ + | sed -E 's/.*\.//' | sort -u)" + _present="$(existing_tables | sort -u)" + _missing="$(comm -23 <(printf '%s\n' "$_present") <(printf '%s\n' "$_covered") 2>/dev/null)" + if [ -n "$_missing" ]; then + warn "these tables exist in ${CATALOG}.${SCHEMA} but are NOT in the Genie space, so" + warn " Genie cannot query them and they get no grant:" + printf '%s\n' "$_missing" | sed 's/^/ - /' >&2 + fi +} + +for sid in $RESOLVED_IDS; do + [ -n "$AGENT_SP" ] && grant_space_run "$sid" "$AGENT_SP" +done + +# The Phase 0 answer decides this, and it is honoured exactly. +# +# An earlier fix here forced the grant whenever bootstrap had created the space itself, +# on the reasoning that guests cannot use the space otherwise. That reasoning is right +# and the implementation was wrong: Phase 0 presents GRANT_GUEST_SPACE_ACCESS as a +# controlling input, so overriding it made the runbook do something other than what it +# told the operator -- the same defect as any misleading message, just with permissions. +# Two runs reported it independently. The ask now defaults to yes and is put for every +# path, so the guest flow works by default without anyone's answer being discarded. +# +# Saying no is legitimate, and its cost is stated rather than left to be discovered. +# "Not granting now" and "decided not to grant" are different facts, and collapsing them +# produced exactly the misleading message this comment block warns about. Phase 3f creates +# the space before any SP exists, so it cannot grant and deliberately passes no --grant-guest; +# a missing flag defaulted to no, and the run told the operator their yes meant the guest SP +# would get NO access -- while Phase 6c went on to grant it. The reader is left believing +# their Phase 0 answer was discarded. Deferral gets its own state and names where it lands. +if [ "${DEFER_GRANTS:-no}" = "yes" ]; then + case "$GENIE_SPACE_SOURCE" in + created|reused-ours) + note "no grants here: the agent and guest service principals do not exist yet." + note " Phase 6c grants CAN_RUN on space $GENIE_SPACE_ID and SELECT on its tables," + note " honouring GRANT_GUEST_SPACE_ACCESS." ;; + esac +elif [ "$GRANT_GUEST" != "yes" ]; then + case "$GENIE_SPACE_SOURCE" in + created|reused-ours) + warn "GRANT_GUEST_SPACE_ACCESS=no, so the guest SP gets NO access to space" + warn " $GENIE_SPACE_ID. Guest users will reach the app and be unable to ask" + warn " data questions. Re-run Phase 6c with GRANT_GUEST_SPACE_ACCESS=yes to" + warn " grant CAN_RUN on the space and SELECT on its tables." ;; + esac +fi + +if [ "${DEFER_GRANTS:-no}" = "yes" ]; then + : +elif [ "$GRANT_GUEST" = "yes" ] && [ -n "$GUEST_SP" ]; then + for sid in $RESOLVED_IDS; do grant_space_run "$sid" "$GUEST_SP"; done + # shellcheck disable=SC2086 + grant_table_select "$GUEST_SP" $RESOLVED_IDS +elif [ "$GRANT_GUEST" = "yes" ]; then + warn "no --guest-sp given, so the guest SP has NO access to the Genie space." + warn "Guest users will not be able to ask data questions. Re-run Phase 6c after" + warn "Phase 6b has created the guest SP, passing --guest-sp \$GUEST_SP_CLIENT_ID." +fi + +# ─── 4. decide the app's Genie mode ─────────────────────────────────────────── +# Only claim space mode if the space is really there. agent.py raises ValueError +# when GENIE_MCP_MODE=space and GENIE_SPACE_ID is empty, which fails the app boot. +if [ -n "$GENIE_SPACE_ID" ]; then + if space_readable "$GENIE_SPACE_ID" 90; then + GENIE_MCP_MODE="space" + else + # Falling back is a real cost: the app loses the curated space and lands on + # Genie Agent, which guest users cannot use. Say what was given up and why, so + # this is never mistaken for "no space was wanted". + warn "abandoning space $GENIE_SPACE_ID (source: $GENIE_SPACE_SOURCE) — staying on workspace-wide Genie" + case "$GENIE_SPACE_SOURCE" in + created|reused-ours) + warn " this space WAS created and granted successfully in this run, so an" + warn " unreadable GET is more likely a transient API failure than a missing" + warn " space. Re-run Phase 6c: it reuses the space by title instead of" + warn " creating a second one, and the app then redeploys in space mode." ;; + esac + GENIE_SPACE_ID=""; GENIE_MCP_MODE="one" + fi +fi +[ "$GENIE_MCP_MODE" = "space" ] || GENIE_SPACE_ID="" + +printf 'SEED_STATUS=%s\n' "$SEED_STATUS" +printf 'SEED_TABLE_COUNT=%s\n' "$SEED_TABLE_COUNT" +printf 'GENIE_SPACE_ID=%s\n' "$GENIE_SPACE_ID" +printf 'GENIE_SPACE_IDS=%s\n' "$RESOLVED_IDS" +printf 'GENIE_SPACE_SOURCE=%s\n' "$GENIE_SPACE_SOURCE" +printf 'GENIE_MCP_MODE=%s\n' "$GENIE_MCP_MODE" diff --git a/scripts/lib/corp-network.sh b/scripts/lib/corp-network.sh new file mode 100644 index 0000000..c329920 --- /dev/null +++ b/scripts/lib/corp-network.sh @@ -0,0 +1,411 @@ +# shellcheck shell=bash +# corp-network.sh — corporate-network handling shared by scripts/bootstrap.sh and the +# Phase 0 block in BOOTSTRAP.md. +# +# source scripts/lib/corp-network.sh +# firefly_bridge_corp_network +# +# WHY THIS FILE EXISTS (ENV-0) +# The TLS trust and registry bridges used to live only in bootstrap.sh, while BOOTSTRAP.md +# merely *described* them in prose. Phase 0 of the runbook contained no runnable commands +# at all, so anyone following it top-to-bottom — which is exactly what the README's +# "Open in Cursor" badge instructs — reached the first install step with no bridge set and +# hit a blocked public registry. Keeping one implementation, sourced by both, is what stops +# the runbook and the runner from drifting apart again. +# +# Safe to source from an interactive shell: nothing here sets -e, and every bridge is a +# no-op when the corresponding public registry is already reachable. Never hardcode a +# mirror — every value is read from the user's own configuration. + +# ─── minimal helpers (defined only if the caller hasn't already) ────────────── +# Must work under BOTH bash (bootstrap.sh) and zsh (a user sourcing this from their shell, +# per BOOTSTRAP.md Phase 0a — zsh is the macOS default). `declare -F name` is NOT a +# function test in zsh: it declares a float and returns 0, so a `declare -F x || x() {…}` +# guard silently skips the definition and every call dies with "command not found". +_ff_is_func() { + if [ -n "${ZSH_VERSION:-}" ]; then + eval '(( ${+functions[$1]} ))' + else + declare -F "$1" >/dev/null 2>&1 + fi +} + +_ff_is_func note || note() { echo " $*"; } +_ff_is_func ok || ok() { echo " ✓ $*"; } +_ff_is_func warn || warn() { echo " ⚠ $*"; } +_ff_is_func fail || fail() { echo " ✗ $*"; } + +: "${INPUTS_DIR:=$HOME/.firefly-bootstrap}" +# Re-apply the default INSIDE the function. The `: "${INPUTS_DIR:=...}"` above +# runs once, at source time; INPUTS_DIR is read again whenever this is called, so +# a caller that exports it empty in between sends the CA bundle to +# /proxy-ca-bundle.pem — an unwritable root path that then breaks curl, the +# GitHub installs, and uv's CPython download with UnknownIssuer. Five E2E runs +# hit it and each one had to recover by hand with TLS_PEM_PATH. +_ff_is_func init_inputs_dir || \ + init_inputs_dir() { + [ -n "${INPUTS_DIR:-}" ] || INPUTS_DIR="$HOME/.firefly-bootstrap" + mkdir -p "$INPUTS_DIR"; chmod 700 "$INPUTS_DIR" + } + +# Single source of truth for the pnpm version this project installs. Must match the +# `packageManager` field in package.json. +: "${PNPM_VERSION:=10.34.5}" + +# Silence pnpm's/npm's update notifier. It advertises "Update available! 10.34.5 -> +# 12.0.0-alpha.16" — the EXACT alpha this pin exists to avoid, because it ignores +# onlyBuiltDependencies and fails with ERR_PNPM_IGNORED_BUILDS. A tool telling the +# reader to upgrade to the one version that breaks the build is worse than noise: +# following its advice is the failure. Both pnpm and npm honour this. +export NPM_CONFIG_UPDATE_NOTIFIER=false + +# ─── TLS trust for intercepting proxies ────────────────────────────────────── +# Python (quickstart, Databricks SDK), Node, curl and uv each consult a different trust +# store, so an intercepting proxy has to be bridged into all of them. + +apply_tls_bundle() { # $1 = pem path — export the trust vars every toolchain reads + # CURL_CA_BUNDLE is required for the Phase 9 smoke-test curls: curl uses the system + # store (/etc/ssl/cert.pem) by default and rejects the proxy's cert (000) otherwise. + export TLS_PEM_PATH="$1" REQUESTS_CA_BUNDLE="$1" SSL_CERT_FILE="$1" \ + NODE_EXTRA_CA_CERTS="$1" CURL_CA_BUNDLE="$1" +} + +# On success sets PROXY_CA_BUNDLE / PROXY_CA_ADDED / PROXY_CA_ROOT_CN / PROXY_CA_ROOT_FP. +# Returns non-zero if TLS to PyPI already validates (no MITM) or no CA could be derived. +derive_proxy_ca_bundle() { + local probe="pypi.org" + # If the system store already validates PyPI, there's no untrusted MITM to handle. + curl -sSf -o /dev/null --max-time 12 "https://${probe}/simple/" 2>/dev/null && return 1 + local chain tmpd sysbundle capem f added=0 lastca="" + chain=$(printf '' | openssl s_client -connect "${probe}:443" -showcerts 2>/dev/null) || return 1 + tmpd=$(mktemp -d) || return 1 + awk -v d="$tmpd" '/-----BEGIN CERTIFICATE-----/{n++} n>0{print > (d"/c-" n ".pem")}' <<<"$chain" + sysbundle=$(python3 -c 'import certifi;print(certifi.where())' 2>/dev/null || echo /etc/ssl/cert.pem) + [[ -f "$sysbundle" ]] || sysbundle=/etc/ssl/cert.pem + init_inputs_dir + capem="$INPUTS_DIR/proxy-ca-bundle.pem" + cat "$sysbundle" > "$capem" + # c-1 is the server leaf; every cert after it is a CA in the presented chain. + for f in "$tmpd"/c-*.pem; do + [[ "$f" == "$tmpd/c-1.pem" ]] && continue + openssl x509 -in "$f" -noout >/dev/null 2>&1 || continue + cat "$f" >> "$capem"; added=$((added+1)); lastca="$f" + done + if [[ "$added" -eq 0 || -z "$lastca" ]]; then rm -f "$capem"; rm -rf "$tmpd"; return 1; fi + PROXY_CA_ROOT_CN=$(openssl x509 -in "$lastca" -noout -subject 2>/dev/null || true | sed -E 's/.*CN ?= ?//; s#.*/CN=##') + PROXY_CA_ROOT_FP=$(openssl x509 -in "$lastca" -noout -fingerprint -sha256 2>/dev/null || true | sed 's/.*=//') + rm -rf "$tmpd" + chmod 600 "$capem" 2>/dev/null || true + PROXY_CA_BUNDLE="$capem"; PROXY_CA_ADDED="$added" + return 0 +} + +# ─── uv ← pip index bridge ─────────────────────────────────────────────────── +# uv does NOT read pip.conf or PIP_INDEX_URL (verified). On corporate networks that block +# public PyPI and route pip through an internal mirror (Artifactory/Nexus/…), uv silently +# hits pypi.org and fails — even though the user's `pip` works. Bridge the user's OWN +# already-configured pip index into uv via UV_DEFAULT_INDEX. +detect_pip_index() { + local v f + if command -v python3 >/dev/null 2>&1; then + v=$(python3 -m pip config get global.index-url 2>/dev/null | tr -d '[:space:]' || true) + [[ -n "$v" && "$v" != "None" ]] && { echo "$v"; return; } + fi + for f in "${PIP_CONFIG_FILE:-}" "$HOME/.config/pip/pip.conf" "$HOME/.pip/pip.conf" /etc/pip.conf; do + [[ -f "$f" ]] || continue + v=$(awk -F= '/^[[:space:]]*index-url[[:space:]]*=/{gsub(/[[:space:]]/,"",$2);print $2; exit}' "$f") + [[ -n "$v" ]] && { echo "$v"; return; } + done +} + +bridge_pip_index_to_uv() { + # Respect an explicit uv config the user already set. + [[ -n "${UV_DEFAULT_INDEX:-}${UV_INDEX_URL:-}" ]] && return 0 + [[ -f "$HOME/.config/uv/uv.toml" ]] && return 0 + local idx; idx=$(detect_pip_index) + [[ -z "$idx" ]] && return 0 + case "$idx" in *pypi.org/simple*) return 0 ;; esac # already public PyPI — nothing to bridge + assert_pypi_index_sanctioned "pip index-url (bridge)" "$idx" || return 1 + export UV_DEFAULT_INDEX="$idx" + PIP_BRIDGED_INDEX="$idx" +} + +# ─── PyPI reachability preflight ───────────────────────────────────────────── +# Mirror of firefly_preflight_npm_registry, for the same failure shape one layer down. +# bridge_pip_index_to_uv can only forward an index the user already has. When pip is +# unconfigured AND public PyPI is blocked (corp egress policy), the bridge correctly +# no-ops and uv fails eight phases later with a bare +# "Failed to fetch: https://pypi.org/simple// ... 503 Service Unavailable". +# Observed on a clean corp VM on 2026-07-21 and again on 2026-07-25. Name the cause here. +# +# This deliberately does NOT choose an index. The runbook's contract is that every value +# comes from the caller's own config; hardcoding a mirror would also stamp that host into +# any regenerated uv.lock and ship it to every downstream consumer (see #63/#66). +firefly_effective_pypi_index() { + if [[ -n "${UV_DEFAULT_INDEX:-}" ]]; then echo "${UV_DEFAULT_INDEX}"; return; fi + if [[ -n "${UV_INDEX_URL:-}" ]]; then echo "${UV_INDEX_URL}"; return; fi + local uv_toml="$HOME/.config/uv/uv.toml" v + if [[ -f "$uv_toml" ]]; then + v=$(awk -F'"' '/^[[:space:]]*url[[:space:]]*=/{print $2; exit}' "$uv_toml") + [[ -n "$v" ]] && { echo "$v"; return; } + fi + echo "https://pypi.org/simple/" +} + +# Returns non-zero when uv's effective index is unreachable, so callers can gate on it. +firefly_preflight_pypi_index() { + local idx; idx=$(firefly_effective_pypi_index) + if curl -fsS -o /dev/null --max-time 15 "$idx" 2>/dev/null; then + ok "package index reachable: $idx" + return 0 + fi + fail "package index unreachable: $idx" + case "$idx" in + *pypi.org/simple*) + note "uv reads neither pip.conf nor PIP_INDEX_URL — it has its own config. Public PyPI" + note "looks blocked here and no uv index is set, so nothing could be bridged. Set ONE:" + note " python3 -m pip config set global.index-url # then re-run Phase 0a" + note " export UV_DEFAULT_INDEX=" + note "Use your organization's approved mirror. This runbook will not pick one for you." + ;; + *) + note "An index is configured but did not answer. Check VPN/proxy reachability, or that" + note "the intercepting-proxy CA is trusted (re-run bootstrap.sh --trust-proxy-ca)." + ;; + esac + note "Do NOT work around this by disabling TLS verification." + return 1 +} + +# ─── unsanctioned PyPI proxy policy ────────────────────────────────────────── +# `pypi-proxy.dev.databricks.com` is NOT the sanctioned index for this project, and it was +# implicated in the original Apps install timeouts (GAP-8/GAP-11): a lock that stamps .dev +# into every package source is a deployment hazard. +# +# Be precise about what is and is not established: +# PROVEN — .dev URLs are stamped in databricks-apps/guest-manager/uv.lock, and have +# been since that file's first commit (5a78d80, 2026-04-13). Not a regression. +# PROVEN — .dev is NOT a dead host. From a corp laptop it resolves and serves full +# package indexes (HTTP 200, byte-comparable to .cloud). Do not describe it +# as dead; that claim is falsifiable and false. +# NOT PROVEN — whether .dev is reachable from the Databricks Apps runtime egress, which is +# where the timeouts actually happened. That is the check that would justify +# hard-failing rather than warning, and it has not been run. +# +# Because the strongest claim is "unsanctioned + implicated", this defaults to a WARNING and +# only fails hard when FIREFLY_STRICT_PYPI_PROXY=1. Flip the default once someone confirms +# .dev is unreachable from an Apps container. +: "${FIREFLY_UNSANCTIONED_PYPI_HOST:=pypi-proxy.dev.databricks.com}" +: "${FIREFLY_CANONICAL_PYPI_INDEX:=https://pypi-proxy.cloud.databricks.com/simple}" +: "${FIREFLY_STRICT_PYPI_PROXY:=0}" + +# Returns non-zero only in strict mode, so callers can `|| return 1` without forcing exits. +assert_pypi_index_sanctioned() { + local label="$1" value="$2" + [[ -z "$value" ]] && return 0 + case "$value" in + *"${FIREFLY_UNSANCTIONED_PYPI_HOST}"*) + if [[ "$FIREFLY_STRICT_PYPI_PROXY" == "1" ]]; then + fail "$label uses the unsanctioned index ${FIREFLY_UNSANCTIONED_PYPI_HOST}" + note "Set it to ${FIREFLY_CANONICAL_PYPI_INDEX}, then regenerate any uv.lock that stamped .dev." + return 1 + fi + warn "$label uses ${FIREFLY_UNSANCTIONED_PYPI_HOST} (not the sanctioned index)." + note "Prefer ${FIREFLY_CANONICAL_PYPI_INDEX}. Set FIREFLY_STRICT_PYPI_PROXY=1 to make this fatal." + return 0 + ;; + esac + return 0 +} + +# Returns non-zero rather than exiting, so tests and the --check-pypi-proxy flag can call it +# without killing the shell. Under `set -e` a bare call still aborts bootstrap. +reject_dead_pypi_proxy_config() { + local rc=0 + assert_pypi_index_sanctioned "UV_DEFAULT_INDEX" "${UV_DEFAULT_INDEX:-}" || rc=1 + assert_pypi_index_sanctioned "UV_INDEX_URL" "${UV_INDEX_URL:-}" || rc=1 + assert_pypi_index_sanctioned "pip index-url" "$(detect_pip_index)" || rc=1 + local uv_toml="$HOME/.config/uv/uv.toml" + if [[ -f "$uv_toml" ]] && grep -q "$FIREFLY_UNSANCTIONED_PYPI_HOST" "$uv_toml" 2>/dev/null; then + assert_pypi_index_sanctioned "$uv_toml" "$FIREFLY_UNSANCTIONED_PYPI_HOST" || rc=1 + fi + return "$rc" +} + +# Lockfiles are checked-in, deterministic artifacts — a stamped .dev ships to every consumer +# and cannot be corrected by the person running bootstrap. This stays fatal regardless of +# FIREFLY_STRICT_PYPI_PROXY, which only governs the caller's own live pip/uv config. +assert_uv_locks_not_dead_pypi_proxy() { + local f bad=0 + for f in "$@"; do + [[ -f "$f" ]] || continue + if grep -q "$FIREFLY_UNSANCTIONED_PYPI_HOST" "$f"; then + fail "$f stamps ${FIREFLY_UNSANCTIONED_PYPI_HOST} into package sources" + bad=1 + fi + done + if [[ "$bad" -eq 1 ]]; then + note "Fix: point pip/uv at ${FIREFLY_CANONICAL_PYPI_INDEX}, then rewrite ONLY the host:" + note " sed -i '' 's#${FIREFLY_UNSANCTIONED_PYPI_HOST}#pypi-proxy.cloud.databricks.com#g' " + note "A full 'rm -f && uv lock' also re-resolves every version — see #63/#66." + return 1 + fi + return 0 +} + +# Whole-repo check used by `bootstrap.sh --check-pypi-proxy` and the hermetic tests. +check_pypi_proxy_state() { + local repo_dir="$1" rc=0 + reject_dead_pypi_proxy_config || rc=1 + assert_uv_locks_not_dead_pypi_proxy \ + "$repo_dir/agent-build/uv.lock" \ + "$repo_dir/databricks-apps/guest-manager/uv.lock" || rc=1 + return "$rc" +} + +# ─── corepack ← npm registry bridge ────────────────────────────────────────── +# Kept for parity with the uv bridge and for anyone who opts into corepack manually. +# It is NOT part of the supported pnpm path: `npm install -g pnpm@$PNPM_VERSION` uses the +# user's configured registry directly and needs no bridge (see ENV-0 and +# firefly_preflight_npm_registry below). +detect_npm_registry() { + local v f + if command -v npm >/dev/null 2>&1; then + v=$(npm config get registry 2>/dev/null | tr -d '[:space:]' || true) + [[ -n "$v" && "$v" != "undefined" ]] && { echo "$v"; return; } + fi + for f in "$HOME/.npmrc" "${PREFIX:-/usr/local}/etc/npmrc" /etc/npmrc; do + [[ -f "$f" ]] || continue + v=$(awk -F= '/^[[:space:]]*registry[[:space:]]*=/{gsub(/[[:space:]]/,"",$2);print $2; exit}' "$f") + [[ -n "$v" ]] && { echo "$v"; return; } + done +} + +bridge_npm_registry_to_corepack() { + export COREPACK_ENABLE_DOWNLOAD_PROMPT="${COREPACK_ENABLE_DOWNLOAD_PROMPT:-0}" + [[ -n "${COREPACK_NPM_REGISTRY:-}" ]] && return 0 + local reg; reg=$(detect_npm_registry) + [[ -z "$reg" ]] && return 0 + case "$reg" in *registry.npmjs.org*) return 0 ;; esac # already public npm — nothing to bridge + export COREPACK_NPM_REGISTRY="${reg%/}" # strip trailing slash → avoid '//pnpm' 404s + NPM_BRIDGED_REGISTRY="$reg" +} + +# ─── npm reachability preflight ────────────────────────────────────────────── +# The bridges above can only forward a mirror the user has already configured. When npm +# still points at public npm AND public npm is blackholed (corp /etc/hosts sinkhole, DNS +# blackhole, firewall drop), every bridge legitimately no-ops and the first install fails +# with an opaque ECONNREFUSED / ETIMEDOUT / 503. Surface the remedy instead of the raw +# error. Returns non-zero if the pinned pnpm cannot be resolved. +# +# Probe with the EXACT operation the install performs — resolving pnpm@$PNPM_VERSION. +# Do NOT use `npm ping`: corporate mirrors commonly proxy package/tarball routes but not +# service endpoints, so `/-/ping` 404s on a mirror that serves installs perfectly well +# (verified against npm-proxy.dev.databricks.com: `npm ping` → 404, `npm view pnpm@10.34.5 +# version` → 10.34.5). This is the same mirror behaviour that 404s the `pnpm/latest` +# dist-tag. A ping-based gate would falsely block exactly the users it exists to help. +firefly_preflight_npm_registry() { + command -v npm >/dev/null 2>&1 || { fail "npm not found — install Node.js 18+ first."; return 1; } + local reg; reg=$(detect_npm_registry); reg="${reg:-}" + # Bound the failure path: npm's default retry/backoff takes ~70s against an unreachable + # host, which reads as a hang. Fail fast — this is a preflight, not the install. + if npm view "pnpm@${PNPM_VERSION}" version \ + --fetch-retries=0 --fetch-timeout=15000 >/dev/null 2>&1; then + ok "npm registry serves pnpm@${PNPM_VERSION}: $reg" + return 0 + fi + fail "cannot resolve pnpm@${PNPM_VERSION} from npm registry: $reg" + case "$reg" in + *registry.npmjs.org*|'') + note "Public npm appears blocked here and npm is not pointed at an approved mirror." + note "Set your registry, then re-run this phase:" + note " npm config set registry " + ;; + *) + note "npm is configured for a mirror but it did not answer. Check VPN/proxy reachability," + note "or that the intercepting-proxy CA is trusted (re-run bootstrap.sh --trust-proxy-ca)." + ;; + esac + note "Do NOT work around this with --registry https://registry.npmjs.org or by disabling TLS." + return 1 +} + +# ─── orchestrator ──────────────────────────────────────────────────────────── +# Non-interactive by design, so it is safe to source from the runbook. bootstrap.sh calls +# the primitives directly instead, because it runs a richer 3-way TLS prompt. +# FIREFLY_TRUST_PROXY_CA=1 auto-trust a detected proxy CA without prompting +# TLS_PEM_PATH= use a CA bundle you already have +firefly_bridge_corp_network() { + # $HOME/bin holds the gh / databricks binaries installed in Phase 1 and + # $HOME/.local/bin holds uv's, so both have to exist before anything writes to them. + mkdir -p "$HOME/bin" "$HOME/.local/bin" 2>/dev/null || true + + # ...and both have to be ON PATH, here, in the sourced helper. + # + # This function used to only mkdir them while the comment claimed they were "on PATH + # from Phase 0 onward" and the runbook told readers the same thing. The export in fact + # lived in scripts/bootstrap.sh, which a reader following BOOTSTRAP.md by hand never + # runs. So Phase 0a's promise — that re-sourcing this file restores a fresh shell — + # was false for the one thing every later phase depends on: after Phase 1, databricks + # and uv were simply not findable, and the reader had to invent + # `export PATH="$HOME/bin:$HOME/.local/bin:$PATH"` at the top of every new shell. + # + # Prepended idempotently: this file gets re-sourced once per shell across ten phases, + # and a naive prepend would grow PATH without bound. + for _ffdir in "$HOME/bin" "$HOME/.local/bin"; do + case ":$PATH:" in + *":$_ffdir:"*) ;; + *) PATH="$_ffdir:$PATH" ;; + esac + done + unset _ffdir + export PATH + + # uv ships its OWN bundled cert store (rustls/webpki) and ignores the OS keychain by + # default, so it rejects an intercepting proxy's cert (UnknownIssuer) even when Node + # succeeds. Harmless off-proxy — the platform store already trusts public CAs. + export UV_SYSTEM_CERTS="${UV_SYSTEM_CERTS:-1}" + + if [[ -n "${TLS_PEM_PATH:-}" && -f "${TLS_PEM_PATH}" ]]; then + apply_tls_bundle "$TLS_PEM_PATH" + ok "TLS trust from TLS_PEM_PATH → SSL_CERT_FILE / REQUESTS_CA_BUNDLE / NODE_EXTRA_CA_CERTS / CURL_CA_BUNDLE" + elif derive_proxy_ca_bundle; then + warn "Intercepting HTTPS proxy detected — its chain terminates in a PRIVATE root CA." + note " Root CN: ${PROXY_CA_ROOT_CN:-}" + note " SHA-256: ${PROXY_CA_ROOT_FP:-}" + note " Copied to: ${PROXY_CA_BUNDLE:-}" + if [[ "${FIREFLY_TRUST_PROXY_CA:-}" == "1" ]]; then + apply_tls_bundle "$PROXY_CA_BUNDLE" + ok "FIREFLY_TRUST_PROXY_CA=1 → proxy CA trusted for this session." + else + note "Trust it only if that SHA-256 matches your organization's known root CA. To trust:" + note " FIREFLY_TRUST_PROXY_CA=1 firefly_bridge_corp_network" + note " or, if you already have a bundle: TLS_PEM_PATH= firefly_bridge_corp_network" + fi + else + ok "No intercepting proxy detected (TLS to PyPI validates against the system store)." + fi + + bridge_pip_index_to_uv + if [[ -n "${PIP_BRIDGED_INDEX:-}" ]]; then + ok "uv index bridged from pip config → UV_DEFAULT_INDEX=$PIP_BRIDGED_INDEX" + elif [[ -n "${UV_DEFAULT_INDEX:-}${UV_INDEX_URL:-}" ]]; then + note "uv index already set via env — leaving as-is." + else + note "No pip mirror configured — uv will use public PyPI." + # Fine on open internet, fatal behind egress policy that blocks pypi.org. Distinguish + # the two now rather than at Phase 3a. Non-fatal here: this orchestrator is sourced + # from the reader's own shell and must not exit it. Phase 3a gates on the same check. + firefly_preflight_pypi_index \ + || note "Phase 3 will fail until an index is configured — see the guidance above." + fi + + bridge_npm_registry_to_corepack + if [[ -n "${NPM_BRIDGED_REGISTRY:-}" ]]; then + ok "corepack registry bridged from npm config → COREPACK_NPM_REGISTRY=$COREPACK_NPM_REGISTRY" + elif [[ -n "${COREPACK_NPM_REGISTRY:-}" ]]; then + note "corepack registry already set via env — leaving as-is." + else + note "No npm mirror configured — npm will use public npm." + fi +} diff --git a/scripts/lib/runbook.sh b/scripts/lib/runbook.sh new file mode 100644 index 0000000..841d857 --- /dev/null +++ b/scripts/lib/runbook.sh @@ -0,0 +1,817 @@ +# shellcheck shell=bash +# runbook.sh — the helpers BOOTSTRAP.md invokes, shared with scripts/bootstrap.sh. +# +# source scripts/lib/runbook.sh +# +# WHY THIS FILE EXISTS +# Same reason as corp-network.sh, one layer up. BOOTSTRAP.md used to *call* +# store_secret / read_secret and *describe* the CLI installs, while the only real +# implementations lived in bootstrap.sh. Three consequences, all observed on the +# 2026-07-25 fresh-install run: +# +# * store_secret / read_secret were undefined for anyone following the runbook, +# and the script's read_secret had a different signature anyway +# (read_secret VARNAME KEY vs. the runbook's $(read_secret KEY)), +# so even copying it across did not work. +# * The Phase 1b/1e CLI installs were comments, so `databricks` and `gh` were +# never installed on a machine that did not already have them. +# * The Neon project-id parser was fixed in bootstrap.sh and left broken in the +# markdown, because the two were separate copies. +# +# One implementation, sourced by both, is what stops that recurring. +# +# Safe to source from an interactive shell: nothing here sets -e or exits, and +# every function returns rather than terminating the caller. Must work under BOTH +# bash and zsh — BOOTSTRAP.md tells the reader to source this from their own +# shell, and zsh is the macOS default. + +# ─── minimal helpers (defined only if the caller hasn't already) ────────────── +# `declare -F name` is NOT a function test in zsh: it declares a float and returns +# 0. Same guard as corp-network.sh; duplicated so each lib is independently +# sourceable. +_ff_is_func() { + if [ -n "${ZSH_VERSION:-}" ]; then + eval '(( ${+functions[$1]} ))' + else + declare -F "$1" >/dev/null 2>&1 + fi +} + +_ff_is_func note || note() { echo " $*"; } +_ff_is_func ok || ok() { echo " ✓ $*"; } +_ff_is_func warn || warn() { echo " ⚠ $*"; } +_ff_is_func fail || fail() { echo " ✗ $*"; } + +# ─── state.env: secret storage ─────────────────────────────────────────────── +# A gitignored, chmod-600 file under $REPO_DIR. Deliberately not the macOS +# Keychain: the target machine may have no Python `keyring` wired up. +# +# On-disk format is `export KEY=<%q-quoted>` — consumers must SOURCE this file, +# never grep it. (A verifier once grepped '^PREVIEW_URL=' and silently missed +# every value because of the `export ` prefix.) +: "${STATE_DIR:=}" +: "${STATE_FILE:=}" + +init_state_dir() { + local base="${1:-${REPO_DIR:-}}" + # Falling back to $PWD made the state file depend on the caller's directory, so a + # phase run from anywhere but the repo silently read a DIFFERENT (usually absent) + # state.env: Phase 9 decided GUEST_API_SECRET was "0 chars" and spent three minutes + # reminting and redeploying a secret that was already stored correctly. Recover + # REPO_DIR from inputs.env first -- it is written there by firefly_store_inputs -- so + # the location is a property of the bootstrap rather than of the shell's cwd. + if [ -z "$base" ]; then + local inputs="${INPUTS_DIR:-$HOME/.firefly-bootstrap}/inputs.env" + [ -f "$inputs" ] && base="$(sed -nE 's/^REPO_DIR=(.*)$/\1/p' "$inputs" | tail -1)" + fi + [ -n "$base" ] || base="$PWD" + STATE_DIR="${base}/.firefly-bootstrap" + STATE_FILE="${STATE_DIR}/state.env" + mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR" +} + +store_secret() { # store_secret KEY VALUE + local key="$1" val="$2" + init_state_dir + # Storing an empty value over a good one is how PREVIEW_URL became "": Phase 8d does + # store_secret PREVIEW_URL "$APP_ORIGIN", and in a fresh shell APP_ORIGIN was unset, + # so the key was overwritten with nothing. Phase 9's curls then returned HTTP 000 with + # no message to explain it. An empty write is almost always a lost variable, so say so + # and keep what is already there. + if [ -z "$val" ]; then + local existing + existing="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${key}=" "$STATE_FILE" 2>/dev/null | tail -1)" + if [ -n "$existing" ]; then + warn "refusing to overwrite $key with an empty value - keeping what is stored." + warn " the variable you passed was unset in this shell; nothing was lost." + return 0 + fi + warn "$key stored EMPTY (the value passed in was unset in this shell)." + fi + local tmp="${STATE_FILE}.tmp" + touch "$STATE_FILE" + # Drop EVERY prior form of this key, not just `^export KEY=`. A line written as + # `KEY=...` or with leading whitespace used to survive the filter, so state.env + # ended up holding two assignments; sourcing takes the last one, and Phase 9's + # summary printed a stale GENIE_SPACE_ID while the deployment used the right one. + grep -vE "^[[:space:]]*(export[[:space:]]+)?${key}=" "$STATE_FILE" > "$tmp" 2>/dev/null || true + printf 'export %s=%q\n' "$key" "$val" >> "$tmp" + mv "$tmp" "$STATE_FILE" + chmod 600 "$STATE_FILE" + ok "$key → state.env" +} + +load_secrets() { + init_state_dir + # shellcheck disable=SC1090 + [ -f "$STATE_FILE" ] && . "$STATE_FILE" + return 0 +} + +# read_secret KEY → prints the value on stdout; returns 1 if unset or empty. +# +# One argument, stdout. This is the form every BOOTSTRAP.md call site uses, and +# it is the contract. Note the deliberate absence of ${!key}: bash-only indirect +# expansion is what produced `(eval):1: bad substitution` under the guest zsh. +# Intended as $(read_secret KEY), which subshells the `.` so the caller's +# environment is not filled with every secret in the file. +read_secret() { + local key="$1" + init_state_dir + [ -f "$STATE_FILE" ] || return 1 + # Sourcing silently takes the LAST assignment, so a duplicate resolves to whichever + # writer happened to go second and the caller cannot tell. Phase 9 reported a stale + # GENIE_SPACE_ID this way while the deployment had used the correct one. Say it out + # loud rather than returning one of two answers as though it were the only one. + local dups + dups="$(grep -cE "^[[:space:]]*(export[[:space:]]+)?${key}=" "$STATE_FILE" 2>/dev/null || echo 0)" + if [ "${dups:-0}" -gt 1 ]; then + # Straight to stderr, NOT via warn(): warn() echoes to stdout, and every caller + # reads this function as VALUE="$(read_secret KEY)". Routing a diagnostic through + # it would fold the warning text into the value -- the same defect that made + # genie-data-setup.sh emit progress into its own key=value output. + printf ' ⚠ state.env has %s assignments of %s; using the last. Values seen:\n' \ + "$dups" "$key" >&2 + grep -E "^[[:space:]]*(export[[:space:]]+)?${key}=" "$STATE_FILE" 2>/dev/null \ + | sed 's/^/ /' >&2 || true + fi + # shellcheck disable=SC1090 + . "$STATE_FILE" + local val + val="$(eval "printf '%s' \"\${$key-}\"")" + [ -n "$val" ] || return 1 + printf '%s\n' "$val" +} + +# require_secret VARNAME KEY — assign to VARNAME, or explain and return 1. +# Returns rather than exits so it is safe to source; callers that must stop +# should write `require_secret X Y || exit 1`. +require_secret() { + local __var="$1" __key="$2" __val + __val="$(read_secret "$__key")" || __val="" + if [ -z "$__val" ]; then + fail "state.env: $__key is empty (run the earlier phase that stores it first)" + return 1 + fi + eval "$__var=\$__val" +} + +# ─── CLI installers (no Homebrew) ──────────────────────────────────────────── +# Idempotent: each is a no-op when the tool is already on PATH. $HOME/bin is +# created here rather than assumed — a previous regression was an installer that +# wrote into a $HOME/bin that did not exist yet. + +# A floor, not a preference: used only when api.github.com cannot say what "latest" is. Bump it +# when convenient; every phase that matters asserts the CLI works rather than its version. +: "${FIREFLY_DATABRICKS_CLI_FALLBACK:=1.9.0}" + +firefly_install_databricks_cli() { + command -v databricks >/dev/null 2>&1 && { + ok "databricks already installed: $(databricks --version 2>/dev/null | head -1)"; return 0; } + local arch tag tmp + case "$(uname -m)" in + arm64|aarch64) arch=arm64 ;; + x86_64) arch=amd64 ;; + *) fail "unsupported architecture: $(uname -m)"; return 1 ;; + esac + # Resolving "latest" costs one call against api.github.com, which allows 60 per hour PER IP + # unauthenticated. A machine running the runbook repeatedly exhausts that, and the failure is a + # 403 whose body says "API rate limit exceeded" while the old message said "could not resolve + # the latest Databricks CLI release" -- sending the reader to look at networking or at the + # Databricks release page, neither of which is the problem. It also killed the whole run: + # Phases 3a-6 and 8b-9 all die without the CLI, with no recovery path. + # + # So: authenticate when a token is available, and when the API cannot answer, fall back to a + # pinned version rather than failing. The download host is not API-rate-limited, so the + # fallback works exactly when the lookup does not. + local gh_auth=() api_err tag_body + local tok="${GH_TOKEN:-${GITHUB_TOKEN:-}}" + [ -z "$tok" ] && command -v gh >/dev/null 2>&1 && tok="$(gh auth token 2>/dev/null || true)" + [ -n "$tok" ] && gh_auth=(-H "Authorization: Bearer $tok") + + api_err="$(mktemp)" + tag_body="$(curl -fsSL ${gh_auth[@]+"${gh_auth[@]}"} \ + https://api.github.com/repos/databricks/cli/releases/latest 2>"$api_err")" || tag_body="" + tag="$(printf '%s' "$tag_body" | python3 -c "import sys,json +try: print(json.load(sys.stdin)['tag_name'].lstrip('v')) +except Exception: print('')" 2>/dev/null)" + + if [ -z "$tag" ]; then + tag="$FIREFLY_DATABRICKS_CLI_FALLBACK" + warn "could not ask api.github.com which release is latest:" + warn " $(head -1 "$api_err" 2>/dev/null | cut -c1-140)" + if [ -z "$tok" ]; then + warn " Unauthenticated api.github.com allows 60 requests/hour per IP, and a 403 here is" + warn " usually that limit rather than a network or Databricks problem. Set GH_TOKEN (or" + warn " run 'gh auth login') to raise it." + fi + warn " Installing the pinned v${tag} instead - the download host is not rate-limited." + fi + rm -f "$api_err" + tmp=$(mktemp -d) + curl -fsSL "https://github.com/databricks/cli/releases/download/v${tag}/databricks_cli_${tag}_darwin_${arch}.zip" \ + -o "$tmp/db.zip" || { rm -rf "$tmp"; fail "download failed"; return 1; } + unzip -qo "$tmp/db.zip" -d "$tmp" || { rm -rf "$tmp"; fail "unzip failed"; return 1; } + mkdir -p "$HOME/bin" && cp "$tmp/databricks" "$HOME/bin/databricks" && chmod +x "$HOME/bin/databricks" + rm -rf "$tmp" + export PATH="$HOME/bin:$PATH" + ok "databricks installed to \$HOME/bin ($(databricks --version 2>/dev/null | head -1))" +} + +firefly_install_gh() { + command -v gh >/dev/null 2>&1 && { + ok "gh already installed: $(gh --version 2>/dev/null | head -1)"; return 0; } + local arch tag tmp + case "$(uname -m)" in + arm64|aarch64) arch=macOS_arm64 ;; + x86_64) arch=macOS_amd64 ;; + *) fail "unsupported architecture: $(uname -m)"; return 1 ;; + esac + tag=$(curl -fsSL https://api.github.com/repos/cli/cli/releases/latest \ + | python3 -c "import sys,json;print(json.load(sys.stdin)['tag_name'].lstrip('v'))") || { + fail "could not resolve the latest GitHub CLI release"; return 1; } + tmp=$(mktemp -d) + curl -fsSL "https://github.com/cli/cli/releases/download/v${tag}/gh_${tag}_${arch}.zip" \ + -o "$tmp/gh.zip" || { rm -rf "$tmp"; fail "download failed"; return 1; } + unzip -qo "$tmp/gh.zip" -d "$tmp" || { rm -rf "$tmp"; fail "unzip failed"; return 1; } + mkdir -p "$HOME/bin" && cp "$tmp"/gh_*/bin/gh "$HOME/bin/gh" && chmod +x "$HOME/bin/gh" + rm -rf "$tmp" + export PATH="$HOME/bin:$PATH" + ok "gh installed to \$HOME/bin ($(gh --version 2>/dev/null | head -1))" +} + +# ─── CLI output parsers ────────────────────────────────────────────────────── +# These have fixture tests (scripts/test-parsing.sh). Keeping one copy is the +# point: the markdown's copy of the Neon parser read a top-level 'id' for months +# after the script's copy was fixed to read project.id. + +extract_vercel_preview_url() { + grep -oE 'https://[^ ]*\.vercel\.app' | tail -1 +} + +# `neonctl projects create --output json` nests the project under a "project" +# key. The flat fallback covers older CLI versions. +extract_neon_project_id() { + python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('project',{}).get('id') or d.get('id',''))" +} + +# firefly_neon_project_id [ORG_FLAG...] — id of the project named +# $NEON_PROJECT_NAME, or empty. Neon project names are NOT unique (ids are), so +# every lookup is by name against the list. +firefly_neon_project_id() { + NEON_PROJECT_NAME="$NEON_PROJECT_NAME" neonctl projects list "$@" --output json 2>/dev/null \ + | python3 -c "import os,sys,json;d=json.load(sys.stdin);ps=d.get('projects',d) if isinstance(d,dict) else d;print(next((p['id'] for p in ps if p.get('name')==os.environ['NEON_PROJECT_NAME']),''))" +} + +# ─── bundle assertions ─────────────────────────────────────────────────────── + +# The committed agent/databricks.yml carries placeholder resource bindings that +# `uv run scripts/quickstart.py` (Phase 3a) rewrites for the target workspace. +# Deploying without that rewrite fails with a 404 that names only the stale id +# — "Node ID 123237888438046 does not exist" — which points nowhere useful. +: "${FIREFLY_PLACEHOLDER_EXPERIMENT_ID:=123237888438046}" + +assert_bundle_quickstart_ran() { # assert_bundle_quickstart_ran + local yml="${1:-agent-build/databricks.yml}" + [ -f "$yml" ] || { fail "$yml not found — run scripts/assemble_agent.sh (Phase 2) first."; return 1; } + # Re-apply the default HERE, not just at source time. The id is set with + # `: "${VAR:=...}"` at the top of this file, which runs once; if the caller's + # shell has it exported empty, the grep below becomes `grep -q ""` — and that + # matches EVERY line, so a bundle quickstart had already rewritten correctly + # fails the assert and the reader is told to re-run Phase 3a. Three E2E runs + # lost time to that. + [ -n "${FIREFLY_PLACEHOLDER_EXPERIMENT_ID:-}" ] \ + || FIREFLY_PLACEHOLDER_EXPERIMENT_ID=123237888438046 + if grep -q "$FIREFLY_PLACEHOLDER_EXPERIMENT_ID" "$yml"; then + fail "$yml still holds the placeholder experiment id ($FIREFLY_PLACEHOLDER_EXPERIMENT_ID)." + note "Phase 3a (quickstart) has not completed against this workspace, so the bundle" + note "still points at the authoring workspace. Deploying now returns a 404 naming that" + note "id. Re-run Phase 3a, then retry this phase." + return 1 + fi + ok "bundle resource bindings were written by quickstart" +} + +# Phase 3e. Three rules that must hold simultaneously in the bundle's sync.exclude: +# pyproject.toml NOT excluded — the Apps build needs it to find the dep list +# uv.lock excluded — forces a plain `uv sync` that can use UV_FIND_LINKS +# vendor-wheels/ NOT excluded — the local wheels must reach the build +# Parsed with a YAML-aware reader rather than a regex: the exclude list opens +# with comment lines, and a naive `-\s` scan reads it as empty and "passes" a +# broken config (or fails a correct one, which is what happened on 2026-07-25). +check_sync_exclude_rules() { # check_sync_exclude_rules + local yml="${1:-agent-build/databricks.yml}" + [ -f "$yml" ] || { fail "$yml not found"; return 1; } + python3 - "$yml" <<'PY' +import re, sys +text = open(sys.argv[1]).read() +m = re.search(r'^sync:\s*$', text, re.M) +if not m: + print(" \u2717 no sync: block found"); sys.exit(1) +block, started = [], False +for line in text[m.start():].splitlines()[1:]: + if line.strip() and not line[:1].isspace(): + break # dedented to a new top-level key + s = line.strip() + if s.startswith('exclude:'): + started = True; continue + if started and s.startswith('- '): + block.append(s[2:].strip()) + elif started and s and not s.startswith('#') and not s.startswith('- '): + break +rules = { + 'pyproject.toml': (False, 'the Apps build needs it to find the dependency list'), + 'uv.lock': (True, 'forces a plain `uv sync`, so UV_FIND_LINKS can supply wheels'), + 'vendor-wheels': (False, 'the vendored wheels must reach the build host'), +} +bad = 0 +for name, (must_exclude, why) in rules.items(): + present = any(name in e for e in block) + if present != must_exclude: + state = "excluded" if present else "not excluded" + want = "excluded" if must_exclude else "NOT excluded" + print(f" \u2717 {name} is {state}; it must be {want} \u2014 {why}") + bad = 1 +if bad: + print(f" sync.exclude currently: {block}") + sys.exit(1) +print(f" \u2713 sync.exclude rules hold ({len(block)} entries)") +PY +} + +# ─── SQL over a warehouse (Phase 6c) ───────────────────────────────────────── +# Every SQL step in this runbook used to be a `note` telling the reader to open a +# warehouse session and paste it themselves, so nothing SQL-shaped was ever +# actually executed or verified. Phase 6c needs real execution (seeding, grants, +# table counts), so this is the one primitive that runs a statement and returns +# rows. +# +# `wait_timeout` maxes out at 50s server-side, which a 16-table CTAS can exceed, +# so a statement that is still PENDING/RUNNING when the wait expires is polled to +# completion rather than reported as a failure. on_wait_timeout=CONTINUE is what +# keeps it running instead of being cancelled at the 50s mark. +# +# Output: one line per result row, tab-separated. Non-zero exit means the +# statement did not reach SUCCEEDED, with the server's message on stderr. +: "${FIREFLY_SQL_DEADLINE:=900}" # seconds to wait for one statement + +firefly_sql() { # firefly_sql [profile] + local wh="$1" sql="$2" prof="${3:-${DB_PROFILE:-}}" + [ -n "$wh" ] || { fail "firefly_sql: no warehouse id given"; return 2; } + [ -n "$prof" ] || { fail "firefly_sql: no Databricks profile given"; return 2; } + + local req; req="$(mktemp -t ffsql)" || return 2 + FF_WH="$wh" FF_SQL="$sql" python3 -c 'import json,os,sys +sys.stdout.write(json.dumps({ + "warehouse_id": os.environ["FF_WH"], + "statement": os.environ["FF_SQL"], + "wait_timeout": "50s", + "on_wait_timeout": "CONTINUE", +}))' > "$req" || { rm -f "$req"; fail "firefly_sql: could not build request"; return 2; } + + local out rc=0 + # `|| rc=$?` rather than a bare assignment followed by `rc=$?`: under `set -e` the + # assignment aborts the shell the moment the CLI exits non-zero, so the rc check + # below -- and the diagnostic it prints -- were unreachable in exactly the case they + # exist for. The `||` makes it a compound command, which errexit tolerates. + out="$(databricks api post /api/2.0/sql/statements --json "@$req" --profile "$prof" 2>&1)" || rc=$? + rm -f "$req" + if [ "$rc" -ne 0 ]; then + fail "firefly_sql: statement submit failed" + printf '%s\n' "$out" | head -3 >&2 + return 1 + fi + + # PENDING/RUNNING → poll. Anything else is terminal and parsed below. + local sid deadline state + sid="$(printf '%s' "$out" | python3 -c 'import sys,json +try: print((json.load(sys.stdin) or {}).get("statement_id") or "") +except Exception: print("")' 2>/dev/null)" + state="$(printf '%s' "$out" | python3 -c 'import sys,json +try: print(((json.load(sys.stdin) or {}).get("status") or {}).get("state") or "") +except Exception: print("")' 2>/dev/null)" + + deadline=$(( $(date +%s) + FIREFLY_SQL_DEADLINE )) + while [ "$state" = "PENDING" ] || [ "$state" = "RUNNING" ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + databricks api post "/api/2.0/sql/statements/$sid/cancel" --profile "$prof" >/dev/null 2>&1 + fail "firefly_sql: statement $sid still $state after ${FIREFLY_SQL_DEADLINE}s (cancelled)" + return 1 + fi + sleep 5 + out="$(databricks api get "/api/2.0/sql/statements/$sid" --profile "$prof" 2>&1)" || { + fail "firefly_sql: polling $sid failed"; return 1; } + state="$(printf '%s' "$out" | python3 -c 'import sys,json +try: print(((json.load(sys.stdin) or {}).get("status") or {}).get("state") or "") +except Exception: print("")' 2>/dev/null)" + done + + printf '%s' "$out" | python3 -c 'import sys,json +try: + d = json.load(sys.stdin) or {} +except Exception: + sys.stderr.write("firefly_sql: unparseable response\n"); sys.exit(1) +st = d.get("status") or {} +if st.get("state") != "SUCCEEDED": + msg = (st.get("error") or {}).get("message", "") + sys.stderr.write("firefly_sql: %s %s\n" % (st.get("state"), msg[:400])); sys.exit(1) +for row in ((d.get("result") or {}).get("data_array") or []): + print("\t".join("" if c is None else str(c) for c in row)) +' +} + +# ─── Phase 6: restore the context a fresh shell dropped ─────────────────────── +# Phases 6, 6b and 6c read WAREHOUSE_ID, GUEST_SP_CLIENT_ID, SP_CLIENT_ID, +# GENIE_MCP_MODE and GENIE_SPACE_ID straight out of the shell. Anyone who opened a new +# terminal -- or any agent running the phases as separate commands -- lost them, and +# every resulting error named something other than the empty variable: +# +# WAREHOUSE_ID="" -> PATCH /permissions/warehouses/ -> +# "No API found for 'PATCH /permissions/warehouses/'" +# which reads as a wrong API route +# GUEST_SP_CLIENT_ID="" -> "Principal: ServicePrincipalName() does not exist" +# which reads as a SCIM problem +# GENIE_MCP_MODE="" -> `if [ "$GENIE_MCP_MODE" = "space" ]` SILENTLY skips and +# the app never receives genie_space_id at all +# +# Rederive from state.env first, then from the workspace. Whatever cannot be recovered +# is named, because "WAREHOUSE_ID is empty" is the one message that points at the fix. +firefly_restore_phase6_context() { # firefly_restore_phase6_context [profile] + local prof="${1:-${DB_PROFILE:-}}" v + [ -n "$prof" ] || { fail "firefly_restore_phase6_context: no profile"; return 1; } + + if [ -z "${WAREHOUSE_ID:-}" ]; then + v="$(read_secret WAREHOUSE_ID 2>/dev/null || true)" + [ -n "$v" ] || v="$(databricks warehouses list -o json --profile "$prof" 2>/dev/null \ + | python3 -c 'import sys,json +try: d = json.load(sys.stdin) or [] +except Exception: d = [] +ws = d if isinstance(d, list) else (d.get("warehouses") or []) +print(ws[0].get("id","") if ws else "")' 2>/dev/null || true)" + [ -n "$v" ] && { WAREHOUSE_ID="$v"; export WAREHOUSE_ID; note "restored WAREHOUSE_ID=$v"; } + fi + + if [ -z "${GUEST_SP_CLIENT_ID:-}" ]; then + v="$(read_secret GUEST_SP_CLIENT_ID 2>/dev/null || true)" + [ -n "$v" ] || v="$(databricks service-principals list \ + --filter 'displayName eq "firefly-guest-sp"' -o json --profile "$prof" 2>/dev/null \ + | python3 -c 'import sys,json +try: l = json.load(sys.stdin) or [] +except Exception: l = [] +m = [s for s in l if s.get("displayName") == "firefly-guest-sp"] +print(m[0].get("applicationId","") if m else "")' 2>/dev/null || true)" + [ -n "$v" ] && { GUEST_SP_CLIENT_ID="$v"; export GUEST_SP_CLIENT_ID + note "restored GUEST_SP_CLIENT_ID=$v"; } + fi + + if [ -z "${SP_CLIENT_ID:-}" ] && [ -n "${AGENT_APP_NAME:-}" ]; then + v="$(databricks apps get "$AGENT_APP_NAME" -o json --profile "$prof" 2>/dev/null \ + | python3 -c 'import sys,json +try: print((json.load(sys.stdin) or {}).get("service_principal_client_id") or "") +except Exception: print("")' 2>/dev/null || true)" + [ -n "$v" ] && { SP_CLIENT_ID="$v"; export SP_CLIENT_ID; note "restored SP_CLIENT_ID=$v"; } + fi + + # Both stores are consulted, because the two values are not written to the same one: + # store_secret puts GENIE_SPACE_ID in state.env, while Phase 3f records GENIE_MCP_MODE with + # firefly_store_input, which writes inputs.env. Reading only state.env meant the mode was + # never found, and the "assuming space" line below announced a guess about a value Phase 3f + # had already written down. A run reported it: the message is accurate about what this + # function could see and wrong about what was known. + for v in GENIE_SPACE_ID GENIE_MCP_MODE; do + eval "[ -n \"\${$v:-}\" ]" && continue + eval "$v=\"\$(read_secret $v 2>/dev/null || true)\"; export $v" + eval "[ -n \"\${$v:-}\" ]" || \ + eval "$v=\"\$(firefly_read_input $v 2>/dev/null || true)\"; export $v" + eval "[ -n \"\${$v:-}\" ]" && note "restored $v=$(eval "printf '%s' \"\$$v\"")" + done + # A space id with no mode anywhere is the silent-skip case: say so rather than skipping. + if [ -n "${GENIE_SPACE_ID:-}" ] && [ -z "${GENIE_MCP_MODE:-}" ]; then + GENIE_MCP_MODE="space"; export GENIE_MCP_MODE + note "GENIE_SPACE_ID is set but GENIE_MCP_MODE is in neither state.env nor inputs.env" + note " - defaulting to space, which matches a space id being present" + fi + return 0 +} + +# firefly_require ... — refuse to build a request out of an empty variable. +# Without this the empty value reaches the API and the API complains about something +# else entirely: an empty warehouse id becomes a missing route, an empty principal +# becomes a non-existent service principal. +firefly_require() { + local missing="" k + for k in "$@"; do + eval "[ -n \"\${$k:-}\" ]" || missing="$missing $k" + done + [ -z "$missing" ] && return 0 + fail "these variables are empty:$missing" + note "Phase 6 needs them in the shell. Restore them with:" + note " firefly_restore_phase6_context \"\$DB_PROFILE\"" + return 1 +} + +# ─── Apps: let one deployment finish before starting the next ───────────────── +# Phase 6c redeploys the app with genie_mcp_mode=space immediately after Phase 4 +# deployed it, and the Apps API refuses that with +# 400 Cannot update app ... as there is a pending deployment in progress for less +# than 20 minutes +# The run recovered because `bundle run` waits, but the deploy error arrived next to +# an unrelated "WAL recovery failed" and read as a hard failure in the middle of a +# phase that had otherwise worked. +# +# Polls until the active deployment reaches a terminal state. Returns 0 when settled +# and 0 on timeout too: the caller should attempt its deploy either way, because a +# stuck poll must not be the thing that stops the phase. Every capture is `|| true` +# for the reason invariant 31 exists. +firefly_wait_app_deploy_settled() { # [timeout_s] + local app="$1" prof="$2" timeout="${3:-600}" waited=0 state + [ -n "$app" ] && [ -n "$prof" ] || return 0 + while [ "$waited" -lt "$timeout" ]; do + state="$(databricks apps get "$app" -o json --profile "$prof" 2>/dev/null \ + | python3 -c 'import sys,json +try: d = json.load(sys.stdin) or {} +except Exception: print(""); raise SystemExit +print(((d.get("active_deployment") or {}).get("status") or {}).get("state") or "")' 2>/dev/null || true)" + case "$state" in + # No app or no deployment yet: nothing to wait behind. + "") return 0 ;; + SUCCEEDED|FAILED|CANCELLED) return 0 ;; + esac + [ "$waited" -eq 0 ] && note "waiting for the in-flight deployment of '$app' to finish (state=$state)" + sleep 15 + waited=$((waited + 15)) + done + warn "deployment of '$app' still $state after ${timeout}s - attempting the deploy anyway." + warn "if it returns 'pending deployment in progress', re-run this phase; nothing is lost." + return 0 +} + +# ─── IP allowlist: three outcomes, never two ───────────────────────────────── +# There were two copies of this check and they reached OPPOSITE conclusions on the +# same workspace. Phase 1b surfaced "IP access list is not available in the pricing +# tier of this workspace"; Phase 9 discarded stderr, failed to parse, and printed +# "ok: no enabled IP allowlist on this workspace - the app can reach it". +# +# That is worse than a confusing message. It is a false all-clear on a control that +# decides whether the data plane works at all, and it appears in the phase people +# read when something has already gone wrong. +# +# "Could not determine" is a distinct answer from "there is none" — and a workspace +# whose TIER has no IP-allowlist feature is a third thing again. That one reads like +# a failure ("Error: IP access list is not available in the pricing tier of this +# workspace") but is actually a determinate, reassuring answer: the feature does not +# exist here, so nothing can be enabled and the data-plane risk does not apply. +# Reporting it as "could not check" is needlessly alarming; reporting it as a bare +# "none" throws away why. Echoes exactly one of: +# none checked, nothing enabled +# enabled: checked, these are enabled +# unavailable: the tier has no such feature — implies none, safely +# unknown: could not check — say why, never imply safety +firefly_ip_allowlist_status() { # firefly_ip_allowlist_status [profile] + local prof="${1:-${DB_PROFILE:-}}" raw + [ -n "$prof" ] || { echo "unknown:no profile given"; return 0; } + # `|| true` is load-bearing. On a tier without the feature the CLI exits 1, and an + # assignment from a failing command substitution aborts the shell under `set -e` in + # BOTH bash and zsh. The first version of this helper omitted it, so the branch + # written to handle the pricing tier gracefully instead killed Phase 1b and Phase 9 + # on exactly the workspaces that hit it -- a worse failure than the wrong "ok" it + # replaced, and one that only appears when the caller enables errexit. + raw="$(databricks api get /api/2.0/ip-access-lists --profile "$prof" 2>&1 || true)" + FF_RAW="$raw" python3 -c ' +import json, os, re, sys +raw = (os.environ.get("FF_RAW") or "").strip() +if not raw: + print("unknown:no response from the API"); sys.exit() +try: + d = json.loads(raw) +except ValueError: + # Not JSON: the CLI or the API said something. Pass its own words through, but + # separate the tier answer from a genuine failure — it is determinate, and + # calling it "could not check" sends the reader hunting an auth problem. + flat = " ".join(raw.split()) + if re.search(r"not available in the pricing tier|not supported.{0,30}tier" + r"|requires.{0,20}(premium|enterprise)", flat, re.I): + print("unavailable:" + flat[:160]) + else: + print("unknown:" + flat[:160]) + sys.exit() +if not isinstance(d, dict): + print("unknown:unexpected response shape"); sys.exit() +on = [l.get("label") or "?" for l in (d.get("ip_access_lists") or []) + if l.get("enabled") and l.get("list_type") == "ALLOW"] +print("enabled:" + " / ".join(on) if on else "none") +' +} + +# ─── Lakebase: report what got BOUND, not what was asked for ───────────────── +# Passing --app-name for an app that already exists makes quickstart bind Lakebase +# from that app's existing configuration and ignore --lakebase-create-new. The +# requested project is never created, Phase 3a still reports PASS, and every later +# summary prints LAKEBASE_NAME — a resource that does not exist. Observed across +# two passes: pass 1 created firefly-lb-0727083127, pass 2 asked for +# firefly-lb-0727090103, quickstart bound the first, and the summary named the +# second. +# +# The bound project is discoverable: quickstart writes agent-build/.env and +# patches databricks.yml, and both carry the endpoint path +# `projects//branches/-branch/...`. Read the name out of that rather +# than trusting the request. +firefly_bound_lakebase() { # firefly_bound_lakebase [agent_build_dir] + local dir="${1:-${REPO_DIR:-$PWD}/agent-build}" f + for f in "$dir/.env" "$dir/databricks.yml"; do + [ -f "$f" ] || continue + sed -nE 's|.*projects/([A-Za-z0-9_-]+)/branches/.*|\1|p' "$f" | head -1 | grep . && return 0 + done + return 1 +} + +# Reconcile the request against reality and let reality win, loudly. +firefly_reconcile_lakebase() { # firefly_reconcile_lakebase [agent_build_dir] + local bound + bound="$(firefly_bound_lakebase "$@" 2>/dev/null)" || { + warn "could not determine which Lakebase project quickstart bound" + return 0 + } + if [ -n "$bound" ] && [ "$bound" != "${LAKEBASE_NAME:-}" ]; then + warn "quickstart bound Lakebase '$bound', NOT the requested '${LAKEBASE_NAME:-}'." + note "An existing --app-name wins over --lakebase-create-new: the app's own" + note "Lakebase binding is reused and the requested project is never created." + note "Reporting the bound name from here on, so the summary matches reality." + LAKEBASE_NAME="$bound" + export LAKEBASE_NAME + # An in-shell export is not enough. Non-secret answers are re-sourced from + # inputs.env on a later shell or a resumed run, which would resurrect the + # requested-but-never-created name in the summary. Persist the truth. + firefly_store_input LAKEBASE_NAME "$bound" + else + ok "Lakebase '$bound' matches the requested name" + fi +} + +# Persist a non-secret answer to inputs.env, the file a resumed run re-sources. +# bootstrap.sh has its own store_input; this is the library equivalent so the +# runbook and anything sourcing this lib can persist too, instead of the value +# living only in one shell. +# firefly_store_inputs [KEY...] — persist the Phase 0 answers, defaulting to every +# [ASK] row in BOOTSTRAP.md. Guarded by invariant 34 so the list cannot drift. +# +# This exists because the runbook only ever said that bootstrap.sh saves answers to +# ~/.firefly-bootstrap/inputs.env, and showed nothing for anyone working through the +# phases by hand. A reader therefore had to invent the loop, and what they reached for +# was `for k in ...; do firefly_store_input "$k" "${!k}"; done` -- bash-only indirect +# expansion, which raises `bad substitution` under zsh, the macOS default shell. That +# same hazard had already been documented on read_secret, so leaving the safe form +# unwritten is what let it recur somewhere new. Offer the loop rather than the trap. +firefly_store_inputs() { + local k + # Positional parameters, not `for k in $keys`. zsh does NOT word-split an unquoted + # parameter, so that loop ran exactly once with the entire list as a single key name + # and stored nothing -- the first version of this helper had that bug, in the very + # function written to spare the reader a shell portability trap. `set --` with literal + # words and `for k in "$@"` behave identically in bash and zsh. + if [ "$#" -eq 0 ]; then + set -- DATABRICKS_HOST DB_PROFILE UC_CATALOG UC_SCHEMA \ + SEED_SAMPLE_DATA GENIE_SPACE_IDS CREATE_GENIE_SPACE GRANT_GUEST_SPACE_ACCESS \ + AGENT_APP_NAME DATABRICKS_ACCOUNT_ID LAKEBASE_NAME NEON_PROJECT_NAME \ + VERCEL_TEAM VERCEL_PROJECT REPO_DIR + fi + for k in "$@"; do + # `eval` deliberately, NOT ${!k}: portable to bash and zsh alike. Unset keys store + # as empty rather than erroring, so a partially answered Phase 0 still persists. + eval "firefly_store_input \"\$k\" \"\${$k-}\"" + done + ok "Phase 0 answers → ${INPUTS_DIR:-$HOME/.firefly-bootstrap}/inputs.env" +} + +# firefly_read_input KEY → prints the stored answer, or nothing when absent. +# +# The writer existed without a reader, which is why init_state_dir had to inline its own +# sed to recover REPO_DIR. A phase that persists an answer and a later phase that needs it +# back should not each invent the parsing. +firefly_read_input() { # firefly_read_input KEY + local key="$1" file + [ -n "$key" ] || return 1 + file="${INPUTS_DIR:-$HOME/.firefly-bootstrap}/inputs.env" + [ -f "$file" ] || return 1 + # Last write wins, matching how store_input appends after filtering. + sed -nE "s/^${key}=(.*)\$/\\1/p" "$file" | tail -1 +} + +firefly_store_input() { # firefly_store_input KEY VALUE + local key="$1" val="$2" dir file + [ -n "$key" ] || return 1 + dir="${INPUTS_DIR:-$HOME/.firefly-bootstrap}" + file="$dir/inputs.env" + mkdir -p "$dir" 2>/dev/null || return 1 + # The dedupe must not hang off grep's exit status. `grep -v` exits 1 when it filters out + # EVERY line, which is exactly the case where the file holds only this key -- so the + # chained `&& mv` never ran, the original survived, and the append produced a second + # copy. Storing the same answer twice therefore duplicated it, and the file grew on every + # re-run. Readers that take the last match hid this; a reader taking the first would have + # returned a stale value. + if [ -f "$file" ]; then + grep -v "^${key}=" "$file" > "$file.tmp" 2>/dev/null || : + mv "$file.tmp" "$file" 2>/dev/null || : + fi + printf '%s=%s\n' "$key" "$val" >> "$file" +} + +# Set expectations BEFORE quickstart runs, not after it has surprised you. +# The create path reads as though it will provision the named instance right up +# until the post-hoc warning appears; if the app already exists, its binding is +# going to win and that is knowable in advance. +firefly_warn_existing_app_wins() { # firefly_warn_existing_app_wins + local app="$1" prof="$2" + [ -n "$app" ] && [ -n "$prof" ] || return 0 + if databricks apps get "$app" --profile "$prof" >/dev/null 2>&1; then + warn "app '$app' already exists, so ITS Lakebase binding will win here." + note "--lakebase-create-new will not provision '${LAKEBASE_NAME:-}'; the name is" + note "reconciled after quickstart and the summary will show what was bound." + # quickstart prints its `bundle deployment bind` suggestion on BOTH paths, and the + # first version of this pre-empt only defused the app-absent one -- so on the + # app-exists path it still read as an actionable next step. Phase 4's plain deploy + # and run work without binding either way. + note "quickstart will also suggest \`databricks bundle deployment bind\`. Ignore it:" + note "Phase 4's plain \`bundle deploy\` / \`bundle run\` is the documented path." + else + # The opposite case needs pre-empting too. quickstart.py answers --app-name for + # an app that does not exist with + # Could not fetch app details: App with name '' does not exist or is deleted + # and then suggests `databricks bundle deployment bind` / `bundle deploy`. On a + # first run that reads like a recovery path for an app someone deleted, when the + # truth is that nothing is wrong and Phase 4 has not created it yet. Say so + # before the message appears rather than leaving the reader to interpret it. + note "app '$app' does not exist yet - expected on a first run." + note "quickstart will print \"does not exist or is deleted\" and suggest" + note "\`bundle deployment bind\`. Ignore both: Phase 4 creates the app. No bind" + note "step is needed, and a fresh Lakebase WILL be provisioned as requested." + fi +} + +# ─── sha256sum shim (Phase 1: before the uv installer) ─────────────────────── +# The astral.sh uv installer verifies its own download with `sha256sum`, which +# does not exist on stock macOS — the tool there is `shasum`. So it prints +# +# skipping sha256 checksum verification (it requires the 'sha256sum' command) +# +# and installs an UNVERIFIED binary. On a clean VM that is the default path, and +# the message scrolls past in the middle of a long install. We are fetching an +# executable over the network; declining to check its integrity is not something +# to accept silently just because the installer offers to. +# +# `shasum -a 256` is byte-identical to `sha256sum` for both hashing and -c check +# mode (verified), so a two-line shim on PATH restores the installer's own +# verification rather than working around it. +firefly_ensure_sha256sum() { + command -v sha256sum >/dev/null 2>&1 && return 0 + if ! command -v shasum >/dev/null 2>&1; then + warn "neither sha256sum nor shasum found — installers cannot verify checksums" + return 1 + fi + mkdir -p "$HOME/bin" || return 1 + cat > "$HOME/bin/sha256sum" <<'SHIM' +#!/bin/sh +# Shim: stock macOS ships `shasum`, not `sha256sum`. Output and -c behaviour are +# identical for SHA-256, so installers that require sha256sum can verify their +# downloads instead of skipping the check. +exec shasum -a 256 "$@" +SHIM + chmod +x "$HOME/bin/sha256sum" + case ":$PATH:" in *":$HOME/bin:"*) ;; *) PATH="$HOME/bin:$PATH"; export PATH ;; esac + ok "provided a sha256sum shim so installers can verify their downloads" +} + +# ─── Vercel API context (Phases 8a and 8e) ─────────────────────────────────── +# Sets V_TOKEN / V_ORG / V_PROJ. Both phases need them and 8a used to be the only +# place they were derived, so running 8e in a fresh shell produced an empty +# Authorization header — the verify curl failed and reported "production does not +# serve $APP_ORIGIN" for a deployment that was perfectly fine. Deriving in one +# reusable place means a new shell cannot change the answer. +# +# Token precedence: explicit env first (CI / token-based setups), then the CLI's +# macOS store, then its XDG location. A hardcoded path is a silent failure for +# anyone whose Vercel auth lives elsewhere. +firefly_vercel_context() { # firefly_vercel_context [repo_dir] + local repo="${1:-${REPO_DIR:-$PWD}}" auth + V_TOKEN="${VERCEL_TOKEN:-${V_TOKEN:-}}" + for auth in "$HOME/Library/Application Support/com.vercel.cli/auth.json" \ + "$HOME/.local/share/com.vercel.cli/auth.json"; do + [ -n "$V_TOKEN" ] && break + [ -f "$auth" ] || continue + V_TOKEN=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("token",""))' "$auth" 2>/dev/null || echo "") + done + [ -n "$V_TOKEN" ] || warn "no Vercel token found — project API calls will fail" + + if [ -f "$repo/.vercel/project.json" ]; then + V_ORG=$(python3 -c "import json;print(json.load(open('$repo/.vercel/project.json'))['orgId'])" 2>/dev/null || echo "") + V_PROJ=$(python3 -c "import json;print(json.load(open('$repo/.vercel/project.json'))['projectId'])" 2>/dev/null || echo "") + else + warn "$repo/.vercel/project.json not found — run Phase 8a (vercel link) first" + fi + export V_TOKEN V_ORG V_PROJ +} + +# Convenience: the single scalar a lot of Phase 6c checks want (COUNT(*), etc). +firefly_sql_scalar() { # firefly_sql_scalar [profile] + firefly_sql "$1" "$2" "${3:-${DB_PROFILE:-}}" | head -1 | cut -f1 +} diff --git a/scripts/new-guest-link.sh b/scripts/new-guest-link.sh new file mode 100755 index 0000000..84eef21 --- /dev/null +++ b/scripts/new-guest-link.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# new-guest-link.sh — Mint a fresh guest login link for a deployed Firefly app. +# +# Guest links are single-use and expire ~10 minutes after they are minted; a +# refresh or back-navigation also consumes one. The link Phase 9 produces is +# therefore usually dead by the time anyone clicks it, and re-running bootstrap +# to get a live one is absurd. This replays only the three guest API calls. +# +# Usage: +# bash scripts/new-guest-link.sh # from $REPO_DIR +# bash scripts/new-guest-link.sh --open # also open it in a browser +# bash scripts/new-guest-link.sh --state PATH # a different state.env +# bash scripts/new-guest-link.sh --host URL # override the workspace URL +# +# Reads PREVIEW_URL, GUEST_API_SECRET, GUEST_SP_CLIENT_ID and GUEST_SP_SECRET +# from $REPO_DIR/.firefly-bootstrap/state.env, and DATABRICKS_HOST from +# ~/.firefly-bootstrap/inputs.env — the host is a Phase 0 answer, not a secret, +# so bootstrap never writes it to state.env. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STATE="${FIREFLY_STATE_ENV:-$ROOT/.firefly-bootstrap/state.env}" +INPUTS="${FIREFLY_INPUTS_ENV:-$HOME/.firefly-bootstrap/inputs.env}" +OPEN_IT=0 +APP="" +HOST="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --state) STATE="$2"; shift 2 ;; + --app) APP="$2"; shift 2 ;; + --host) HOST="$2"; shift 2 ;; + --open) OPEN_IT=1; shift ;; + -h|--help) sed -n '2,18p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +if [[ ! -f "$STATE" ]]; then + echo "ERROR: no state file at $STATE" >&2 + echo " Run bootstrap first, or pass --state ." >&2 + exit 1 +fi +# inputs.env first so state.env wins on any key both define. +# shellcheck disable=SC1090 +[[ -f "$INPUTS" ]] && { set -a; source "$INPUTS"; set +a; } +# shellcheck disable=SC1090 +set -a; source "$STATE"; set +a + +APP="${APP:-${PREVIEW_URL:-}}" +[[ -n "$APP" ]] || { echo "ERROR: PREVIEW_URL not in $STATE (pass --app)" >&2; exit 1; } + +# The workspace URL is what the app builds its OAuth token endpoint from, and it +# must never default to $APP. Doing so made the app POST client_credentials to +# /oidc/v1/token, which answers with the app's own 404 HTML page — +# surfacing in the browser as "Failed to obtain Databricks OAuth token" with a +# page of HTML as the detail, pointing nowhere near the real cause. +HOST="${HOST:-${DATABRICKS_HOST:-}}" +HOST="${HOST%/}" +if [[ -z "$HOST" ]]; then + echo "ERROR: DATABRICKS_HOST is not set." >&2 + echo " Looked in $INPUTS and $STATE." >&2 + echo " Pass it explicitly: --host https://.cloud.databricks.com" >&2 + exit 1 +fi +if [[ "$HOST" == "${APP%/}" ]]; then + echo "ERROR: DATABRICKS_HOST equals the app origin ($APP)." >&2 + echo " That mints a guest record whose OAuth token endpoint is the app" >&2 + echo " itself, and every guest sign-in then fails with" >&2 + echo " \"Failed to obtain Databricks OAuth token\"." >&2 + exit 1 +fi + +for v in GUEST_API_SECRET GUEST_SP_CLIENT_ID GUEST_SP_SECRET; do + [[ -n "${!v:-}" ]] || { echo "ERROR: $v missing from $STATE" >&2; exit 1; } +done + +# A 128-char secret is a real `openssl rand -hex 64`. Anything short is the +# redacted placeholder `vercel env pull` hands back, which 401s every call. +if [[ ${#GUEST_API_SECRET} -ne 128 ]]; then + echo "ERROR: GUEST_API_SECRET is ${#GUEST_API_SECRET} chars, expected 128." >&2 + echo " That is the redacted placeholder, not the secret. See Phase 9." >&2 + exit 1 +fi + +post() { # path json + curl -sS -X POST "$APP$1" \ + -H "X-API-Key: $GUEST_API_SECRET" \ + -H "Content-Type: application/json" \ + -d "$2" +} + +die_with() { echo "ERROR: $1" >&2; [[ -n "${2:-}" ]] && echo " response: ${2:0:300}" >&2; exit 1; } + +WS_RESP="$(post /api/guest/workspaces \ + "{\"name\":\"Guest Access\",\"workspaceUrl\":\"$HOST\"}")" +WS_ID="$(printf '%s' "$WS_RESP" \ + | python3 -c "import sys,json;print(json.load(sys.stdin)['workspace']['id'])" 2>/dev/null)" +[[ -n "$WS_ID" ]] || die_with "could not create a guest workspace record" "$WS_RESP" + +SPN_RESP="$(post /api/guest/spns \ + "{\"name\":\"Guest SPN\",\"clientId\":\"$GUEST_SP_CLIENT_ID\",\"clientSecret\":\"$GUEST_SP_SECRET\",\"guestWorkspaceId\":\"$WS_ID\"}")" +SPN_ID="$(printf '%s' "$SPN_RESP" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print((d.get('spn') or d).get('id',''))" 2>/dev/null)" +[[ -n "$SPN_ID" ]] || die_with "could not attach the guest service principal" "$SPN_RESP" + +GU_RESP="$(post /api/guest/users "{\"orgName\":\"Guest Org\",\"spnId\":\"$SPN_ID\"}")" +read -r EMAIL URL < <(printf '%s' "$GU_RESP" | python3 -c " +import sys,json +d=json.load(sys.stdin); g=d.get('guestUser',d) +print(g.get('email',''), g.get('loginUrl','')) +" 2>/dev/null) +[[ -n "${URL:-}" ]] || die_with "no loginUrl returned" "$GU_RESP" + +echo "guest user: $EMAIL" +echo "" +echo "$URL" +echo "" +echo "Single use, valid ~10 minutes. Open it ONCE - a refresh or back-navigation" +echo "invalidates it. Re-run this script for another." + +[[ "$OPEN_IT" == "1" ]] && { command -v open >/dev/null && open "$URL"; } +exit 0 diff --git a/scripts/test-bootstrap-phase1a.sh b/scripts/test-bootstrap-phase1a.sh new file mode 100755 index 0000000..d2739da --- /dev/null +++ b/scripts/test-bootstrap-phase1a.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Hermetic regression test for bootstrap Phase 1a on a FRESH $HOME. +# +# Adapted from scripts/test-bootstrap-corepack.sh on branch fix/corepack-mkdir-home-bin +# (PR #67) — the sandbox/stub scaffolding and the fresh-HOME approach are that author's. +# Reframed from mechanism to OUTCOME so it stays valid regardless of how pnpm is installed: +# +# #67 asserted "corepack created a pnpm shim in $HOME/bin" (mechanism) +# this asserts "$HOME/bin exists AND pnpm is on the pin" (outcome) +# +# It therefore covers BOTH reported bugs at once: +# #67 — the script assumed $HOME/bin existed and broke when it did not +# (`corepack enable --install-directory` errors ENOENT and does NOT mkdir; +# verified directly. $HOME/bin is on PATH from Phase 0 and is written to by the +# gh/databricks/uv installers, so it must exist early regardless of pnpm's +# install method). +# #69 — corepack must not be used to fetch pnpm: it ignores the npm registry setting +# and hits registry.npmjs.org, which fails wherever public npm is blocked. +# +# Scope: runs against stubbed network/auth commands on the local machine, so it covers +# fresh-HOME behaviour and the version pin. It does NOT cover the blocked-registry case — +# CI runners have unrestricted npm access. That path needs a corp-network VM. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BOOTSTRAP="$SCRIPT_DIR/bootstrap.sh" +PIN="$(sed -nE 's/.*PNPM_VERSION:=([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' "$SCRIPT_DIR/lib/corp-network.sh" | head -1)" + +[[ -x "$BOOTSTRAP" ]] || { echo "bootstrap.sh not executable: $BOOTSTRAP" >&2; exit 1; } +[[ -n "$PIN" ]] || { echo "could not read PNPM_VERSION from lib/corp-network.sh" >&2; exit 1; } + +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT +TEST_HOME="$TEST_ROOT/home" +STUB_BIN="$TEST_ROOT/stubs" +NPM_PREFIX="$TEST_ROOT/npm-global" +CALLS="$TEST_ROOT/calls.log" +mkdir -p "$TEST_HOME" "$STUB_BIN" +: > "$CALLS" + +write_stub() { + local name="$1"; shift + { echo '#!/usr/bin/env bash'; printf '%s\n' "$@"; } > "$STUB_BIN/$name" + chmod +x "$STUB_BIN/$name" +} + +# npm: records what bootstrap asked for, and on a global install actually materializes a +# pnpm binary reporting the pinned version — so the assertion below is a real outcome check +# rather than a spy on the command string. +write_stub npm ' +echo "npm $*" >> "$CALLS" +case "$*" in + "config get registry") echo "https://registry.npmjs.org/" ;; + "config get prefix") echo "$NPM_PREFIX" ;; + view*) echo "'"$PIN"'" ;; # preflight resolve probe + "install -g pnpm@'"$PIN"'") + mkdir -p "$NPM_PREFIX/bin" + printf "#!/usr/bin/env bash\necho '"$PIN"'\n" > "$NPM_PREFIX/bin/pnpm" + chmod +x "$NPM_PREFIX/bin/pnpm" + ;; +esac +exit 0' + +# corepack: never delegated to the real binary. Any enable/prepare is an ENV-0 violation, +# so just record the invocation and let the assertions decide. +write_stub corepack ' +echo "corepack $*" >> "$CALLS" +exit 0' + +# Phase 0 probes: a successful curl means "no intercepting proxy", keeping the run offline. +write_stub curl 'exit 0' + +# Phase 1 auth/tooling are deliberate no-ops — the target is bootstrap control flow, not +# external OAuth or service availability. +write_stub vercel ' +case "${1:-}" in --version) echo "vercel-test" ;; whoami) echo "test-user" ;; esac +exit 0' +write_stub gh ' +case "${1:-}" in --version) echo "gh version test" ;; auth) exit 0 ;; esac +exit 0' +write_stub neonctl ' +case "${1:-}" in --version) echo "neonctl-test" ;; me) echo "test-user" ;; esac +exit 0' +write_stub databricks ' +case "${1:-}" in --version) echo "Databricks CLI test" ;; auth|workspace) exit 0 ;; esac +exit 0' +write_stub uv ' +[[ "${1:-}" == "--version" ]] && echo "uv-test" +exit 0' + +[[ ! -e "$TEST_HOME/bin" ]] || { + echo "test fixture invalid: $TEST_HOME/bin already exists" >&2; exit 1; } + +# Phase 0 answers, in prompt order. An empty string accepts the prompt's default; the three +# non-empty entries are the prompts that have no default and re-ask until answered. Kept as +# an explicit array rather than a heredoc — blank lines are significant here, and a heredoc +# silently shifts every subsequent answer if one is lost to whitespace trimming. +ANSWERS=( + "https://example.cloud.databricks.com" # DATABRICKS_HOST (required) + "" # DB_PROFILE + "" # UC_CATALOG + "" # UC_SCHEMA + "" # SEED_SAMPLE_DATA (default yes) + "" # GENIE_SPACE_IDS (default None) + "" # CREATE_GENIE_SPACE (only asked when GENIE_SPACE_IDS is None) + "" # AGENT_APP_NAME + "" # LAKEBASE_NAME + "00000000-0000-0000-0000-000000000000" # DATABRICKS_ACCOUNT_ID (required) + "" # REPO_DIR + "test-team" # VERCEL_TEAM (required) + "" # NEON_PROJECT_NAME + "" # VERCEL_PROJECT +) + +OUTPUT="$TEST_ROOT/bootstrap.out" +if ! printf '%s\n' "${ANSWERS[@]}" | env \ + HOME="$TEST_HOME" \ + PATH="$STUB_BIN:/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin" \ + NPM_PREFIX="$NPM_PREFIX" \ + CALLS="$CALLS" \ + bash "$BOOTSTRAP" --stop-after=1 >"$OUTPUT" 2>&1 +then + echo "FAIL: bootstrap exited before completing Phase 1" >&2 + echo "---- bootstrap output ----" >&2; cat "$OUTPUT" >&2 + exit 1 +fi + +PASS=0; FAIL=0 +pass() { echo "PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo "FAIL: $1" >&2; FAIL=$((FAIL + 1)); } +dump() { echo "---- bootstrap output ----" >&2; cat "$OUTPUT" >&2; + echo "---- recorded calls ----" >&2; cat "$CALLS" >&2; } + +# #67: the script must not assume $HOME/bin exists. +if [[ -d "$TEST_HOME/bin" ]]; then + pass "fresh HOME: bootstrap created \$HOME/bin (#67)" +else + fail "bootstrap did not create \$HOME/bin on a fresh HOME (#67)" +fi + +# #69: pnpm must not be fetched through corepack. +if grep -qE '^corepack (enable|prepare)' "$CALLS"; then + fail "bootstrap invoked 'corepack enable/prepare' — blocked on corporate networks (ENV-0, #69)" + grep -E '^corepack' "$CALLS" | sed 's/^/ /' >&2 +else + pass "pnpm was not fetched via corepack (ENV-0, #69)" +fi + +# Outcome: pnpm installed, at the pin, from the user's configured registry. +if grep -qF "npm install -g pnpm@$PIN" "$CALLS"; then + pass "pnpm installed via npm at the pinned version ($PIN)" +else + fail "expected 'npm install -g pnpm@$PIN'" +fi + +if [[ -x "$NPM_PREFIX/bin/pnpm" ]] && [[ "$("$NPM_PREFIX/bin/pnpm" --version)" == "$PIN" ]]; then + pass "resulting pnpm reports the pinned version ($PIN)" +else + fail "pnpm binary missing or wrong version (expected $PIN)" +fi + +if grep -q "Stopped after Phase 1" "$OUTPUT"; then + pass "bootstrap completed Phase 1" +else + fail "bootstrap did not complete Phase 1" +fi + +echo +echo "Results: $PASS passed, $FAIL failed" +[[ "$FAIL" -eq 0 ]] || { dump; exit 1; } diff --git a/scripts/test-runbook-lib.sh b/scripts/test-runbook-lib.sh new file mode 100755 index 0000000..43f1f94 --- /dev/null +++ b/scripts/test-runbook-lib.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# test-runbook-lib.sh — hermetic tests for scripts/lib/runbook.sh. +# +# bash scripts/test-runbook-lib.sh +# +# No network, no cloud, no $HOME writes. Every case below is a defect observed on the +# 2026-07-25 fresh-install run; each assertion is the thing that would have caught it. +# +# The state helpers are exercised under BOTH bash and zsh on purpose. BOOTSTRAP.md tells +# the reader to source this library from their own shell, zsh is the macOS default, and +# the previous implementation used ${!key} — bash-only indirect expansion, which produced +# "(eval):1: bad substitution" on a real run and left DATABASE_URL empty. + +set -uo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +LIB="scripts/lib/runbook.sh" +FAILED=0 +pass() { echo " ✓ $*"; } +bad() { echo " ✗ $*"; FAILED=1; } + +echo "== scripts/lib/runbook.sh ==" + +# ── state.env round-trip, under every shell the runbook supports ───────────── +for sh in bash zsh; do + command -v "$sh" >/dev/null 2>&1 || { pass "$sh not installed — skipped."; continue; } + if out=$("$sh" -c ' + set -u + source '"$LIB"' + REPO_DIR=$(mktemp -d); export REPO_DIR + # Same shell metacharacters a real connection string carries (: / @ ? & and a + # space), without being shaped like one — secret scanners flag the real shape. + secret="scheme://user:pw@host/path?flag=yes&opt=a b" + store_secret DATABASE_URL "$secret" >/dev/null + got=$(read_secret DATABASE_URL) || { echo "read failed"; exit 1; } + [ "$got" = "$secret" ] || { echo "round-trip mismatch: [$got]"; exit 1; } + read_secret ABSENT >/dev/null 2>&1 && { echo "absent key should fail"; exit 1; } + require_secret OUT DATABASE_URL || { echo "require_secret failed"; exit 1; } + [ "$OUT" = "$secret" ] || { echo "require_secret wrong value"; exit 1; } + require_secret X ABSENT >/dev/null 2>&1 && { echo "require_secret should fail on absent"; exit 1; } + [ "$(stat -f %Lp "$REPO_DIR/.firefly-bootstrap/state.env")" = "600" ] \ + || { echo "state.env not 0600"; exit 1; } + rm -rf "$REPO_DIR"; echo ok + ' 2>&1) && [[ "$out" == *ok* ]]; then + pass "state.env round-trip under $sh (metachars, absent keys, 0600)." + else + bad "state.env round-trip failed under $sh: $out" + fi +done + +# shellcheck source=scripts/lib/runbook.sh +source "$LIB" + +# ── parsers ────────────────────────────────────────────────────────────────── +# `neonctl projects create --output json` nests the id under "project". The runbook +# read a top-level 'id' for ten days after bootstrap.sh was fixed, because they were +# separate copies; the create then succeeded server-side while the parse failed, +# orphaning a project. +got=$(echo '{"project":{"id":"cold-flower-29604204","name":"firefly-genie"}}' | extract_neon_project_id) +[[ "$got" == "cold-flower-29604204" ]] \ + && pass "extract_neon_project_id reads the nested project.id." \ + || bad "extract_neon_project_id nested shape: got [$got]" + +got=$(echo '{"id":"legacy-flat-123"}' | extract_neon_project_id) +[[ "$got" == "legacy-flat-123" ]] \ + && pass "extract_neon_project_id falls back to a flat id." \ + || bad "extract_neon_project_id flat shape: got [$got]" + +got=$(printf 'Vercel CLI 56.3.1\nInspect: https://vercel.com/x/y\nProduction: https://firefly-genie-a1b2c3.vercel.app [2s]\n' \ + | extract_vercel_preview_url) +[[ "$got" == "https://firefly-genie-a1b2c3.vercel.app" ]] \ + && pass "extract_vercel_preview_url strips CLI chatter." \ + || bad "extract_vercel_preview_url: got [$got]" + +# ── bundle assertions ──────────────────────────────────────────────────────── +# The committed bundle carries the authoring workspace's experiment id. Deploying it +# unrewritten returns "Node ID does not exist (404)" and names nothing useful. +if assert_bundle_quickstart_ran agent/databricks.yml >/dev/null 2>&1; then + bad "assert_bundle_quickstart_ran passed on the committed placeholder — it must fail." +else + pass "assert_bundle_quickstart_ran rejects the committed placeholder." +fi + +TMP=$(mktemp -d) +sed "s/${FIREFLY_PLACEHOLDER_EXPERIMENT_ID}/987654321/" agent/databricks.yml > "$TMP/rewritten.yml" +if assert_bundle_quickstart_ran "$TMP/rewritten.yml" >/dev/null 2>&1; then + pass "assert_bundle_quickstart_ran accepts a quickstart-rewritten bundle." +else + bad "assert_bundle_quickstart_ran rejected a rewritten bundle." +fi + +# ── sync.exclude rules ─────────────────────────────────────────────────────── +# The exclude list opens with a comment mentioning pyproject.toml and vendor-wheels/. +# A bare grep therefore reports both as excluded (bootstrap.sh did, and failed a correct +# config); a naive '-\s' scan reads the list as empty and passes anything (an agent did). +if check_sync_exclude_rules agent/databricks.yml >/dev/null 2>&1; then + pass "check_sync_exclude_rules accepts the correct committed config." +else + bad "check_sync_exclude_rules rejected the correct committed config." +fi + +sed 's|^ - uv.lock$| - uv.lock\n - pyproject.toml|' agent/databricks.yml > "$TMP/no-pyproject.yml" +sed '/^ - uv.lock$/d' agent/databricks.yml > "$TMP/no-lock.yml" +sed 's|^ - uv.lock$| - uv.lock\n - vendor-wheels/**|' agent/databricks.yml > "$TMP/no-wheels.yml" +for case_ in no-pyproject:pyproject.toml no-lock:uv.lock no-wheels:vendor-wheels; do + f="${case_%%:*}"; what="${case_##*:}" + if check_sync_exclude_rules "$TMP/$f.yml" >/dev/null 2>&1; then + bad "check_sync_exclude_rules passed a config that breaks $what." + else + pass "check_sync_exclude_rules catches a broken $what rule." + fi +done +rm -rf "$TMP" + +# ── index preflight ────────────────────────────────────────────────────────── +# Must resolve an index without contacting one; the reachability probe itself needs +# network and is exercised by the E2E harness, not here. +# shellcheck source=scripts/lib/corp-network.sh +source scripts/lib/corp-network.sh +got=$(UV_DEFAULT_INDEX="https://example.invalid/simple" firefly_effective_pypi_index) +[[ "$got" == "https://example.invalid/simple" ]] \ + && pass "firefly_effective_pypi_index prefers UV_DEFAULT_INDEX." \ + || bad "firefly_effective_pypi_index UV_DEFAULT_INDEX: got [$got]" + +got=$(unset UV_DEFAULT_INDEX UV_INDEX_URL; HOME=/nonexistent firefly_effective_pypi_index) +[[ "$got" == "https://pypi.org/simple/" ]] \ + && pass "firefly_effective_pypi_index falls back to public PyPI when unconfigured." \ + || bad "firefly_effective_pypi_index default: got [$got]" + +echo +if [[ "$FAILED" -eq 0 ]]; then + echo "All runbook-lib tests pass." +else + echo "One or more runbook-lib tests FAILED (see ✗ above)." >&2 +fi +exit "$FAILED" diff --git a/src/app/api/agent-proxy/[[...path]]/route.ts b/src/app/api/agent-proxy/[[...path]]/route.ts new file mode 100644 index 0000000..b01f74c --- /dev/null +++ b/src/app/api/agent-proxy/[[...path]]/route.ts @@ -0,0 +1,232 @@ +import { NextRequest, NextResponse } from "next/server"; +import { headers } from "next/headers"; +import { eq } from "drizzle-orm"; +import { getAuthInstance } from "@/lib/auth-dynamic"; +import { getDatabricksSpnToken } from "@/lib/databricks-spn-authtoken"; +import { db } from "@/db"; +import { organization } from "@/db/schema"; + +// Vercel-native reverse proxy for the managed-memory agent Databricks App. +// +// Mirrors what the stock Go proxy does (inject a bearer, strip forwarded +// headers, relax frame headers) but resolves the token from the *current +// user's* mapped SPN (guest / BYOD supported) via getDatabricksSpnToken, so a +// guest never sees the Databricks OAuth wall. The agent iframe loads +// /api/agent-proxy/ (same origin) and every asset/api request is caught by this +// catch-all and forwarded to the App with the injected token. +// +// The agent chat UI (built with base:"./" + the fetch shim) emits relative +// asset URLs and prefixes /api/ calls with /api/agent-proxy, so both static +// assets and the streaming /api/chat SSE route back through here. + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; +// Chat responses stream via SSE; allow a long-running function. +export const maxDuration = 300; + +const AGENT_APP_URL = (process.env.DATABRICKS_AGENT_APP_URL ?? "").replace( + /\/$/, + "", +); + +// The path this proxy is mounted at (matches the [[...path]] route location). +const MOUNT = "/api/agent-proxy"; + +// Request headers we must not forward upstream. +const STRIPPED_REQUEST_HEADERS = new Set([ + "host", + "cookie", + "authorization", + "connection", + "content-length", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-forwarded-port", + "x-real-ip", + // Never let the browser revalidate the HTML document upstream: a 304 would + // return no body, so our per-request HTML rewrites (, force-light) + // could never reach the client and a stale injected copy would stick. + // Assets use hashed URLs with long max-age, so losing their 304s is harmless. + "if-none-match", + "if-modified-since", +]); + +// Response headers we must not pass back (Node fetch already decoded the body, +// and we set our own framing policy). +const STRIPPED_RESPONSE_HEADERS = new Set([ + "content-encoding", + "content-length", + "transfer-encoding", + "connection", + "content-security-policy", + "x-frame-options", +]); + +async function resolveToken(): Promise< + | { ok: true; token: string } + | { ok: false; status: number; body: unknown } +> { + const auth = await getAuthInstance(); + const session = await auth.api.getSession({ headers: await headers() }); + + const activeOrgId = session?.session?.activeOrganizationId; + if (!session || !activeOrgId) { + return { + ok: false, + status: 401, + body: { error: "No active organization in session" }, + }; + } + + const [org] = await db + .select() + .from(organization) + .where(eq(organization.id, activeOrgId)) + .limit(1); + + if (!org?.workspaceUrl) { + return { + ok: false, + status: 400, + body: { error: "No workspace URL configured for this organization" }, + }; + } + + const workspaceUrl = org.workspaceUrl.replace(/\/$/, ""); + const tokenResult = await getDatabricksSpnToken( + workspaceUrl, + undefined, + session.user.email, + activeOrgId, + ); + + if (!tokenResult.success) { + return { + ok: false, + status: tokenResult.error.status, + body: { + error: tokenResult.error.error, + details: tokenResult.error.details, + }, + }; + } + + return { ok: true, token: tokenResult.data.accessToken }; +} + +async function proxy( + req: NextRequest, + context: { params: Promise<{ path?: string[] }> }, +): Promise { + if (!AGENT_APP_URL) { + return NextResponse.json( + { error: "DATABRICKS_AGENT_APP_URL is not configured" }, + { status: 503 }, + ); + } + + const auth = await resolveToken(); + if (!auth.ok) { + return NextResponse.json(auth.body, { status: auth.status }); + } + + const { path } = await context.params; + const suffix = (path ?? []).join("/"); + const search = req.nextUrl.search; + const targetUrl = `${AGENT_APP_URL}/${suffix}${search}`; + + // Copy inbound headers minus the ones we strip, then inject the bearer. + const outboundHeaders = new Headers(); + req.headers.forEach((value, key) => { + if (!STRIPPED_REQUEST_HEADERS.has(key.toLowerCase())) { + outboundHeaders.set(key, value); + } + }); + outboundHeaders.set("Authorization", `Bearer ${auth.token}`); + + const method = req.method.toUpperCase(); + const hasBody = method !== "GET" && method !== "HEAD"; + + const init: RequestInit = { + method, + headers: outboundHeaders, + redirect: "manual", + }; + if (hasBody) { + // Buffer the request body and send it with a fixed Content-Length. Streaming + // it with duplex:"half" produces a chunked request that the Databricks Apps + // front door rejects ("Proxy error:"). Chat POST bodies are small; only the + // RESPONSE needs to stream (SSE), which it still does below. + const bodyBuffer = await req.arrayBuffer(); + if (bodyBuffer.byteLength > 0) { + init.body = bodyBuffer; + } + } + + let upstream: Response; + try { + upstream = await fetch(targetUrl, init); + } catch (err) { + return NextResponse.json( + { error: "Upstream agent app request failed", details: String(err) }, + { status: 502 }, + ); + } + + const responseHeaders = new Headers(); + upstream.headers.forEach((value, key) => { + if (!STRIPPED_RESPONSE_HEADERS.has(key.toLowerCase())) { + responseHeaders.set(key, value); + } + }); + // Same-origin iframe: allow the frontend to embed the proxied app. + responseHeaders.set("X-Frame-Options", "SAMEORIGIN"); + responseHeaders.set("Content-Security-Policy", "frame-ancestors 'self'"); + + // The mount (/api/agent-proxy) has no trailing slash (Next strips it), so the + // app's relative "./assets/..." URLs would resolve against /api/. Inject a + // so every relative URL resolves under the proxy mount. Also force the + // chat UI to light mode: it uses next-themes with defaultTheme="system", which + // follows the OS (dark) and clashes with Firefly's light-only UI. Seeding the + // next-themes storage key + removing the "dark" class before the app bundle + // runs pins it to light with no flash. Only the HTML document is + // buffered+rewritten; assets and the chat SSE stream untouched. + const contentType = upstream.headers.get("content-type") ?? ""; + if (contentType.includes("text/html")) { + const html = await upstream.text(); + const forceLight = + ""; + const withBase = html.replace( + /]*)?>/i, + (m) => `${m}${forceLight}`, + ); + responseHeaders.delete("content-length"); + // The injected HTML is generated per-request; never cache/revalidate it so + // future injections always reach the browser (the app's ETag never changes). + responseHeaders.delete("etag"); + responseHeaders.delete("last-modified"); + responseHeaders.set("Cache-Control", "no-store, must-revalidate"); + return new Response(withBase, { + status: upstream.status, + statusText: upstream.statusText, + headers: responseHeaders, + }); + } + + return new Response(upstream.body, { + status: upstream.status, + statusText: upstream.statusText, + headers: responseHeaders, + }); +} + +export const GET = proxy; +export const POST = proxy; +export const PUT = proxy; +export const PATCH = proxy; +export const DELETE = proxy; +export const HEAD = proxy; +export const OPTIONS = proxy; diff --git a/src/app/api/databricks/unity-catalog/catalogs/route.ts b/src/app/api/databricks/unity-catalog/catalogs/route.ts index efd41a0..a03d3b9 100644 --- a/src/app/api/databricks/unity-catalog/catalogs/route.ts +++ b/src/app/api/databricks/unity-catalog/catalogs/route.ts @@ -8,8 +8,17 @@ import { headers } from "next/headers"; export const dynamic = "force-dynamic"; -// Catalogs that guest users are allowed to see -const GUEST_ALLOWED_CATALOG_PREFIXES = ["firefly"]; +// Catalogs that guest users are allowed to browse in the Catalog Explorer (case-insensitive +// prefix match). Env-configurable so a deployment can align it with the chosen UC catalog +// WITHOUT a source edit — set GUEST_ALLOWED_CATALOG_PREFIXES="firefly,workspace" (comma- +// separated). Defaults to ["firefly"] when unset. NOTE: widening this only affects what +// guests can BROWSE; actual data access is still governed by the guest SP's UC grants. +const GUEST_ALLOWED_CATALOG_PREFIXES = ( + process.env.GUEST_ALLOWED_CATALOG_PREFIXES ?? "firefly" +) + .split(",") + .map((p) => p.trim().toLowerCase()) + .filter(Boolean); // Cache tag for catalog data export const CATALOGS_CACHE_TAG = "UNITY_CATALOG_CATALOGS"; diff --git a/src/app/docs/architecture/authentication/databricks-identity/page.tsx b/src/app/docs/architecture/authentication/databricks-identity/page.tsx index de72ccd..8b6e1bb 100644 --- a/src/app/docs/architecture/authentication/databricks-identity/page.tsx +++ b/src/app/docs/architecture/authentication/databricks-identity/page.tsx @@ -1265,7 +1265,7 @@ export default async function DatabricksIdentityAuthPage() { Analytics. Explore our solutions to see it in action.

Explore Solutions diff --git a/src/app/docs/architecture/lakehouse-apps-proxy/page.tsx b/src/app/docs/architecture/lakehouse-apps-proxy/page.tsx index cbfd79f..c18e3e6 100644 --- a/src/app/docs/architecture/lakehouse-apps-proxy/page.tsx +++ b/src/app/docs/architecture/lakehouse-apps-proxy/page.tsx @@ -6,9 +6,23 @@ import { PageTitle, } from "@/components/docs/section"; import Link from "next/link"; +import { SOLUTION_DOCS } from "@/lib/solution-docs"; +const LAKEHOUSE_HUB_SLUGS = [ + "embedding-apps", + "notebook-editor", + "code-editor", + "agent", + "sql-editor", + "data-catalog", + "pipeline-editor", +] as const; export default function LakehouseAppsProxyPage() { + const hubSolutions = LAKEHOUSE_HUB_SLUGS.map( + (slug) => SOLUTION_DOCS.find((s) => s.slug === slug)! + ); + return (
@@ -21,7 +35,11 @@ export default function LakehouseAppsProxyPage() { The proxy uses a session-cookie architecture — a short-lived JWT is exchanged for an opaque HttpOnly session cookie, so no Databricks tokens ever appear in URLs, browser storage, or logs. Documentation is organized into - dedicated pages below. + dedicated pages below — see also the{" "} + + full solutions index + + .

@@ -56,72 +74,55 @@ export default function LakehouseAppsProxyPage() {
- -

Embedding Databricks Apps w/o SSO

-

- Session-cookie proxy architecture: JWT exchange, SPN token management, - HttpOnly proxy_sid cookies, WebSocket support, and wildcard - domain routing for production. -

- - - -

Notebook Editor

-

- Interactive Python notebooks powered by Marimo with reactive - execution and rich outputs. -

- - - -

Code Editor

-

- VS Code-style development environment with terminal access, - Git integration, and LSP support. -

- - - -

SQL Editor

-

- Native SQL query interface with warehouse integration, - streaming results, and catalog autocomplete. -

- - - -

Data Catalog

-

- Hierarchical Unity Catalog browser with lazy loading, - metadata display, and BYOD support. -

- + {hubSolutions.map((solution) => { + const featured = solution.slug === "embedding-apps"; + const vercelProxy = solution.slug === "agent"; - -

Pipeline Editor

-

- Visual node-based pipeline designer with drag-and-drop - nodes and Delta Live Tables execution. -

- + return ( + +
+

+ {solution.title} +

+ + {solution.embeddingLabel} + +
+

+ {solution.description} + {vercelProxy && ( + <> + {" "} + Uses /api/agent-proxy on the + same origin — no Go proxy required. + + )} + {featured && ( + <> + {" "} + Session-cookie proxy architecture with JWT exchange, SPN token + management, WebSocket support, and wildcard domain routing for + production. + + )} +

+ + ); + })}
diff --git a/src/app/docs/architecture/overview/page.tsx b/src/app/docs/architecture/overview/page.tsx index 87ebc35..913549c 100644 --- a/src/app/docs/architecture/overview/page.tsx +++ b/src/app/docs/architecture/overview/page.tsx @@ -1,6 +1,7 @@ import { promises as fs } from "fs"; import path from "path"; import Link from "next/link"; +import { SOLUTION_DOCS } from "@/lib/solution-docs"; import { MermaidDiagram } from "@/components/mermaid-diagram"; import { Section, @@ -441,6 +442,42 @@ export default async function ArchitectureOverviewPage() {
+ {/* Platform Solutions */} +
+ +

+ FireFly ships seven documented solutions — from Go-proxy iframe editors + to native React components and the Vercel-native Agent Panel. See the{" "} + + solutions index + {" "} + for embedding-pattern details. +

+ + +
+ {SOLUTION_DOCS.map((solution) => ( + +
+

+ {solution.title} +

+ + {solution.embeddingLabel} + +
+

+ {solution.description} +

+ + ))} +
+
+ {/* Architecture Sections Overview */}
diff --git a/src/app/docs/architecture/scalability/page.tsx b/src/app/docs/architecture/scalability/page.tsx index 2934e05..4280dce 100644 --- a/src/app/docs/architecture/scalability/page.tsx +++ b/src/app/docs/architecture/scalability/page.tsx @@ -568,14 +568,28 @@ spec:
-
-
+
+

Code Editor

VSCode-based editor for notebooks, Python, SQL with full IDE features (IntelliSense, debugging, Git).

-
+ + + +

Agent Panel

+

+ Genie Agent + managed-memory chat assistant embedded via the + Vercel-native /api/agent-proxy route. +

+

Notebook Viewer

@@ -593,6 +607,13 @@ spec:

+

+ See the{" "} + + full solutions index + {" "} + for all documented platform capabilities. +

diff --git a/src/app/docs/solutions/agent/page.tsx b/src/app/docs/solutions/agent/page.tsx new file mode 100644 index 0000000..67b493f --- /dev/null +++ b/src/app/docs/solutions/agent/page.tsx @@ -0,0 +1,338 @@ +import { promises as fs } from "fs"; +import path from "path"; +import { MermaidDiagram } from "@/components/mermaid-diagram"; +import { + Section, + SectionContainer, + ContentBlock, + HighlightBox, + CodeBlock, + PageTitle, +} from "@/components/docs/section"; +import Link from "next/link"; + +async function loadMermaidFile(filename: string): Promise { + const filePath = path.join( + process.cwd(), + "public/solutions/agent", + filename + ); + return await fs.readFile(filePath, "utf-8"); +} + +export default async function AgentPage() { + const architecture = await loadMermaidFile("architecture.mermaid"); + + return ( +
+ +
+
+ + Solutions + +
+ Agent Panel +

+ A slide-out chat assistant that answers questions over your + workspace data with Genie Agent and remembers context across sessions + with managed memory — embedded without exposing Databricks SSO. +

+
+ + {/* Overview Section */} +
+ +

+ The Agent panel embeds a Databricks App built from the{" "} + agent-openai-agents-sdk template + (vendored as a git submodule under{" "} + vendor/app-templates). It pairs the + OpenAI Agents SDK with two capabilities: Genie Agent{" "} + for natural-language questions over Unity Catalog data, and{" "} + managed memory for durable, per-user context. +

+

+ Unlike the code and notebook editors, the agent is embedded through + a Vercel-native reverse proxy rather than the Go + proxy — so guests never see a Databricks login. See{" "} + + Backend Configuration + {" "} + for how the proxy mints tokens and forwards requests. +

+ + + +
    +
  • Genie Agent answers over workspace data, with source attribution
    • +
  • Managed long-term memory (UC store) across conversations
    • +
  • Guest / BYOD users work — the proxy mints their mapped SPN token
    • +
  • Same-origin embedding; users never see a Databricks login
    • +
  • No Go proxy or Cloud Run dependency for the agent
    • +
+ +
+ + {/* How It Works Section */} +
+ +

+ When a user opens the panel, the iframe loads{" "} + /api/agent-proxy on the same origin. + The route resolves the session and active organization, mints a + workspace bearer token from the user's mapped SPN, and forwards + to DATABRICKS_AGENT_APP_URL. The + HTML document is rewritten (a <base>{" "} + tag plus a forced light theme) so the chat UI's relative assets + resolve under the mount and match the light Firefly UI. Chat + responses stream back as Server-Sent Events. +

+ + +
+ +
+
+ + {/* Genie Agent + Memory Section */} +
+ +

+ The agent answers data questions with Genie Agent — a + curated Genie space — served over the Genie MCP endpoint + (/api/2.0/mcp/genie/<space_id>). The{" "} + ask_genie tool calls{" "} + genie_ask and polls{" "} + genie_poll_response until complete, + authenticating with the agent App's service principal. Answers are + scoped to that space's curated tables, joins, and instructions, and a + space is the only object a guest service principal can be granted{" "} + CAN_RUN on — which is what makes the + guest flow work at all. +

+

+ Managed memory persists per-user context in a Unity Catalog store, so + the agent can recall earlier facts and preferences across sessions. +

+ + + +
    +
  • Genie-first: data questions call ask_genie before asking the user to clarify
    • +
  • Concrete assets: broad prompts request catalogs, schemas, tables, key columns, and row counts
    • +
  • Attribution: replies surface Genie asset links, and the panel shows plain-text “Powered by Genie” — deliberately not a link, since guests have no workspace access to follow it
    • +
  • Memory: relevant context is read/written to the UC memory store per user
    • +
+ +
+ + {/* Backend Configuration Section */} +
+ +

+ The frontend panel is gated by an env flag and points the proxy at the + deployed agent App. Genie and memory are configured at the{" "} + agent App layer in{" "} + agent/databricks.yml (not the frontend + environment). +

+ + +
+ +

+ Unlike the code and notebook editors, the agent is{" "} + not embedded through the Go proxy. It uses a{" "} + Vercel-native reverse proxy — a Next.js route + at /api/agent-proxy — that + mints the current user's (or guest's){" "} + + SSO-mapped Service Principal + {" "} + token and forwards requests (including the streaming chat) to the + agent App. No Go proxy or Cloud Run is required. +

+ + + +
    +
  • Resolves the session + active organization, then mints a workspace bearer from the user's mapped SPN (guest / BYOD supported)
    • +
  • Injects the bearer and forwards HTTP + SSE (streaming chat) to DATABRICKS_AGENT_APP_URL
    • +
  • Relaxes frame headers for same-origin embedding
    • +
  • Rewrites the HTML document (<base> tag + forced light theme) so relative assets resolve under the mount and match the Firefly UI
    • +
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableWhereDescription
NEXT_PUBLIC_AGENT_ENABLEDFrontendShow the Agent panel when set to true
DATABRICKS_AGENT_APP_URLFrontendDeployed agent App URL the proxy forwards to
GENIE_MCP_MODEAgent Appspace — the default, and the supported configuration
GENIE_SPACE_IDAgent AppGenie space the agent answers from. Required — the app refuses to start without it rather than answering from a different backend
DATABRICKS_MEMORY_STOREAgent AppFully-qualified name (catalog.schema.name) of the UC memory store backing durable cross-session memory. You must create this UC securable and grant the app SP READ/WRITE_MEMORY_STORE after deploy (scripts/setup_memory_store.py) — it is not created by quickstart or the bundle, and is not the Lakebase instance. Without it, memory silently no-ops.
+
+
+ +
+ +

+ The deployable app is assembled from the pristine submodule plus the + local overlay in agent/, then + deployed as a Databricks App bundle. +

+ + +{`# Fetch the submodule (first time only) +git submodule update --init + +# Merge vendor submodule + agent/ overlay into ./agent-build (gitignored) +bash scripts/assemble_agent.sh + +cd agent-build + +# Pre-vendor linux/cp311 wheels so the Apps build installs offline (UV_FIND_LINKS) +# instead of depending on the build container's flaky PyPI egress. +bash scripts/vendor_wheels.sh + +# Validate, then deploy. deploy_agent.sh runs deploy -> run -> enable-memory: +# it deploys + builds + starts (the frontend's first-build Drizzle migrations +# succeed on their own — the bundle's Postgres resource binding is +# CAN_CONNECT_AND_CREATE, so no manual Lakebase grant is needed), then creates the +# UC memory store and grants the app SP READ/WRITE on it (the one post-deploy step +# that durable memory actually requires). On a non-default catalog, pass +# --var catalog=main (it forwards to bundle). +databricks bundle validate -p +bash scripts/deploy_agent.sh + +# Then point DATABRICKS_AGENT_APP_URL at the deployed app URL`} + + +

+ The project README.md (“Build & + deploy the agent app”) is the canonical procedure and covers the + operational notes: the first request returns 503 while the + container builds (confirm readiness from the runtime logs — + Both frontend and backend are ready! — or + the live app_status.state, since the deployment status goes + green when the container command starts, before the port binds), the two Python + versions in play (local quickstart needs + 3.12; the Apps runtime is cp311, which is why wheels are vendored for + 3.11), and the local agent-build git + boundary that assemble_agent.sh creates so + the bundle syncs files. +

+ +
+
+ + {/* Related Documentation Section */} + + +
+ ); +} diff --git a/src/app/docs/solutions/code-editor/page.tsx b/src/app/docs/solutions/code-editor/page.tsx index ccdf4cf..00e7ddd 100644 --- a/src/app/docs/solutions/code-editor/page.tsx +++ b/src/app/docs/solutions/code-editor/page.tsx @@ -28,7 +28,9 @@ export default async function CodeEditorPage() {
- Solutions + + Solutions +
Code Editor

@@ -324,6 +326,16 @@ export default async function CodeEditorIframe() {

+ +

Agent Panel

+

+ Genie + managed-memory chat alongside your editors +

+ +
- Solutions + + Solutions +
Data Catalog

@@ -389,6 +391,16 @@ const validateCatalogs = async (orgId: string) => {

+ +

Agent Panel

+

+ Natural-language queries over catalog data via Genie Agent +

+ + { const filePath = path.join( @@ -32,7 +37,9 @@ export default async function EmbeddingAppsPage() {
- Solutions + + Solutions +
Embedding Databricks Apps w/o SSO

@@ -78,30 +85,86 @@ export default async function EmbeddingAppsPage() {

- This proxy architecture powers several embedded applications: + FireFly uses three embedding patterns. This page documents the{" "} + Go proxy iframe path; see the{" "} + + solutions index + {" "} + for the full catalog.

-
- -

Notebook Editor

-

- Interactive Python notebooks powered by Marimo -

- + +

Go proxy iframe apps

+

+ Notebook and Code Editor embed Databricks Lakehouse Apps through the + Go reverse proxy and ProxyIframe{" "} + session-cookie flow described on this page. +

+ - -

Code Editor

-

- VS Code-style development environment -

- +
+ {GO_PROXY_SOLUTIONS.map((solution) => ( + +

{solution.title}

+

+ {solution.description} +

+ + ))} +
+ + +

Vercel-native proxy iframe

+

+ The Agent Panel uses a same-origin Next.js route at{" "} + /api/agent-proxy instead of the Go + proxy. It mints the user's mapped SPN token and forwards HTTP + SSE + to the deployed agent App. +

+ + +
+ {VERCEL_PROXY_SOLUTIONS.map((solution) => ( + +

{solution.title}

+

+ {solution.description} +

+ + ))} +
+ + +

Native React components

+

+ SQL Editor, Data Catalog, and Pipeline Editor are native components + that call Databricks APIs through Next.js API routes — no iframe + embedding required. +

+ + +
+ {NATIVE_SOLUTIONS.map((solution) => ( + +

{solution.title}

+

+ {solution.description} +

+ + ))}
@@ -898,6 +961,46 @@ CMD ["/proxy"]`}

+ +

Agent Panel

+

+ Genie + managed-memory chat via the Vercel-native proxy +

+ + + +

SQL Editor

+

+ Native SQL query interface with warehouse integration +

+ + + +

Data Catalog

+

+ Unity Catalog browser with BYOD support +

+ + + +

Pipeline Editor

+

+ Visual pipeline design with DLT integration +

+ +
- Solutions + + Solutions +
Notebook Editor

@@ -378,6 +380,16 @@ export default async function NotebookEditorIframe() {

+ +

Agent Panel

+

+ Chat assistant for natural-language data questions +

+ + +
+

+ {solution.title} +

+ + {solution.embeddingLabel} + +
+

+ {solution.description} +

+ + ); +} + +export default function SolutionsIndexPage() { + const overview = SOLUTION_DOCS.find((s) => s.slug === "embedding-apps")!; + + return ( +
+ +
+
Solutions
+ Platform Solutions +

+ FireFly ships seven documented solutions — three embedding patterns + (Go proxy iframe, Vercel-native proxy iframe, and native React) — all + sharing SSO-mapped SPN authentication so guests never see a Databricks + login. +

+
+ +
+ +

+ Start with the proxy architecture overview, then drill into the + individual solution that matches your use case. +

+ + +
+ +
+ +

+ Notebook and Code Editor embed Databricks Lakehouse Apps through the + Go reverse proxy and ProxyIframe{" "} + session-cookie flow. +

+ +
+ {GO_PROXY_SOLUTIONS.map((solution) => ( + + ))} +
+
+ +
+ +

+ The Agent Panel uses a same-origin Next.js route at{" "} + /api/agent-proxy — no Go proxy or + Cloud Run dependency. The route mints the user's mapped SPN token + and forwards HTTP + SSE to the deployed agent App. +

+ +
+ {VERCEL_PROXY_SOLUTIONS.map((solution) => ( + + ))} +
+
+ +
+ +

+ SQL Editor, Data Catalog, and Pipeline Editor are implemented as + native React components that call Databricks APIs through Next.js API + routes — no iframe embedding required. +

+ +
+ {NATIVE_SOLUTIONS.map((solution) => ( + + ))} +
+
+ +
+ +

+ For authentication, request flow, and production deployment guidance, + see the architecture docs. +

+
+ + Apps Proxy hub + + + Architecture overview + + + SSO-Mapped SPN + +
+ +
+ +
+ ); +} diff --git a/src/app/docs/solutions/pipeline-editor/page.tsx b/src/app/docs/solutions/pipeline-editor/page.tsx index 6b361a1..f236081 100644 --- a/src/app/docs/solutions/pipeline-editor/page.tsx +++ b/src/app/docs/solutions/pipeline-editor/page.tsx @@ -28,7 +28,9 @@ export default async function PipelineEditorPage() {
- Solutions + + Solutions +
Pipeline Editor

@@ -472,6 +474,16 @@ def \${node.id.replace('-', '_')}():

+ +

Agent Panel

+

+ Genie Agent chat assistant for pipeline and catalog questions +

+ +
- Solutions + + Solutions +
SQL Editor

@@ -401,6 +403,16 @@ function WarehouseSelector({ onSelect }) {

+ +

Agent Panel

+

+ Ask questions over workspace data with Genie Agent +

+ + { const token = searchParams.get("token"); @@ -27,10 +30,12 @@ function GuestLoginContent() { if (token) { // One-time token login (preferred, secure) + setAttemptedLink(true); setStatus("auto-login"); verifyOneTimeToken(token); } else if (paramEmail && paramPassword) { // Legacy: email/password login + setAttemptedLink(true); setStatus("auto-login"); doEmailLogin( decodeURIComponent(paramEmail), @@ -121,7 +126,39 @@ function GuestLoginContent() { ); } - // Manual login form (shown when no URL params or after error) + // A shared login link failed — show a clear message instead of a manual email/password + // form the guest has no credentials for. "Invalid token" = the one-time link was already + // used (opened/refreshed twice) or never existed; "expired" = older than its 10-min TTL. + if (status === "error" && attemptedLink) { + const expired = /expire/i.test(error || ""); + return ( +
+ + + This login link can’t be used + + +

+ {expired + ? "This guest login link has expired." + : "This guest login link is invalid or has already been used."} +

+

+ Guest login links are single-use and expire{" "} + 10 minutes after they’re created — opening one twice (a + refresh or back navigation counts) also invalidates it. Please ask whoever + shared it to send you a fresh link. +

+ {error && ( +

Details: {error}

+ )} + + +
+ ); + } + + // Manual login form (shown when no URL params, or a manual-form submit failed) return (
diff --git a/src/app/page.tsx b/src/app/page.tsx index c14d651..4487bb3 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -14,6 +14,7 @@ import { GitBranch, Lock, Upload, + Bot, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; @@ -32,11 +33,31 @@ const FEATURE_TABS = [ "Extension support for Python, SQL, and more", ], icon: Code, + docsHref: "/docs/solutions/code-editor", visual: { title: "Development Environment", items: ["VS Code Server", "Terminal Access", "Git Integration", "Extensions"], }, }, + { + id: "agent-panel", + label: "Agent Panel", + title: "Genie + Managed-Memory Chat", + description: + "A slide-out assistant that answers natural-language questions over workspace data and remembers context across sessions.", + features: [ + "Genie Agent queries with source attribution", + "Managed long-term memory per user", + "Embedded via Vercel-native SPN proxy", + "Guest and BYOD users supported", + ], + icon: Bot, + docsHref: "/docs/solutions/agent", + visual: { + title: "Agent Panel", + items: ["Genie Agent", "Managed Memory", "Chat UI", "SPN Proxy"], + }, + }, { id: "drag-drop-etl", label: "Drag & Drop ETL", @@ -50,6 +71,7 @@ const FEATURE_TABS = [ "Automatic code generation", ], icon: GitBranch, + docsHref: "/docs/solutions/pipeline-editor", visual: { title: "Pipeline Studio", items: ["Source Connectors", "Transformations", "Data Preview", "Auto-Deploy"], @@ -68,6 +90,7 @@ const FEATURE_TABS = [ "Data masking and encryption", ], icon: Lock, + docsHref: "/docs/architecture/security", visual: { title: "Security Controls", items: ["RBAC", "Audit Logs", "Data Masking", "Encryption"], @@ -86,6 +109,7 @@ const FEATURE_TABS = [ "REST API data ingestion", ], icon: Upload, + docsHref: "/docs/solutions/data-catalog", visual: { title: "Data Sources", items: ["File Upload", "Cloud Storage", "Databases", "REST APIs"], @@ -127,7 +151,7 @@ export default function Home() { Get Started
@@ -253,9 +277,16 @@ export default function Home() { ))} - +
+ {activeFeature.docsHref && ( + + )} + +
{/* Right side - Visual card */} @@ -328,6 +359,13 @@ export default function Home() { +