diff --git a/.claude/commands/runtests.md b/.claude/commands/runtests.md new file mode 100644 index 00000000000..ed4201619d9 --- /dev/null +++ b/.claude/commands/runtests.md @@ -0,0 +1,48 @@ +--- +description: Run the project's automated checks (TypeScript, mypy, pytest if scoped) and summarise failures. +allowed-tools: Bash, Read +--- + +Run the standard validation suite for whatever scope the user gave (default: both backend + frontend). Output a tight summary with pass/fail counts and the first error per failing tool — don't paste full logs back unless the user asks. + +# Steps + +1. **Decide scope from the user's message.** + - If they wrote `/runtests backend` → only backend checks. + - If `/runtests web` or `/runtests frontend` → only frontend checks. + - If `/runtests ` → only checks scoped to that path (mypy on Python files, tsc on TS files, pytest on tests dir). + - Otherwise → run both backend and frontend. + +2. **Backend checks (run sequentially, stop reporting on each):** + - `cd backend && python -m mypy . 2>&1 | tail -50` — mypy across the whole backend. + - `cd backend && pytest -x --tb=short 2>&1 | tail -50` — fast-fail pytest. Most modules don't have tests, so a "no tests collected" result is normal; that's not a failure. + - `cd backend && alembic heads 2>&1` — confirm single head (multiple heads means a migration was added on a parallel branch and needs merging). + +3. **Frontend checks:** + - `cd web && npx tsc --noEmit --incremental 2>&1 | tail -40` — TypeScript across `web/src/`. + - `cd web && npm run lint 2>&1 | tail -40` — ESLint. + - `cd web && npx prettier --check "src/**/*.{ts,tsx,js,jsx}" 2>&1 | tail -20` — prettier formatting check. + +4. **Skip what's not relevant.** Don't run frontend checks for backend-only changes and vice versa. If you can infer from `git diff --stat` that only `backend/` was touched, skip frontend. + +5. **Summarise**, format like: + + ``` + Backend + mypy ✓ clean (or ✗ N errors — first: ) + pytest ✓ N passed (or ✗ N failed) + alembic ✓ single head + Frontend + tsc ✓ clean (or ✗ N errors — first: ) + lint ✓ clean + prettier ✓ clean + ``` + +6. **If anything failed**, follow up with the user: "X failures above. Want me to fix them, or are you debugging a specific area?" Don't auto-fix unless explicitly told to. + +# Notes + +- Everything in this file's command list is in `.claude/settings.json`'s allow-list, so none of it should prompt for permission. +- `pytest` runs against the live local stack (Postgres / Vespa / model server). If they're not running, integration-style tests will fail — flag that as "stack not running" rather than as test failures. +- Don't run `npx playwright test` from this command. E2E is too slow for an inner-loop check; use it on demand only. +- Don't run `pre-commit run --all-files` from here either — that's slow and overlaps with the per-tool checks above. The user can `pre-commit run` themselves before commit. diff --git a/.claude/hooks/check_changed.sh b/.claude/hooks/check_changed.sh new file mode 100755 index 00000000000..d29018c5dd8 --- /dev/null +++ b/.claude/hooks/check_changed.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Stop hook: validates whatever Claude touched in the current working tree. +# Runs once per Claude response (not per Edit), so a multi-file task +# triggers a single type-check pass rather than N redundant ones. +# +# Scope: +# - If any web/**/*.ts(x) is dirty -> run `tsc --noEmit --incremental` +# across the whole web project (file-scoped tsc would need its own +# tsconfig; full project + incremental cache is the right balance). +# - If any backend/**/*.py is dirty -> run mypy *only on those files* +# (file-scoped mypy is fast). +# +# Output is piped through `tail` so it can't dominate Claude's context. +# Exits 0 even on failure — we want Claude to see the error in its next +# turn, not block the response from completing. +set -u + +PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(git rev-parse --show-toplevel 2>/dev/null)}" +[ -z "$PROJECT_DIR" ] && exit 0 +cd "$PROJECT_DIR" || exit 0 + +# Modified files in the working tree (staged + unstaged + untracked). +# Filename is field 2 in `git status --porcelain` output. We strip the +# "XY " prefix manually so paths with spaces survive. +DIRTY=$(git status --porcelain 2>/dev/null | sed 's/^...//') +[ -z "$DIRTY" ] && exit 0 + +ts_changed=false +py_changed=() + +while IFS= read -r f; do + case "$f" in + web/*.ts|web/*.tsx|web/**/*.ts|web/**/*.tsx) ts_changed=true ;; + backend/*.py|backend/**/*.py) py_changed+=("${f#backend/}") ;; + esac +done <<< "$DIRTY" + +# --- TypeScript ---------------------------------------------------------- +if [ "$ts_changed" = true ] && [ -d "web" ]; then + echo "── tsc (web) ──" + ( cd web && npx --no-install tsc --noEmit --incremental 2>&1 | tail -40 ) || true +fi + +# --- Python (mypy on changed files only) --------------------------------- +# Prefer the project venv's Python (where mypy is installed). Fall back to +# python3 / python on PATH if the venv doesn't exist. +PY="" +if [ -x "$PROJECT_DIR/.venv/bin/python" ]; then + PY="$PROJECT_DIR/.venv/bin/python" +elif command -v python3 >/dev/null 2>&1; then + PY="$(command -v python3)" +elif command -v python >/dev/null 2>&1; then + PY="$(command -v python)" +fi + +if [ "${#py_changed[@]}" -gt 0 ] && [ -d "backend" ] && [ -n "$PY" ]; then + echo "── mypy (changed files) ──" + ( cd backend && "$PY" -m mypy --no-error-summary "${py_changed[@]}" 2>&1 | tail -25 ) || true +fi + +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000000..8182b19ca3a --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "_comment": "Project-shared Claude Code settings. Commit this file. Personal overrides go in .claude/settings.local.json (gitignored). Permissions allow Claude to run validation commands without prompting; the deny list always wins. Hooks auto-run type-checkers after Edit/Write so type regressions surface immediately.", + + "permissions": { + "allow": [ + "Bash(git status:*)", + "Bash(git diff:*)", + "Bash(git log:*)", + "Bash(git show:*)", + "Bash(git branch:*)", + "Bash(git stash list)", + "Bash(git remote -v)", + + "Bash(ls:*)", + "Bash(grep:*)", + "Bash(find:*)", + "Bash(cat:*)", + "Bash(head:*)", + "Bash(tail:*)", + "Bash(wc:*)", + "Bash(tree:*)", + "Bash(file:*)", + + "Bash(ps aux)", + "Bash(ps -ef)", + "Bash(docker ps:*)", + "Bash(docker logs:*)", + "Bash(docker exec * psql:*)", + "Bash(docker network inspect:*)", + "Bash(redis-cli ping)", + "Bash(redis-cli llen:*)", + "Bash(redis-cli info:*)", + + "Bash(python -m mypy:*)", + "Bash(pytest:*)", + "Bash(pre-commit run:*)", + "Bash(black --check:*)", + "Bash(alembic heads)", + "Bash(alembic history:*)", + "Bash(alembic current)", + "Bash(alembic check)", + "Bash(PYTHONPATH=* python -c:*)", + "Bash(PYTHONPATH=* python -m:*)", + + "Bash(npx tsc --noEmit:*)", + "Bash(npx tsc --noEmit --incremental:*)", + "Bash(npm run lint:*)", + "Bash(npm run typecheck:*)", + "Bash(npx prettier --check:*)", + "Bash(npx playwright test --list:*)", + + "Bash(gh api repos/:*)", + "Bash(gh pr view:*)", + "Bash(gh issue view:*)", + "Bash(gh pr list:*)", + "Bash(gh issue list:*)", + "Bash(gh pr checks:*)", + "Bash(gh run view:*)", + "Bash(gh run list:*)" + ], + + "deny": [ + "Bash(git push:*)", + "Bash(git push --force:*)", + "Bash(git push -f:*)", + "Bash(git reset --hard:*)", + "Bash(git checkout --:*)", + "Bash(git clean -fd:*)", + "Bash(git branch -D:*)", + "Bash(rm -rf /:*)", + "Bash(rm -rf ~:*)", + "Bash(sudo:*)", + "Bash(docker compose down -v:*)", + "Bash(docker volume rm:*)", + "Bash(docker rm -f:*)", + "Bash(alembic downgrade:*)", + "Bash(npm publish:*)", + "Bash(gh pr merge:*)", + "Bash(gh pr close:*)" + ] + }, + + "env": { + "PYTHONDONTWRITEBYTECODE": "1" + }, + + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR/.claude/hooks/check_changed.sh\"" + } + ] + } + ] + } +} diff --git a/.gitignore b/.gitignore index 99f113fb404..be58436ef73 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,19 @@ *.sw? docker-compose.dev.yml venv/ + +# Claude Code: keep shared project settings (.claude/settings.json) but not +# personal overrides or session state. +.claude/settings.local.json +.claude/projects/ +.claude/cache/ + +# Claude Code /export outputs (timestamped conversation transcripts). +# Pattern: YYYY-MM-DD-HHMMSS-.txt — typically dropped into the cwd +# from which Claude Code was launched. +[0-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]-*.txt + +# Stray LLM API request payloads sometimes dumped during local debugging. +# These contain prompt content (chat history, internal references, names). +backend/requestdata.json +requestdata.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..077c1153846 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,635 @@ +# AGENTS.md + +Operating notes for AI coding agents working in this repo. Read this before +making non-trivial changes — it captures the architecture facts that aren't +obvious from a top-down read of the code, plus the footguns this codebase +has bitten previous sessions with. + +If you're a human contributor, this is also a useful onboarding companion +to `CONTRIBUTING.md` (which is more about *how to set up*; this is more +about *how the system actually works*). + +> **⚠️ Read this before applying anything you've seen in upstream Onyx.** +> This repo is a Danswer fork, branded "Darwin" internally, that diverged +> from upstream roughly two years ago. Onyx +> ([github.com/onyx-dot-app/onyx](https://github.com/onyx-dot-app/onyx)) +> has since been substantially rewritten — different background-worker +> architecture, different error-handling conventions, different package +> name, different LLM-tracing system, different multi-tenant model. Many +> of upstream's `AGENTS.md` / `CLAUDE.md` rules will actively mislead you +> here. See **"Divergence from upstream Onyx"** below for a concrete +> mapping of what does and doesn't apply. + +--- + +## What this repo is + +- **Origin**: fork of [Danswer](https://github.com/danswer-ai/danswer) + (now renamed [Onyx](https://github.com/onyx-dot-app/onyx) upstream). + This fork is **roughly two years behind upstream** on most modules. +- **Branding**: shipped internally as **"Darwin"** — that name appears + in UI labels, connector name builders, and a few comments. The Python + package is still called `danswer/`; don't rename it unless you mean to. +- **Working branch**: `feature/darwin`. `main` exists but new work lives + on `feature/darwin`. +- **Deployment shape**: monorepo with `backend/` (Python 3.11, FastAPI), + `web/` (Next.js 14 App Router, React 18, TypeScript), and + `deployment/docker_compose/` for local + prod stacks. + +--- + +## Architecture in one paragraph + +A FastAPI backend serves an admin / search UI written in Next.js. Data +sources are pulled by **connectors** (one Python class per source type) +on a fixed cadence into **Vespa** for vector + BM25 search and into +**Postgres** for relational metadata. Indexing runs on a **Dask local +cluster** spawned by `dev_run_background_jobs.py`. **Celery** (with a +SQLAlchemy/Postgres broker) handles a few maintenance tasks — cleanup, +prune, document-set sync, user-group sync — but **does not run indexing**. +LLM calls go through a configurable gateway (this fork defaults to a +UiPath OAuth-secured custom endpoint). + +--- + +## Divergence from upstream Onyx + +The single most likely way to break this codebase is to copy a pattern +from upstream's `AGENTS.md` / `CLAUDE.md` and apply it here. Upstream has +moved on substantially. This table is the explicit map. + +| Concept | Upstream Onyx | This fork | +|---|---|---| +| Package name | `onyx/` | `danswer/` (kept the original) | +| Indexing runtime | Celery `docfetching` + `docprocessing` workers | **Dask `LocalCluster`** in `update.py` (Celery only does maintenance) | +| Number of Celery workers | Eight specialized workers (primary, light, heavy, kg_processing, monitoring, beat, etc.) | One worker + beat, spawned by `dev_run_background_jobs.py` | +| Celery task definition | `@shared_task` under `background/celery/tasks/` | `@celery_app.task` in `background/celery/celery_app.py` | +| Celery broker | Redis | SQLAlchemy/Postgres (`sqla+postgresql+psycopg2://…`) | +| Error handling | `raise OnyxError(OnyxErrorCode.X, …)` everywhere; no `HTTPException` | Plain `HTTPException(status_code=…, detail=…)` is the norm here. `OnyxError` doesn't exist. | +| FastAPI return types | "Don't use `response_model=`, just type the function" | Both styles exist in this fork (the typed-return-annotation form is the majority — `response_model=` only appears once in `connector.py:560`). New endpoints should use the typed-return form. Don't strip the existing `response_model=` without checking serialization behavior. | +| LLM call instrumentation | Every call must open a `LLMFlow`-tagged span via `traced_llm_call(...)` | No tracing system. `LLMFlow` doesn't exist. | +| Multi-tenancy | `alembic -n schema_private upgrade head`; `DynamicTenantScheduler` | Single-tenant. `alembic upgrade head` is the only form. | +| Knowledge Graph processing | Full pipeline (`kg_processing` worker, clustering) | Doesn't exist | +| Logs | `backend/log/_debug.log` (services tail to per-file logs) | No `backend/log/` dir; logs go to stdout of each process | +| Postgres container name | `onyx-relational_db-1` | `danswer-stack-relational_db-1` (only when started with `docker compose -p danswer-stack`, which `CONTRIBUTING.md` mandates; if you skipped `-p`, run `docker ps` to confirm the actual name) | +| Test buckets | `backend/tests/{unit,external_dependency_unit,integration}` + Playwright e2e | No comparable structure here. Most code lacks tests; add tests with the change if practical, otherwise note in PR. | +| Plan template | The "Creating a Plan" section in their `CLAUDE.md` (Issues / Notes / Strategy / Tests) | Useful template; can be borrowed for non-trivial changes here too. | +| Frontend stack | Next.js 15+, React 18+ | Next.js 14.2.x (App Router), React 18 | + +**Rule of thumb when reading upstream code or upstream guidance:** assume +it doesn't apply unless you can verify the same construct exists here. +`grep` for the construct in `backend/danswer/` first. If it's missing, +don't introduce it as part of an unrelated change — that's a substantial +new dependency, not a drive-by. + +**When upstream guidance does still apply:** + +- **Type strictness** (Python with mypy, TypeScript). Both projects are + fully type-annotated and we keep them that way. +- **DB ops live in `db/` directories.** Don't write SQL or ORM queries + outside `backend/danswer/db/` (or `backend/ee/danswer/db/` for EE + features). +- **Never commit secrets.** Encrypted credential storage for connector + creds; OAuth client_id/secret in env, not source. +- **Frontend → backend goes through the frontend's proxy at `:3000/api/...`, + not directly to `:8080`** when you're testing UI flows. The proxy adds + cookies / auth headers the bare backend doesn't know about. +- **`source .venv/bin/activate`** if you hit ImportErrors. Same trick. +- **The plan-writing template** (Issues / Important Notes / Strategy / Tests, + no rollback / timeline) is a good shape for non-trivial work here too. + +--- + +## Top-level layout + +``` +backend/ + danswer/ + background/ + celery/ ← Celery app + maintenance tasks (NOT indexing) + connector_deletion.py ← cleanup_connector_credential_pair_task body + indexing/ ← indexing pipeline (chunking, embeddings) + update.py ← MAIN INDEXING LOOP — Dask LocalCluster + configs/constants.py ← DocumentSource enum (every connector adds here) + connectors/ + /connector.py ← one folder per connector type + factory.py ← DocumentSource → connector class registry + models.py ← Document / Section / BasicExpertInfo + db/ + models.py ← SQLAlchemy: Connector, Credential, + ConnectorCredentialPair, IndexAttempt, + TaskQueueState, Document, ... + index_attempt.py ← scheduling helpers + credentials.py + tasks.py ← Celery task tracking helpers + document_index/vespa/index.py ← Vespa client (read + write) + server/ + documents/ + connector.py ← /api/manage/admin/connector/* routes + credential.py ← /api/manage/credential/* routes + cc_pair.py ← /api/manage/admin/cc-pair/{id} routes + models.py ← Pydantic snapshots (ConnectorIndexingStatus, + IndexAttemptSnapshot, CCPairFullInfo, ...) + manage/ ← admin housekeeping endpoints + danswerbot/slack/listener.py ← Slack bot (separate process) + alembic/versions/ ← migrations (single-head expected) + scripts/ + dev_run_background_jobs.py ← spawns Celery worker + beat + Dask indexer + list_salesforce_account_fields.py + preview_salesforce_accounts.py + dump_salesforce_account.py ← Salesforce-specific dev tooling + +web/src/ + app/admin/ + add-connector/ ← connector tile gallery (sources.ts feeds it) + connector/[ccPairId]/ ← per-cc-pair detail page (status, credential + editor, indexing attempts, delete) + connectors// ← per-source-type setup page (Step 1: creds, + Step 2: manage). Folder name = derived URL. + indexing/status/ ← "Existing Connectors" list with bulk filter / edit + components/admin/connectors/ + ConnectorForm.tsx ← config form + CredentialForm.tsx ← credential form (supports edit mode via + existingCredentialId prop) + table/ + ConnectorsTable.tsx ← used by per-source-type pages + DeleteColumn.tsx ← reads is_deletable from indexing-status response + lib/ + types.ts ← TS mirror of Pydantic snapshots + sources.ts ← tile metadata (icon, displayName, adminUrl?) + connector.ts ← connector REST client + credential.ts ← credential REST client (createCredential, + updateCredential PATCH, linkCredential) + +deployment/docker_compose/ + docker-compose.dev.yml ← local stack (relational_db + index/Vespa + + api_server + web_server + model_server + + background + nginx). Note: no Redis + here — Celery uses Postgres as its broker. +``` + +--- + +## Critical facts that bite + +These are the non-obvious things previous sessions wasted hours on. +Read carefully. + +### 1. Indexing is on Dask, NOT Celery + +`update.py` spawns a `dask.distributed.LocalCluster` (or `SimpleJobClient` +in dev) and submits indexing attempts as Dask futures via +`client.submit(run_indexing_entrypoint, attempt_id, ...)`. Celery in this +codebase only runs the maintenance tasks listed in `celery_app.py`: +`cleanup_connector_credential_pair_task`, `prune_documents_task`, +`sync_document_set_task`, `check_for_*` periodic tasks. + +If you need to change indexing behavior (priority, queueing, concurrency), +**look at `update.py` and Dask docs**, not Celery / Kombu. + +**Concurrency**: `NUM_INDEXING_WORKERS` env var controls Dask cluster +size; default is 1. Safe to scale to N>1. Two independent guards prevent +the failure modes that show up beyond a single worker: + +1. **Per-cc-pair collision guard** — two layers, both leave the row as + `NOT_STARTED` (never FAILED), so the indexing-status table never + accumulates "skipped" failure rows for routine deferral: + - Scheduler-side: `update.py::kickoff_indexing_jobs` defers any + NOT_STARTED attempt whose `(connector, credential, embedding_model)` + tuple already has an IN_PROGRESS attempt. Catches the common case + (manual Re-Index colliding with an auto-scheduled run). + - Worker-side: `try_acquire_cc_pair_lock` (Postgres advisory lock, + session-scoped) covers the true-race case where two NOT_STARTED + rows are submitted in the same scheduler tick. If the lock fails, + the worker **reverts the attempt to NOT_STARTED** (clears + `time_started`) so the next tick re-dispatches it. + Replaces upstream Onyx's per-cc-pair Redis fence pattern. + +2. **Scheduler-side per-source-type cap** + (`background/update.py::kickoff_indexing_jobs`, configured by + `configs/indexing_concurrency.py`): generic cap of + `INDEXING_PER_SOURCE_CAP` (default `1`) attempts per `DocumentSource` + at a time. Before submitting each NOT_STARTED attempt to Dask, the + scheduler counts IN_PROGRESS attempts for the same source and defers + anything that would push it over the cap. Deferred attempts stay + NOT_STARTED — no FAILED rows, no extra `error_msg`. With + `NUM_INDEXING_WORKERS=4` + four GitHub cc-pairs: one runs, three sit + in NOT_STARTED until the running one finishes. Every source type is + its own bucket; connectors that share an external credential (e.g. + `github` + `github_files` share a PAT) are not collapsed — fold them + into a single `DocumentSource` if that matters for your rate limits. + Set `INDEXING_PER_SOURCE_CAP=0` to disable capping entirely. + +When scaling `NUM_INDEXING_WORKERS`, also bump model-server worker +count and Postgres `max_connections` — see CONTRIBUTING.md "Scaling +indexing concurrency" + "Per-Source Indexing Concurrency Caps" for the +full operational checklist. + +### 2. `dev_run_background_jobs.py` swallows subprocess errors + +The dev script spawns Celery worker + beat as `subprocess.Popen` and only +echoes their stdout. **There's no return-code check.** If the worker dies +at boot (wrong `-A`, broker unreachable, bad `--pool` combo), you'll see +the error once in the WORKER: log lines, then `monitor_process` exits +quietly while the parent script keeps running indexing fine — but no +maintenance tasks ever execute, and `task_queue_jobs` rows pile up forever +in `PENDING`. + +Before debugging "stuck deletion / sync / prune" issues, **always confirm +the worker is alive**: `ps aux | grep '[c]elery.*worker'`. If it's not +there, fix whatever's wrong with the boot args and restart the script. + +The current correct `-A` value is `ee.danswer.background.celery.celery_app` +(the module path, not the package — `__init__.py` is empty so the package +form fails with "no attribute 'celery'"). + +### 3. Long-running processes must restart after enum additions + +Adding a value to `DocumentSource` (or any string enum used in +Pydantic models) is a **breaking change for in-memory consumers**. If the +indexer writes a document with `source_type = "github_files"` to Vespa +while the API server / Slack listener is still running with the *old* +enum loaded in memory, every read of that document will fail with a +Pydantic `ValidationError: source_type` once the new value comes back +from Vespa. + +Process restart matrix after enum additions: + +| Process | How it's started | Why it needs restart | +|---|---|---| +| Indexer | `python scripts/dev_run_background_jobs.py` | Constructs Documents with the new source. Restart to register. | +| API server | `uvicorn danswer.main:app …` | Deserializes Vespa results. | +| Slack listener | `python danswer/danswerbot/slack/listener.py` | Same as API. | +| Celery worker / beat | spawned by the dev script | Imports `connectors/factory.py`. | +| Frontend (`npm run dev`) | Hot-reloads modules but `.next/cache` can lag — `rm -rf web/.next` if a tile/source rename doesn't show. | + +### 4. The list endpoint serves both pages + +`/api/manage/admin/connector/indexing-status` is consumed by: +- `/admin/indexing/status` (the list, via `CCPairIndexingStatusTable`) +- *Every* per-source-type page (e.g. `/admin/connectors/sf-account`, + via `ConnectorsTable` + `DeleteColumn`). + +Both deserialize the same `ConnectorIndexingStatus[]` payload. So fields +in the response can't be removed unilaterally — `is_deletable` for +example is only read by `DeleteColumn`, but it's still required because +that column lives on the per-source-type pages. + +When optimizing this endpoint, **compute fields inline from data already +in memory** (latest_index_attempt, connector.disabled) rather than +removing fields. The current implementation in +`server/documents/connector.py::get_connector_indexing_status` does +this — bulk-fetches `latest_index_attempts`, `document_count_info`, and +`cleanup_task_by_name` up front, then computes `is_deletable` and +`deletion_attempt` per row from those dicts. Total query count is O(1) +in the number of cc_pairs. Don't re-introduce per-row helper calls. + +### 5. Frontend admin URL is auto-derived from displayName + +`web/src/lib/sources.ts::fillSourceMetadata` builds `adminUrl` as +`/admin/connectors/${displayName.toLowerCase().replaceAll(" ", "-")}` +*unless* the entry overrides `adminUrl`. So if you change a tile's +`displayName` and the auto-derived URL no longer matches the folder +under `web/src/app/admin/connectors/`, **the tile clicks land on a 404**. + +Three options when renaming: +1. Change displayName + rename the folder to match. +2. Change displayName + add an explicit `adminUrl: "/admin/connectors/"` + override on the source entry. +3. Don't change displayName. + +### 6. Credentials are stored as opaque JSON, no per-source schema + +The `credential` table has a `credential_json: dict` column. The backend +`update_credential` does a wholesale replace — there's no merge, no +field-level update. So edit forms must submit the full credential JSON +including any discriminator fields (e.g. `sf_credential_kind`) or those +get wiped on save. + +### 7. Salesforce SOQL date format + +SOQL accepts both `Z` and `+0000` timezone suffixes, but **`+00:00` +(with the colon) sometimes survives URL-encoding into the query string +as a space**, silently turning the WHERE clause into a no-match. Always +format dates as `%Y-%m-%dT%H:%M:%S.000Z` — see `_soql_datetime` in +`connectors/salesforce/connector.py` for the helper. + +### 8. Office365 auto-parses `application/json` files in SharePoint + +The `office365-rest-python-client==2.5.9` library inspects the response +Content-Type — if a SharePoint file's response comes back as +`application/json` (which happens for any `.json` file in the drive), +the library parses the body into Python objects (list / dict) and +populates `driveitem.get_content().execute_query().value` with that, +instead of bytes. So passing `.value` directly to `io.BytesIO(...)` +crashes with `TypeError: a bytes-like object is required, not 'list'` +on JSON files. + +Past attempts to work around this in the connector were reverted (the +re-serialized text isn't byte-for-byte faithful since the library has +already lost the original whitespace / key order). If you're indexing +JSON files via SharePoint, the right fix is bypassing office365's +auto-parse entirely with a raw `requests.get` against the +`/drives/{drive_id}/items/{item_id}/content` endpoint using the bearer +token. Don't reintroduce the lossy re-serialization. + +--- + +## Common workflows + +### Add a new connector + +Verified against the most recent connector addition (`github_files`). +Touch every one of the files below; missing any of them produces silent +or hard-to-diagnose failures (404 tile, factory KeyError, frontend type +errors, or — worst — the connector silently registering as the wrong +type). + +**Backend** + +1. **Create the package directory** under + `backend/danswer/connectors//` with two files: + - `__init__.py` — empty (required so Python treats it as a package). + - `connector.py` — your connector class implementing one or more of + the abstract bases in `backend/danswer/connectors/interfaces.py`: + `LoadConnector` (full pull), `PollConnector` (incremental by time + window), `IdConnector` (cheap doc-id enumeration for prune). + Raise `ConnectorMissingCredentialError` (from + `connectors/models.py`) when `load_credentials` hasn't been called. + Clone `connectors/github_files/connector.py` as the most recent + minimal example. + +2. Add the new value to the `DocumentSource` enum in + `backend/danswer/configs/constants.py`. Format: + ` = ""`. The string value is what gets + stored on every document and round-trips through Vespa — pick it + carefully, you can't change it later without a migration that + re-keys existing rows. + +3. Register the connector in `backend/danswer/connectors/factory.py`: + - Add the `from danswer.connectors..connector import + Connector` import line at the top. + - Add `DocumentSource.: Connector,` to the + `connector_map` dict in `identify_connector_class`. + +**Frontend** + +4. Add the new source string to the `ValidSources` union in + `web/src/lib/types.ts`. Keep alphabetical order to make merges easier. + +5. Add a `Config` interface to the same `types.ts` matching the + shape your connector's `__init__` reads from + `connector_specific_config`. Field names must be snake_case to match + what the backend deserializes. + +6. **If your connector uses a new credential shape**, add a + `CredentialJson` interface to `types.ts` too. If you're + reusing an existing credential (e.g. `GithubCredentialJson` for the + github-files connector — both share the GitHub PAT), skip this step. + +7. Add a tile entry to `web/src/lib/sources.ts` keyed by the new + `ValidSources` string. Required: `icon`, `displayName`, `category`. + Optional: `adminUrl` override (use it when you can't make the route + folder match the auto-derived URL — see Critical Fact #5). + +8. Create the per-source admin page at + `web/src/app/admin/connectors//page.tsx` where + `` is exactly + `displayName.toLowerCase().replaceAll(" ", "-")` (or whatever you + put in the optional `adminUrl` override). Examples: + `displayName: "Github"` → `connectors/github/`, + `displayName: "GitHub-Files"` → `connectors/github-files/`, + `displayName: "SF-Account"` → `connectors/sf-account/`. + Clone `web/src/app/admin/connectors/github-files/page.tsx` as the + shortest template. + +**Operational** + +9. **Restart every long-running process that imports the enum**: + API server (`dapi`), background jobs (`dbe`), Slack listener + (`dsl`). See Critical Fact #3 — a stale process will fail with + `pydantic.ValidationError: source_type` the moment it reads back + any indexed document of the new type from Vespa. + +10. **Run the migration head check** even if you didn't write a + migration. Adding the enum value alone doesn't need one (it's a + Python-side `str` enum, not a DB enum), but if you also added a + new column for connector-specific config, run `alembic upgrade head`. + +### Add a backend SQL field + +1. Add `Mapped[X]` column to the model in `db/models.py`. Always set + `nullable=False` + `server_default=...` for backfill safety. +2. New Alembic migration in `backend/alembic/versions/`. Run + `alembic heads` first to know what to set as `down_revision`. +3. If the field is exposed via API, add it to the matching Pydantic + snapshot in `server/documents/models.py` and surface it in the + `from_*_db_model` classmethod. +4. Mirror in `web/src/lib/types.ts`. +5. **Use this migration as both upgrade and downgrade rehearsal**: + `alembic upgrade head && alembic downgrade -1 && alembic upgrade head`. + +### Bulk-fetch a per-row computed field + +This codebase has a recurring N+1 pattern in routes that loop over +cc_pairs and call helpers per row. The fix shape is: + +1. Collect the lookup keys for every row up front + (e.g. `[name_cc_cleanup_task(c, cr) for cc_pair in cc_pairs]`). +2. Add a `get_latest_*_by_*` bulk helper in `db/...` that runs *one* SQL + query with `IN (...)` + a "max-per-group" pattern, returns a dict. +3. In the loop body, look up by key from the dict instead of calling the + per-row helper. + +See `db/tasks.py::get_latest_tasks_by_names` and the corresponding +refactor in `server/documents/connector.py::get_connector_indexing_status` +for the pattern. + +### Edit credentials without re-creating the connector + +Backend `PATCH /api/manage/credential/{id}` already exists. Frontend +helper is `lib/credential.ts::updateCredential`. The +`CredentialForm` component supports edit mode via the +`existingCredentialId` prop — set it and the form PATCHes instead of +POSTs. See the per-source-type setup pages for the pattern (sf-account, +sf-kbarticles, github, github-files, slack, confluence, jira, sharepoint +all have inline edit affordances). The cc-pair detail page has a +**generic** credential editor (`CredentialSection.tsx`) that works for +any connector type by introspecting `credential_json` keys. + +### Bulk pause / re-enable connectors + +`/admin/indexing/status` has filter + multi-select + bulk action UI. +Backend just uses the existing `PATCH /api/manage/admin/connector/{id}` +with `disabled: bool` flipped — no special bulk endpoint needed. + +--- + +## Conventions + +### Backend + +- **Type hints required** — `mypy` runs in CI (`python -m mypy .` from + `backend/`). Config lives in `backend/pyproject.toml`. +- **No `print` in non-`__main__` paths** — use `setup_logger()` from + `danswer.utils.logger`. `print()` is fine inside `if __name__ == + "__main__":` blocks of connectors / scripts (there are ~58 of those + for ad-hoc debugging) but should never appear in request handlers, + background jobs, or library code. +- **Time**: always store timezone-aware UTC. `datetime.utcfromtimestamp` + is deprecated in Python 3.12+ — use `datetime.fromtimestamp(s, tz=utc)`. +- **DB sessions**: never long-lived. The pattern is + `with Session(get_sqlalchemy_engine()) as db_session: …`. Long-running + background work eagerly loads relationships then expires the session. +- **Pre-commit**: `black` (formatter), `reorder-python-imports`, + `autoflake` (dead-import remover). All configured in + `.pre-commit-config.yaml`; install with `pre-commit install` from + `backend/`. Don't fight them. + +### Frontend + +- **Tremor** components for admin UI (`@tremor/react`). Tailwind utility + classes for spacing/layout. +- **Yup** for form validation, **Formik** for form state. +- **SWR** for data fetching with `refreshInterval` for live status views. +- **Heroicons / Feather** via `react-icons/fi` and the project's + `@/components/icons/icons.tsx`. +- **TypeScript types in `lib/types.ts`** mirror Pydantic snapshots from + `server/documents/models.py`. Out-of-sync types are the most common + source of frontend regressions. Update both together. + +### Naming + +- Connector classes: `Connector` (e.g. `GithubFilesConnector`). +- Connector pydantic config: `Config` (frontend) / + matching shape on the connector's `__init__`. +- Admin page route folders: kebab-case matching the lowercased + `displayName`. Cross-reference with `lib/sources.ts`. +- Migration filenames: `_.py`. + +--- + +## Footguns / things to avoid + +- **Don't add fields to Yup schemas if they're constants set in + `initialValues`** — TypeScript will complain about the schema-vs-type + shape mismatch. The pattern (used for `sf_credential_kind`) is to keep + the constant in `initialValues` only and let Yup pass it through. +- **Don't run `alembic` from the project root** — it must be run from + `backend/` so the `alembic.ini` resolves correctly. +- **Don't trust upstream copy-paste in connector pages** — `sfkbarticles` + was filtering the wrong source string (`"salesforce"` instead of + `"sfkbarticles"`) for over a year. When cloning a page, search-and- + replace the source name carefully. +- **Don't use `echo -e`** in shell helpers — non-portable across shells. + Use `printf`. +- **Don't `git stash` if the work-in-progress includes new files** — the + stash doesn't include untracked files by default. Use `git stash -u`. +- **Don't drop fields from `/api/manage/admin/connector/indexing-status`** + without checking *every* page that consumes it. See Critical Fact #4. +- **Don't pass the `priority=` kwarg to `SimpleJobClient.submit`** — only + `dask.distributed.Client` honors it. The current code in + `update.py::kickoff_indexing_jobs` checks `isinstance(client, Client)` + before adding the kwarg; preserve that guard. + +--- + +## Notable session-resolved bugs (for context) + +If you see code that looks "wrong but intentional", here are some +historical fixes — useful so you don't accidentally undo them: + +- **`dev_run_background_jobs.py` Celery `-A`** points at + `ee.danswer.background.celery.celery_app` (module form). The package + form (`ee.danswer.background.celery`) doesn't work because the + package's `__init__.py` is empty. +- **`runConnector` in `lib/connector.ts`** sends `credential_ids` + (snake_case). It used to send `credentialIds` (camelCase) which the + backend silently ignored, falling back to "all credentials." +- **`sfkbarticles/page.tsx`** filters `source === "sfkbarticles"`. It used + to filter `source === "salesforce"` (copy-paste from the salesforce + page) which made every salesforce connector show up on both pages. +- **Salesforce credential discriminator (`sf_credential_kind`)** + distinguishes Account vs KB-Articles credentials so the two pages don't + share each other's. Untagged legacy credentials are accepted as + "account" type for back-compat. +- **`indexing_priority` on `IndexAttempt`** — per-attempt, not per-cc-pair + or per-connector. See `update.py` for the Dask handoff. +- **`get_connector_indexing_status` is now O(1) queries** regardless of + cc-pair count — per-row deletion-status lookups were bulk-fetched. Don't + re-introduce per-row lookups in this endpoint. + +--- + +## When to ask the human + +Default to "make the safe small change first, then ask." But these +specifically warrant pausing: + +- **Schema migrations on tables with many rows** (esp. `index_attempt`, + `task_queue_jobs`, `document`, Vespa). Indexes can be slow; migrations + hold transactions. +- **Removing a `DocumentSource` enum value** — any historical document in + Vespa with that source becomes unreadable. +- **Renaming an admin URL** — bookmarks break, and any external + references / docs go stale. +- **Touching the celery `-A` arg or `dev_run_background_jobs.py` + subprocess plumbing** — past breakage was silent (worker died, no logs) + and took days to surface. +- **Bumping `NUM_INDEXING_WORKERS`** in prod — concurrency increase plus + memory growth, plus more Dask scheduler work. +- **Changing default LLM provider** — UiPath gateway requires specific + env vars and OAuth flow; default OpenAI doesn't. + +--- + +## Useful one-liners + +```bash +# Activate venv (the fix for most ImportError surprises) +source .venv/bin/activate + +# Talk to Postgres without leaving the shell +docker exec -it danswer-stack-relational_db-1 psql -U postgres \ + -c "SELECT id, name, source FROM connector LIMIT 10;" + +# Tail Celery worker output (it goes to stdout of dev_run_background_jobs.py) +ps aux | grep '[c]elery.*worker' # confirm it's alive first + +# Inspect the alembic head + chain +cd backend && alembic heads && alembic history --rev-range -3:HEAD + +# Force a Next.js rebuild after enum / source-list changes +rm -rf web/.next && (cd web && npm run dev) +``` + +For everything else — running services, env vars, helper aliases — see +`CONTRIBUTING.md`. + +--- + +## Files to read first when picking up new work + +- `backend/danswer/db/models.py` — the data model; the ER graph in your + head should come from this. +- `backend/danswer/configs/constants.py` — `DocumentSource` and other + string enums that flow through the whole system. +- `backend/danswer/connectors/factory.py` — connector registry; tells you + what types exist and where they live. +- `backend/danswer/server/documents/connector.py` — most admin REST + routes are here. +- `backend/danswer/background/update.py` — the indexing scheduler. If + you're working on indexing performance / priority / scheduling, this + is the file. +- `web/src/lib/types.ts` — frontend's view of backend models. Diverges + occasionally; reconciling it with `server/documents/models.py` is a + recurring task. +- `web/src/lib/sources.ts` — connector tile registry (icon, displayName, + category, optional adminUrl override). The actual TypeScript mirror + of the backend `DocumentSource` enum is the `ValidSources` union in + `web/src/lib/types.ts`. New connector types must be added to *both* + files. + +Don't try to read all of `update.py` or `connector.py` cold — they're +both large. Skim section headings, then dive into the specific function +relevant to the change. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..77e7b9b6abb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,135 @@ +# CLAUDE.md + +This file is the entry point for Claude (and other LLM coding agents) when +working in this repository. To keep one source of truth, the substantive +operating notes live in [`AGENTS.md`](./AGENTS.md). **Read that file +first.** + +The rest of this document is a thin per-agent overlay: things that are +specifically useful for Claude / Claude Code without cluttering the +shared `AGENTS.md`. + +--- + +## Critical pre-flight + +**Before applying anything you've seen in upstream Onyx +(`github.com/onyx-dot-app/onyx`), check the "Divergence from upstream +Onyx" section in `AGENTS.md`.** This fork is roughly two years behind +upstream. Several upstream rules — `OnyxError` everywhere, no +`response_model=`, Celery-based indexing, `@shared_task`, `LLMFlow` +tracing, multi-tenant migrations — **do not apply here** and will break +things if introduced as drive-by changes. + +When in doubt, `grep` for the construct in `backend/danswer/` first. If +it's missing, don't introduce it without asking the human. + +--- + +## What lives where + +- **Architecture, footguns, conventions, recipes**: [`AGENTS.md`](./AGENTS.md) +- **How to set up locally** (env vars, services, dev aliases): + [`CONTRIBUTING.md`](./CONTRIBUTING.md) +- **Repo / monorepo layout** with annotations: AGENTS.md → "Top-level + layout" +- **Common workflows** (add a connector, add a SQL field, edit + credentials, etc.): AGENTS.md → "Common workflows" + +--- + +## Claude-specific working norms + +These are tuned for how Claude Code (and similar agents) tend to behave. +They don't apply only to Claude, but they're written with that workflow +in mind. + +### Don't trust scrollback + +The conversation context can be hundreds of messages by the time you +arrive. Skim `git log -10`, `git status`, and `git diff --stat` before +making any non-trivial change so you're operating on the actual current +state, not a remembered state. The user has been editing files between +your turns; assume the file on disk is canonical, not your memory of it. + +### Prefer minimal, isolated edits + +Most tasks here have come in small, focused changes (often single-file +or single-feature). Don't bundle unrelated cleanups into a feature edit +unless explicitly asked. The repo has long-standing inconsistencies +(camelCase / snake_case mismatches, deprecated `datetime` calls, copied +filter bugs in connector pages) — point them out as observations rather +than silently "fixing" them. + +### Match the pattern, not the upstream + +When adding new connectors or admin pages, the right template is the +**most-recently-edited equivalent in this repo**, not the upstream Onyx +version. Examples: + +- New connector? Clone `connectors/github_files/connector.py`'s shape. +- New per-source-type admin page? Clone `web/src/app/admin/connectors/sf-account/page.tsx`. +- New backend bulk-fetch helper? Mirror `db/tasks.py::get_latest_tasks_by_names`. + +Upstream's version of any of these is likely better-architected but is +also unrecognizable to this fork. + +### Process bounce list + +Several long-running services don't auto-reload on certain kinds of +changes. After any of these edits, ask the human to restart the relevant +process (you can't bounce them from inside the agent loop): + +| Edit type | Restart | +|---|---| +| Add / remove a `DocumentSource` enum value | API server (`dapi`), background jobs (`dbe`), Slack listener (`dsl`). The Slack one is the easiest to forget — historically users saw `pydantic ValidationError: source_type` until they bounced it. | +| Modify a connector's `__init__` signature, credential schema, or factory mapping | Background jobs (`dbe`) — both indexer and Celery worker spawn from there. | +| Modify a SQLAlchemy model | Run `alembic upgrade head` from `backend/` first. Then bounce API server + background jobs. | +| Edit `web/src/lib/sources.ts`, `lib/types.ts`, or any TypeScript-side enum | Most edits hot-reload via `npm run dev`. If a new tile / source / route doesn't appear after a hard refresh, `rm -rf web/.next` and restart `dfe` — `.next/cache` occasionally holds stale module bundles. | + +### When you write a plan + +If the user asks for a plan or you're about to spawn a complex multi-step +change, use the four-section template borrowed from upstream's CLAUDE.md +(it's a good shape): + +1. **Issues to address** — what the change is meant to do. +2. **Important notes** — non-obvious things you found while researching + (e.g. "this endpoint also feeds the per-source pages, so removing + field X would break them"). +3. **Implementation strategy** — high-level. File names, function names, + no code. +4. **Tests** — what you'll write to verify. The fork has scant test + coverage; if you can't add a test, say so explicitly so the human can + spot-check manually. + +Skip "timeline" and "rollback plan" — they don't fit the development +shape here. + +### When to pause + +`AGENTS.md` has a "When to ask the human" list. The non-obvious additions +for Claude specifically: + +- **Anything involving `dev_run_background_jobs.py` subprocess plumbing.** + Past breakage was silent (worker died, no logs) and ate days. Have the + human confirm the worker boots cleanly after any change there. +- **Any change to `update.py`'s scheduler / Dask submission.** Same + reason — failure modes are subtle. +- **Renames that affect URLs.** `displayName` and the route folder are + coupled by `lib/sources.ts::fillSourceMetadata`; a rename without + matching updates is a 404 and breaks bookmarks. + +--- + +## A note on instruction conflicts + +If the conversation contradicts `AGENTS.md`/`CONTRIBUTING.md`, ask which +takes precedence — usually the user's immediate instruction wins, but +sometimes they're testing whether you remember the project rules. If the +user says "do X anyway" after you've flagged the conflict, do X and note +that you're overriding documented guidance. + +If `AGENTS.md` and this file disagree, this file (CLAUDE.md) doesn't +override `AGENTS.md` — it's an overlay. Conflicts mean someone's drifted; +flag it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 116e78b6f19..d4a467248f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,11 +110,45 @@ playwright install #### Dependent Docker Containers -First navigate to `danswer/deployment/docker_compose`, then start up Vespa and Postgres with: +First navigate to `danswer/deployment/docker_compose`, then start Postgres. + +The simplest path is the compose-managed pair (uses a named docker volume for +Vespa's data; data lives until you `docker volume rm`): + +```bash +docker compose -f docker-compose.dev.yml -p danswer-stack up -d relational_db +``` + +If you'd rather pin Vespa's data + logs to host-mounted directories so you +can inspect them outside Docker (and survive `docker compose down -v`), +start Postgres via compose and Vespa via a manual `docker run` on the same +network. Pick any host paths you like: + ```bash -docker compose -f docker-compose.dev.yml -p danswer-stack up -d index relational_db +docker compose -f docker-compose.dev.yml -p danswer-stack up -d relational_db + +export VESPA_VAR_STORAGE="${HOME}/danswer-vespa-data/var" +export VESPA_LOG_STORAGE="${HOME}/danswer-vespa-data/logs" +mkdir -p "$VESPA_VAR_STORAGE" "$VESPA_LOG_STORAGE" + +docker run \ + --network=danswer-stack_default \ + --detach \ + --name vespa \ + --hostname index \ + --volume "$VESPA_VAR_STORAGE":/opt/vespa/var \ + --volume "$VESPA_LOG_STORAGE":/opt/vespa/logs \ + --publish 8081:8081 \ + --publish 19071:19071 \ + vespaengine/vespa:8.277.17 + +# Sanity check: both containers should be on the danswer-stack_default network +docker ps --format '{{ .ID }} {{ .Names }} {{ json .Networks }}' ``` -(index refers to Vespa and relational_db refers to Postgres) + +(index refers to Vespa and relational_db refers to Postgres. The hostname +`index` matters — Danswer reaches Vespa by that DNS name on the shared +network.) #### Running Danswer To start the frontend, navigate to `danswer/web` and run: @@ -135,13 +169,20 @@ powershell -Command " ``` The first time running Danswer, you will need to run the DB migrations for Postgres. -After the first time, this is no longer required unless the DB models change. +After the first time, run this any time you pull new code that touches `db/models.py` +or adds a file under `backend/alembic/versions/`. Navigate to `danswer/backend` and with the venv active, run: ```bash alembic upgrade head +# Verify there's a single head revision after upgrade +alembic heads ``` +If `alembic heads` reports more than one head, two branches landed in parallel +and you'll need to merge them with `alembic merge -m "merge heads" ` +before any new migrations can apply. + Next, start the task queue which orchestrates the background jobs. Jobs that take more time are run async from the API server. @@ -164,6 +205,382 @@ powershell -Command " Note: if you need finer logging, add the additional environment variable `LOG_LEVEL=DEBUG` to the relevant services. +#### Running the Slack bot listener (optional) +If you're working on the Slack bot, run the listener as a separate process +from the project root (so `PYTHONPATH=$(pwd)` resolves the package layout): + +```bash +cd backend +PYTHONPATH=$(pwd) python danswer/danswerbot/slack/listener.py +``` + +The listener imports the same `DocumentSource` enum the indexer uses, so if +you add a new `DocumentSource` value (e.g. when porting a new connector), +restart this process — long-running consumers won't pick up enum additions +until they're reimported, and a stale process will fail with a Pydantic +`ValidationError` on `source_type` whenever Vespa returns docs of the new +source. + +#### Helper Shell Aliases (optional) +The five processes above (frontend, model server, API server, background +jobs, slack listener) each need their own terminal. Dropping the snippet +below into your `~/.zshrc` (or `~/.bashrc`) gives you one short command per +process — each one cd's into the repo, activates the venv, sets the +terminal tab title, and starts the right service. + +Set `DANSWER_HOME` to your local clone of the repo before sourcing: + +```bash +export DANSWER_HOME="${DANSWER_HOME:-$HOME/code/danswer}" + +# Frontend (Next.js dev server, port 3000) +alias dfe='printf "\033]0;Danswer-FrontEnd\007"; cd "$DANSWER_HOME/web" && npm run dev' + +# Helper used by the rest: cd to repo + source venv. Fails loudly if +# either is missing so we don't accidentally run the wrong Python. +_danswer_activate() { + cd "$DANSWER_HOME" || return 1 + if [ ! -f ".venv/bin/activate" ]; then + echo "Error: $DANSWER_HOME/.venv not found." >&2 + return 1 + fi + source .venv/bin/activate +} + +# Model server (local NLP models, port 9000) +dmo() { + printf "\033]0;Danswer-Model\007" + _danswer_activate || return 1 + cd backend && uvicorn model_server.main:app --reload --port 9000 +} + +# API server (FastAPI, port 8080, auth disabled for dev) +dapi() { + printf "\033]0;Danswer-APIServer\007" + _danswer_activate || return 1 + cd backend && AUTH_TYPE=disabled uvicorn danswer.main:app --reload --port 8080 +} + +# Background jobs (indexing loop + Celery worker + Celery beat) +dbe() { + printf "\033]0;Danswer-Backend\007" + _danswer_activate || return 1 + cd backend && python ./scripts/dev_run_background_jobs.py +} + +# Slack listener (only if you're working on the Slack bot) +dsl() { + printf "\033]0;Danswer-Slack-Listener\007" + _danswer_activate || return 1 + cd backend && PYTHONPATH=$(pwd) python danswer/danswerbot/slack/listener.py +} +``` + +Source the file (`source ~/.zshrc`) and you can spin up the whole local +stack as `dfe`, `dmo`, `dapi`, `dbe`, `dsl` — one per terminal tab. The +`\033]0;...\007` escape sets the terminal/tab title so it's easier to find +the right window when juggling five. + +#### Environment Variables +The local stack reads a fair number of environment variables across its +processes (model server, API, background jobs, frontend). The block below +is a working set for a UiPath-internal dev setup — drop it into your +`~/.zshrc` (or a sourced `.envrc`), fill in the placeholders, and source +the file. + +> **Anything in `` is a placeholder.** Generate / fetch a +> real value from the relevant provider — never commit a real secret. + +```bash +# --------------------------------------------------------------------------- +# Local stack toggles +# --------------------------------------------------------------------------- +export ENVIRONMENT=LOCAL +export LOG_LEVEL=info # use `debug` if you need verbose tracing +export IMAGE_TAG=latest +export APPLY_MIGRATIONS=true +export DASK_JOB_CLIENT_ENABLED=true +export NUM_INDEXING_WORKERS=1 # see "Scaling indexing concurrency" below +export CONTINUE_ON_CONNECTOR_FAILURE=true + +# --------------------------------------------------------------------------- +# Service hosts / ports — match the local docker-compose stack +# --------------------------------------------------------------------------- +export VESPA_HOST=localhost +export VESPA_PORT=8081 +export VESPA_FEED_HOST=localhost +export VESPA_FEED_PORT=8081 +export VESPA_CONFIG_SERVER_HOST=localhost +export MODEL_SERVER_HOST=localhost +export MODEL_SERVER_PORT=9000 +export INDEXING_MODEL_SERVER_HOST=localhost +export INDEXING_MODEL_SERVER_PORT=9000 +export REDIS_HOST=cache # matches the compose service name + +# --------------------------------------------------------------------------- +# LLM (Generative AI) — UiPath LLM Gateway via OAuth client credentials +# Replace with your own gateway / model-provider settings if different. +# --------------------------------------------------------------------------- +export GEN_AI_MODEL_PROVIDER=custom +export GEN_AI_API_ENDPOINT='https:///llmgateway_/openai/deployments//chat/completions?api-version=' +export GEN_AI_IDENTITY_ENDPOINT='https:///identity_/connect/token' +export GEN_AI_CLIENT_ID='' +export GEN_AI_CLIENT_SECRET='' +# Leave the following empty when using GEN_AI_MODEL_PROVIDER=custom; they're +# only meaningful for direct OpenAI / Azure / Bedrock providers. +export GEN_AI_API_KEY= +export GEN_AI_MODEL_VERSION= +export GEN_AI_MAX_TOKENS= +export GEN_AI_LLM_PROVIDER_TYPE= +export FAST_GEN_AI_MODEL_VERSION= + +# LLM behavior knobs (saves tokens / time during dev) +export DISABLE_LLM_FILTER_EXTRACTION=true +export DISABLE_LLM_CHOOSE_SEARCH=true +export DISABLE_LLM_CHUNK_FILTER=true +export LOG_ALL_MODEL_INTERACTIONS=true + +# --------------------------------------------------------------------------- +# DB retention windows (all in days). The daily Celery beat task +# `run_retention_policies_task` runs at 08:00 UTC and deletes rows older +# than the windows below. Set any of these to 0 to disable that policy. +# See backend/danswer/db/retention.py for the exact SQL each window uses. +# Run once-off (e.g. against accumulated bloat) via: +# cd backend && python scripts/cleanup_stale_db.py --dry-run +# --------------------------------------------------------------------------- +export RETENTION_DAYS_KOMBU=7 # Celery broker queue (kombu_message) +export RETENTION_DAYS_TASK_QUEUE=30 # task_queue_jobs (terminal rows only) +# index_attempt is retained indefinitely by default. Connector debug +# history is generally worth keeping. To enable pruning, set this to a +# positive integer (and tune KEEP_LAST_N to keep recent attempts even +# when older than the day window): +# export RETENTION_DAYS_INDEX_ATTEMPT=60 +# export RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS=20 +export RETENTION_DAYS_CHAT=30 # chat_session + chat_message (cascade) +export RETENTION_DAYS_PERMISSION_SYNC=30 # permission_sync_run (terminal rows only) +export RETENTION_DAYS_USAGE_REPORTS=90 # usage_reports + file_store rows + LO blobs + +# Batching knobs — DELETEs run as `WHERE id IN (SELECT id ... LIMIT N)` +# in a loop, committing per batch so table locks don't stall Celery +# enqueue/consume. Defaults handle ~1M rows per policy per daily run; +# raise MAX_BATCHES for first-time sweeps against very large bloat. +# After any policy that deletes >= one full batch, the executor runs +# `ANALYZE ` to refresh planner stats. +export RETENTION_BATCH_SIZE=5000 # rows per DELETE statement +export RETENTION_MAX_BATCHES=200 # safety ceiling per policy per run + +# --------------------------------------------------------------------------- +# Analytics rollup — pre-aggregates the admin /analytics page metrics into +# `analytics_daily_rollup` so the dashboard survives chat retention deletes. +# A daily Celery beat task runs at 07:30 UTC (30 min before the retention +# sweep at 08:00 UTC). The endpoints under /api/analytics/admin/{query,user, +# danswerbot} read from this rollup. +# +# How the daily task picks its window: it reads a checkpoint +# `last_rolled_up_to` from `key_value_store` (under key +# `analytics_rollup_state`) and recomputes from +# `(checkpoint - ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS)` through today, +# then advances the checkpoint. So the buffer is just the late-feedback +# grace period (most reactions arrive in minutes, but a "resolved" mark +# from a Slack helper can take a day or two), NOT a fixed window — the +# task self-heals across outages by picking up from where it left off. +# +# Safety: the task refuses to scan further back than +# `(today - (RETENTION_DAYS_CHAT - 2))`. Catastrophic outages leave older +# days at their last-known values rather than zeroing them out from +# already-deleted chat data. +# +# Deployment notes (one-time, after pulling the rollup feature): +# 1. alembic upgrade head # creates analytics_daily_rollup +# 2. python scripts/backfill_analytics_rollup.py # populates from history +# 3. Restart background-deployment so the new beat task is picked up +# Step (2) also seeds the checkpoint to its --end date (today by default), +# so the daily task starts from the right place. +# --------------------------------------------------------------------------- +export ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS=2 # late-feedback grace period + +# --------------------------------------------------------------------------- +# GitHub PAT — used by the `gh` CLI and the GitHub / GitHub-Files connectors +# --------------------------------------------------------------------------- +export GITHUB_TOKEN='' + +# --------------------------------------------------------------------------- +# Salesforce — only needed when running the Salesforce connector __main__ +# blocks or the helper scripts under backend/scripts/ (e.g. +# preview_salesforce_accounts.py). Use single quotes around the password +# to preserve `$`, `<`, `*`, etc. Wrap with a leading space to keep the +# secret out of shell history (zsh: HIST_IGNORE_SPACE). +# --------------------------------------------------------------------------- +export SF_CLIENT_ID='' +export SF_CLIENT_SECRET='' +export SF_USERNAME='' +export SF_PASSWORD='' +# export SF_LOGIN_URL='https://test.salesforce.com' # uncomment for sandbox + +# --------------------------------------------------------------------------- +# Slack bot — only needed when running danswerbot/slack/listener.py +# --------------------------------------------------------------------------- +export DANSWER_BOT_SLACK_APP_TOKEN='' # xapp-… +export DANSWER_BOT_SLACK_BOT_TOKEN='' # xoxb-… + +# --------------------------------------------------------------------------- +# macOS-only PATH addition for Docker Desktop's CLI shims +# --------------------------------------------------------------------------- +if [ -d "/Applications/Docker.app/Contents/Resources/bin" ]; then + export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH" +fi +``` + +A few things worth knowing about this set: + +- **`PYTHONPATH` is *not* exported globally** here. The shell aliases above + set it per-command (`PYTHONPATH=$(pwd)` from inside `backend/`), which + resolves the package layout correctly without polluting other Python + invocations. +- **`NUM_INDEXING_WORKERS=1`** is the default. Each indexing worker is one + Dask process; with a single worker, only one indexing attempt runs at a + time. Bump to 2–4 if you want higher-priority attempts to overlap with + in-flight ones rather than wait for them to finish. +- **`DISABLE_LLM_*` flags** short-circuit several optional LLM calls in the + search pipeline. Useful in dev to keep iteration fast and avoid spending + tokens; turn them off (or unset) if you're testing those code paths. +- **Slack tokens are workspace-specific.** You won't pick up another team's + bot token by accident — but never paste them into a PR or Slack thread. +- **Salesforce credentials**: only needed when running the connector + directly (`python danswer/connectors/salesforce/connector.py`) or the + preview / dump scripts. The runtime indexer reads credentials from the + database, not the environment. + +#### Scaling Indexing Concurrency +By default `NUM_INDEXING_WORKERS=1` — the indexer process spawns a single +Dask worker, so only one indexing attempt runs at a time. Bump this when +you have many connectors and the queue keeps growing. The runtime guards +against the most dangerous race (two attempts for the same cc-pair +running concurrently) with two layers, both of which leave the attempt +in `NOT_STARTED` rather than marking it FAILED: + +1. **Scheduler-side defer** (`update.py::kickoff_indexing_jobs`) — before + submitting a NOT_STARTED attempt to Dask, check whether another + attempt for the same `(connector, credential, embedding_model)` is + already IN_PROGRESS. If yes, skip the submission this tick. +2. **Worker-side advisory lock** (`run_indexing_entrypoint` + + `try_acquire_cc_pair_lock`) — true-race safety net for the case where + two NOT_STARTED rows for the same cc-pair are submitted to two + workers in the same scheduler tick. If the lock fails, the worker + reverts the attempt back to `NOT_STARTED` (clears `time_started`) so + the next scheduler tick picks it up again. **No FAILED row is + produced.** The previous behaviour wrote `skipped_concurrent_cc_pair_run` + FAILED rows; this no longer happens. + +Three downstream things to scale alongside it: + +1. **Model server**: by default it runs as a single uvicorn process. Four + indexing workers calling `/encoder` simultaneously will queue. Run it + with workers: + ```bash + uvicorn model_server.main:app --port 9000 --workers 4 + ``` + (or set the worker count to roughly `NUM_INDEXING_WORKERS`). + +2. **Postgres `max_connections`**: each Dask worker is a subprocess with + its own SQLAlchemy connection pool (default 5 + 10 overflow). At + `NUM_INDEXING_WORKERS=4` the indexer process group alone may reach + ~75 connections; add the API server, Celery worker, and beat + scheduler and you can hit the Postgres default of `100`. Either lower + the per-process pool size or raise the Postgres limit: + ```sql + ALTER SYSTEM SET max_connections = 200; -- requires DB restart + ``` + +3. **Source-side rate limits**: 4 concurrent GitHub workers means PAT + rate limits hit 4× faster, same for Salesforce / Confluence / Jira + API quotas. The runtime caps per-source-group concurrency + independently of `NUM_INDEXING_WORKERS` — see the next section. So + even with `NUM_INDEXING_WORKERS=4`, GitHub's cap stays at 1 by + default, four GitHub repos still serialize, and the other workers + are free to run other sources. + +Both queueing decisions (per-cc-pair collision + per-source cap) leave +attempts as `NOT_STARTED` — neither produces FAILED rows. So the +indexing-status table never accumulates "skipped" failure rows for +routine deferral. Only real indexing errors show up as FAILED. + +#### Per-Source Indexing Concurrency Cap +Even with `NUM_INDEXING_WORKERS > 1`, you typically don't want N parallel +indexers all hammering the same external API — most sources rate-limit +per credential token. The generic rule the runtime enforces: + +> **At most `INDEXING_PER_SOURCE_CAP` indexing attempts per +> `DocumentSource` are submitted to Dask concurrently.** Default is `1`. + +So `NUM_INDEXING_WORKERS=4` + 4 different source types (Slack + GitHub + +Confluence + Jira) gives you a 4× speedup. Same `NUM_INDEXING_WORKERS=4` ++ 4 GitHub cc-pairs (same source) → 1 runs and the other 3 stay in +`NOT_STARTED`. The scheduler reconsiders them on each tick (every 10s); +the moment the running one finishes, the next NOT_STARTED row for that +source gets submitted. + +Adding a new connector requires no config change — every `DocumentSource` +gets its own slot pool automatically. Connectors that happen to share an +external credential token (e.g. `github` and `github_files` both use a +PAT) are *not* collapsed into a single bucket; if that distinction +matters for your rate limits, fold them into a single `DocumentSource` +rather than re-introducing a grouping layer. + +To disable the cap entirely (e.g. you genuinely have separate +credentials per cc-pair and want them parallel): + +```bash +export INDEXING_PER_SOURCE_CAP=0 +``` + +`0` = uncapped (only `NUM_INDEXING_WORKERS` and the per-cc-pair lock +constrain you). Default is `1`. Higher integers also work — `2` would +let two concurrent attempts per source type, etc. + +The cap is enforced in the scheduler +(`background/update.py::kickoff_indexing_jobs`). Before submitting each +NOT_STARTED attempt to Dask, the scheduler counts IN_PROGRESS attempts +for the same source; if that count is already at the cap, the attempt +is left as NOT_STARTED and reconsidered on the next tick. There are no +FAILED rows produced by capping — only a `Deferring indexing attempt +{id} ... cap of N reached` log line. + +#### Common Pitfalls +- **Stuck `cleanup_connector_credential_pair_*` task in `task_queue_jobs`**. + Almost always means the Celery worker spawned by `dev_run_background_jobs.py` + died at startup — the script's `subprocess.Popen` doesn't propagate child + exit codes, so worker boot errors only show up in the per-line WORKER: + output. Look for `Error: Invalid value for '-A' / '--app'` or `'TaskPool' + object has no attribute 'grow'`. Fix whatever's wrong, then restart the + script and the queued tasks drain in seconds. +- **Vespa unreachable on `index:8081`**. The backend resolves Vespa by the + DNS name `index` on the `danswer-stack_default` docker network. If you + ran Vespa via `docker run` (above), make sure both `--network` and + `--hostname` match. `docker network inspect danswer-stack_default` will + show whether Vespa is attached. +- **Frontend doesn't reflect a new connector tile or label**. Next.js's + `.next/cache` can hold stale module bundles after enum / source-list + edits. `rm -rf web/.next && (cd web && npm run dev)` from the repo root, + then hard-refresh. + +### Testing + +See [`TESTING.md`](./TESTING.md) for the full testing reference: the +four orchestrator scripts under `backend/scripts/` (`test_analytics_e2e.py`, +`test_features_e2e.py`, `test_celery_jobs_smoke.py`, `seed_test_data.py`), +their assertion checklists, manual UI smoke steps, stress-test profiles, +the seed-script knob reference, and troubleshooting. + +Quickest path to confidence — assumes a dev DB: + +```bash +cd backend +PYTHONPATH=$(pwd) python scripts/test_analytics_e2e.py --yes # ~30s, full pipeline +PYTHONPATH=$(pwd) python scripts/test_features_e2e.py --yes # ~5s, feature regressions +PYTHONPATH=$(pwd) python scripts/test_celery_jobs_smoke.py --yes # ~10s, broker→worker +``` + ### Formatting and Linting #### Backend For the backend, you'll need to setup pre-commit hooks (black / reorder-python-imports). diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000000..7883fbf8504 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,334 @@ +# TESTING.md + +Smoke tests for the analytics + retention + indexing pipelines, driven +by auto-generated data. Five pieces: + +- `backend/scripts/seed_test_data.py` — pumps in tagged synthetic data. +- `backend/scripts/test_analytics_e2e.py` — end-to-end orchestrator for + the analytics rollup + chat retention pipelines. +- `backend/scripts/test_features_e2e.py` — end-to-end orchestrator for + the rest of this session's features (priority ordering, index_attempt + retention, permission_sync_run terminal-only retention, resolved-button + feedback DB write). +- `backend/scripts/test_celery_jobs_smoke.py` — fires both daily Celery + tasks via `.delay()` against fresh dummy data and waits for the worker + to complete them. Proves the broker → worker → DB pipeline is alive + end-to-end (the same path beat uses for the 07:30 / 08:00 UTC daily + fires). Run this anytime you want confidence the worker is processing + tasks. +- This file — manual UI checklist + how to run each script. + +> **Run only against a dev / staging DB.** Both scripts write *and* delete +> rows. They print the target DB URL and ask for a `yes` confirmation +> unless `--yes` is passed. Tagging via `__test_seed__` is what keeps +> `--clean` from touching real rows, but the rollup truncation in the +> orchestrator's Phase 2a / Phase 8 is unconditional — don't run it +> against prod. + +--- + +## Quick start + +```bash +cd backend + +# 1. One-shot end-to-end test. Seeds, asserts, cleans up after itself. +PYTHONPATH=$(pwd) python scripts/test_analytics_e2e.py +# expected: every phase PASSES, exit 0. + +# 2. If you want to poke around the seeded data manually, keep it after. +PYTHONPATH=$(pwd) python scripts/test_analytics_e2e.py --keep-data --yes + +# 3. Tear it down later. +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --clean --yes +``` + +The orchestrator's last line is either `✅ All phases passed.` or +`❌ N assertion(s) failed.` with PASS/FAIL details for every step. + +```bash +# Companion suite for the non-analytics features (priority, index_attempt +# retention, permission_sync_run, resolved-feedback). Same shape, smaller +# scope. Self-contained — uses its own `__test_features__` tag prefix. +PYTHONPATH=$(pwd) python scripts/test_features_e2e.py --yes + +# Live broker + worker plumbing check — fires both daily Celery tasks +# via `.delay()`, waits for the worker to apply side effects, asserts. +# Uses its own `__test_celery__` tag prefix. ~10 seconds when the worker +# is healthy. If `.get()` hangs, the worker is dead. +PYTHONPATH=$(pwd) python scripts/test_celery_jobs_smoke.py --yes +``` + +--- + +## What the analytics orchestrator covers + +Each phase is documented with what it asserts in +`scripts/test_analytics_e2e.py`. In summary: + +| Phase | What it does | What it asserts | +|---|---|---| +| 1 | Clean + seed 60 days of chats + a small batch of 35–90d "old" chats + search_doc links + slack configs | Sessions / messages / old chats / search_docs all present | +| 2a | Truncate `analytics_daily_rollup` + delete the checkpoint row | (housekeeping) | +| 2b | Run `backfill_analytics_rollup.py` | Rollup table populated, checkpoint row written | +| 3 | Call `fetch_*_from_rollup` directly | ≥60 rows per series, NPS-strict computable, sums look sane | +| 4 | `cleanup_stale_db.py --dry-run --policy=chat` | Dry-run completes without error | +| 5 | `cleanup_stale_db.py --policy=chat` (real run) | Old chats gone, fresh chats untouched, orphan search_docs cleaned | +| 6 | Re-query rollup | Still ≥60 days of data — proves rollup survived retention | +| 7 | Re-run `run_rollup` | Idempotent, checkpoint advances to today | +| 8 | Final cleanup of seeded rows + rollup table + checkpoint | (cleanup) | + +If any phase fails, the script halts and exits non-zero. Re-running is +safe. + +## What the features orchestrator covers + +| Phase | What it does | What it asserts | +|---|---|---| +| 1 | Seed 5 NOT_STARTED `index_attempt` rows with priorities `[0, 5, 10, 0, 3]` and call `get_not_started_index_attempts` | Order is `[10, 5, 3, 0, 0]` (priority DESC), tiebreak is `time_created` ASC, `update_index_attempt_priority` clamps to ceiling=100 and refuses on IN_PROGRESS rows | +| 2 | Seed 25 SUCCESS `index_attempt` rows 70-94d old, run retention with `RETENTION_DAYS_INDEX_ATTEMPT=60` and `KEEP_LAST_N=20` | 5 oldest deleted, 20 newest kept. Acts as a regression check on the status-casing — the column stores enum NAMES (uppercase 'SUCCESS' / 'FAILED'), so the SQL must filter uppercase. A lowercase regression silently no-ops the policy | +| 3 | Seed 8 `permission_sync_run` rows (5 terminal + 3 in_progress, all 90d old), run retention | 5 terminal rows deleted, 3 in_progress preserved. Verifies the safety contract: stuck syncs aren't swept up by retention regardless of age | +| 4 | Synthetic call to `create_chat_message_feedback` mirroring the resolved-button handler signature (`predefined_feedback='resolved'`, `is_positive=None`, `user_id=None` against a slackbot-style session) | Exactly one feedback row is written with the right shape; `chat_message_id` matches the seeded message | + +Tagged with `__test_features__` (separate from `__test_seed__`) — the two +orchestrators don't interfere. + +## What the Celery smoke test covers + +| Step | What it does | Confidence gained | +|---|---|---| +| 1 | Seed 5 old chats (35-90d) + 5 fresh chats (≤6d), tagged `__test_celery__` | (setup) | +| 2 | Snapshot `analytics_daily_rollup` row count + `max(rolled_up_at)` + chat counts | Baseline for diff | +| 3 | `run_analytics_rollup_task.delay()` → `.get(timeout=120)` | Celery client → broker → worker → DB write path is alive for the rollup task body | +| 4 | `run_retention_policies_task.delay()` → `.get(timeout=300)` | Same pipeline alive for retention; worker can execute long(er) tasks | +| 5 | Re-snapshot, diff against before | `max(rolled_up_at)` advanced; 5 old chats deleted; 5 fresh chats untouched | + +If `.get()` hangs (script never exits), the worker isn't picking up +tasks. Check `kubectl exec ... -- tail -f /var/log/celery_worker.log` +and supervisord status — the same `'TaskPool' has no attribute 'grow'` +or `Invalid value for '-A'` failures we've hit before. + +--- + +## Seeding manually + +For one-off experiments without the full orchestrator: + +```bash +# Standard scenario: 60 days × 10 chats/day, mix of Slackbot + UI. +PYTHONPATH=$(pwd) python scripts/seed_test_data.py \ + --days=60 --chats-per-day=10 \ + --slackbot-share=0.7 \ + --feedback-rate=0.6 --like-share=0.5 --resolved-share=0.2 --needs-help-share=0.1 \ + --users=15 --connectors=4 --docs-per-connector=200 \ + --with-old-data --with-search-docs --yes + +# Heavy load (~5k chats over 90 days) +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --days=90 --chats-per-day=50 --yes + +# All-positive feedback (NPS should be ~+100) +PYTHONPATH=$(pwd) python scripts/seed_test_data.py \ + --feedback-rate=1.0 --like-share=1.0 --resolved-share=0.0 --needs-help-share=0.0 \ + --yes + +# All-negative (NPS ~-100) +PYTHONPATH=$(pwd) python scripts/seed_test_data.py \ + --feedback-rate=1.0 --like-share=0.0 --resolved-share=0.0 --needs-help-share=0.0 \ + --yes # remainder maps to dislikes +``` + +After seeding, populate the rollup: + +```bash +PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py --yes 2>/dev/null \ + || PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py +``` + +--- + +## Stress-test profiles (single-line so paste can't break) + +```bash +# Medium — 1 year × 50 chats/day, ~18k chats, 30k docs across 6 sources. +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --days=365 --chats-per-day=50 --users=100 --connectors=6 --docs-per-connector=5000 --with-old-data --with-search-docs && PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py + +# Heavy — 6 months × 200 chats/day, ~36k chats, 160k docs. +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --days=180 --chats-per-day=200 --users=500 --connectors=8 --docs-per-connector=20000 --with-search-docs && PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py + +# Massive — 1 year × 500 chats/day, ~180k chats. Slow seeder (~10 min). +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --days=365 --chats-per-day=500 --users=1000 --connectors=10 --docs-per-connector=50000 && PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py +``` + +The seeder doesn't dedupe — running it N times stacks N× users / chats / +connectors / configs. Useful for compounding without bumping any single +knob too high: + +```bash +for i in 1 2 3 4 5; do PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --seed=$i --days=90 --chats-per-day=100; done && PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py +``` + +### Caveat on what "stress" actually exercises + +The dashboard reads from `analytics_daily_rollup` — one row per UTC +day. So even 365 days × 500 chats/day still renders only ~365 chart +points. Bumping volume mostly stresses **(a)** the snapshot KPIs and +"Docs Indexed by Source" BarList, **(b)** the rollup backfill speed +(per-day SQL aggregation × N days), and **(c)** retention sweep volume +when `--with-old-data` is set. The time-series charts themselves render +the same regardless of underlying chat volume. + +### Edge-case scenarios + +```bash +# All-positive feedback → NPS-strict ≈ +100 +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --feedback-rate=1.0 --like-share=0.7 --resolved-share=0.3 --needs-help-share=0.0 + +# All-negative → NPS-strict ≈ -100 +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --feedback-rate=1.0 --like-share=0.0 --resolved-share=0.0 --needs-help-share=0.0 + +# Slackbot-only workspace +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --yes --slackbot-share=1.0 +``` + +### Quick state check after seeding + +```bash +psql "$POSTGRES_URL" -c "SELECT count(*), min(date), max(date), sum(total_queries), sum(slackbot_total) FROM analytics_daily_rollup;" +``` + +Should show `count` matching your `--days`, with non-zero sums. + +### Knob reference + +| Flag | Stresses | +|---|---| +| `--days=N` | Date-range slider, granularity toggle, rollup row count | +| `--chats-per-day=N` | Per-day SQL aggregation in backfill, total-queries KPI | +| `--users=N` | Active-users KPI, distinct-user counts | +| `--connectors=N` | Docs-per-source BarList width, total docs KPI | +| `--docs-per-connector=N` | Total Docs KPI, snapshot endpoint speed | +| `--with-search-docs` | Orphan-search_doc count after retention | +| `--with-old-data` | Retention sweep volume | +| `--slackbot-share=N` | Auto-resolution rate denominator | +| `--feedback-rate=N` | NPS denominator | +| `--like-share / --resolved-share / --needs-help-share` | NPS sign + magnitude | + +`--slack-configs` and `index attempts per cc-pair` aren't exposed as +flags yet — bump the literal counts in `seed_slack_bot_configs(..., +count=3)` / `seed_index_attempts(..., count_per_pair=4)` if you need +more, or just re-run the seeder. + +--- + +## Manual UI smoke checklist + +Open `/admin/analytics` after seeding (and after a backfill). Verify +visually: + +- [ ] Top KPI row shows non-zero numbers for **Total Queries**, **Peak + Daily Active Users**, **Auto-Resolution Rate**, **NPS — strict**. +- [ ] Second KPI row shows **Total Docs Indexed**, **Slack Channels + Enabled**, **Positive Feedback %**, **Sources Active**. +- [ ] **Users and Query Trend** chart renders with Queries + Active + Users series, both non-zero across the date range. +- [ ] **Feedback Trend** chart renders Likes + Dislikes. +- [ ] **Docs Indexed by Source** BarList shows per-source totals, + sorted DESC. +- [ ] Date range picker → **Last 7 days** → numbers shrink, charts + contract to 7 buckets. +- [ ] Date range picker → **Last 90 days** → charts expand. (Requires + seed `--days=90` or longer.) +- [ ] Granularity dropdown → **Monthly** → charts collapse to monthly + buckets. Subtitles update ("Monthly (Active Users = peak day)"). + Sums roughly match daily totals × num_days. +- [ ] No console errors in browser devtools. No 404 / 500 in network + tab. +- [ ] Empty range (e.g. far future date) → charts show + "No data in this date range" instead of erroring. + +--- + +## Manual Slack feedback verification + +The seed script doesn't simulate Slack button clicks. To verify the +**resolved-button feedback recording** end-to-end, do this manually +once after deploying to a connected Slack workspace: + +1. Have the bot answer a question in a channel it's configured for. +2. Click "I'm all set!" (the immediate-resolved button). +3. Check `chat_feedback`: + ```sql + SELECT id, chat_message_id, is_positive, predefined_feedback, + required_followup, feedback_text + FROM chat_feedback + ORDER BY id DESC LIMIT 5; + ``` + Latest row should have `predefined_feedback = 'resolved'` and the + right `chat_message_id`. +4. In a separate thread, click "I need more help", then have someone + click "Mark Resolved". +5. Two new feedback rows: one with `required_followup=true`, the next + with `predefined_feedback='resolved'` (latest wins for analytics — + the session counts as resolved). + +--- + +## Cleanup + +Always tag-scoped — won't touch real rows. + +```bash +PYTHONPATH=$(pwd) python scripts/seed_test_data.py --clean --yes +``` + +Also useful to reset the rollup pipeline state for a fresh run: + +```sql +TRUNCATE TABLE analytics_daily_rollup; +DELETE FROM key_value_store WHERE key = 'analytics_rollup_state'; +``` + +(The orchestrator does both as part of Phase 8 unless `--keep-data`.) + +--- + +## Troubleshooting + +**"No persona found" / "No embedding_model found":** the seeder needs +the default persona + embedding model rows to satisfy NOT NULL FKs. +Bootstrap by running `alembic upgrade head` and starting the API server +once — it loads default personas and the initial embedding model on +startup. Then re-run the seeder. + +**Phase 5 (real retention) failed with "old chats deleted":** check +your `RETENTION_DAYS_CHAT`. Default is 30. If you've changed it to +something larger than 35 days, the seeded "old" data (35–90 days old) +won't be eligible for deletion. Either lower `RETENTION_DAYS_CHAT` or +re-seed with older data: +```bash +# In seed_test_data.py::seed_old_chat_for_retention, the range is +# rng.randint(35, 90). Increase it if your retention is higher. +``` + +**Phase 7 (rollup idempotency) failed with "checkpoint advanced to +today":** UTC vs local time mismatch. The checkpoint stores ISO date +strings in UTC. Re-running near midnight UTC can flip the date between +phases. Re-run during the day; this is a benign artifact of the test, +not a real bug. + +**`ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS` exceeds `RETENTION_DAYS_CHAT - 2`:** +the rollup logs a warning and caps the window at the safe floor. +Expected — see the rollup module docstring. To suppress, lower +`ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS`. + +**Foreign-key violation on cleanup:** the `--clean` path deletes in +explicit FK order, but if you've manually inserted real data referencing +seeded rows (e.g. a real user added a feedback to a `__test_seed__` +chat), `--clean` will fail. Resolve the dangling reference manually, +then re-run `--clean`. + +**`document_by_connector_credential_pair` row count mismatch:** docs +are linked per cc-pair via `document_by_connector_credential_pair`; +the seeder inserts both the `document` row and the join row. If a +prior partial run left orphans, `--clean` removes them — both tables +are scoped by the `__test_seed__` prefix. diff --git a/backend/Dockerfile b/backend/Dockerfile index 71b3f02856d..a7a5039b6dd 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -94,6 +94,10 @@ COPY supervisord.conf /usr/etc/supervisord.conf # Escape hatch COPY ./scripts/force_delete_connector_by_id.py /app/scripts/force_delete_connector_by_id.py +# One-time analytics backfill (used by the analytics-bootstrap Job; +# safe to keep in the runtime image — it's idempotent and admin-run). +COPY ./scripts/backfill_analytics_rollup.py /app/scripts/backfill_analytics_rollup.py + # Put logo in assets COPY ./assets /app/assets diff --git a/backend/alembic/versions/9d02a9a5ce39_indexing_status_perf_indexes.py b/backend/alembic/versions/9d02a9a5ce39_indexing_status_perf_indexes.py new file mode 100644 index 00000000000..2b74c01d102 --- /dev/null +++ b/backend/alembic/versions/9d02a9a5ce39_indexing_status_perf_indexes.py @@ -0,0 +1,57 @@ +"""Performance indexes for /admin/connector/indexing-status + +Adds covering indexes to the two hot lookup paths used by the connector +indexing-status route: + +- (task_name, id DESC) on task_queue_jobs — used by `get_latest_task` and + the new bulk `get_latest_tasks_by_names`. +- (connector_id, credential_id, embedding_model_id, time_created DESC) + on index_attempt — used by `get_last_attempt` and the bulk + `get_latest_index_attempts` group-by/max pattern. + +These speed up "latest-per-group" queries from full-table scans to +index-only seeks. CONCURRENTLY would be safer in production, but Alembic +migrations run inside a transaction by default; the regular CREATE INDEX +is fine for dev / smaller deployments. Switch to CONCURRENTLY by hand if +the live tables are large enough that an exclusive write lock matters. + +Revision ID: 9d02a9a5ce39 +Revises: 792d1af3dc44 +Create Date: 2026-05-01 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "9d02a9a5ce39" +down_revision = "792d1af3dc44" +branch_labels: None = None +depends_on: None = None + + +def upgrade() -> None: + op.create_index( + "ix_task_queue_jobs_name_id", + "task_queue_jobs", + ["task_name", "id"], + unique=False, + postgresql_using="btree", + ) + op.create_index( + "ix_index_attempt_pair_model_time", + "index_attempt", + [ + "connector_id", + "credential_id", + "embedding_model_id", + "time_created", + ], + unique=False, + postgresql_using="btree", + ) + + +def downgrade() -> None: + op.drop_index("ix_index_attempt_pair_model_time", table_name="index_attempt") + op.drop_index("ix_task_queue_jobs_name_id", table_name="task_queue_jobs") diff --git a/backend/alembic/versions/b5d3f1a9e7c2_chat_ui_perf_indexes.py b/backend/alembic/versions/b5d3f1a9e7c2_chat_ui_perf_indexes.py new file mode 100644 index 00000000000..d93e1d08ef9 --- /dev/null +++ b/backend/alembic/versions/b5d3f1a9e7c2_chat_ui_perf_indexes.py @@ -0,0 +1,60 @@ +"""Performance indexes for chat UI hot paths + +Adds two FK-column indexes that the chat UI hits on every render. Both +columns are FKs in the schema but neither was indexed (Postgres doesn't +auto-index FK columns), so each chat-open / sidebar-list did a full +sequential scan. + +- chat_message(chat_session_id): every "open chat session" page hits + `WHERE chat_session_id = :id ORDER BY parent_message NULLS FIRST` + (db/chat.py::get_chat_messages_by_session). Without this index that's + a seq scan of every chat_message ever stored. Lazy-loaded relationships + (chat_message.tool_calls, .chat_message_feedbacks) make it worse. + +- chat_session(user_id): the chat-history sidebar hits + `WHERE user_id = :user_id` (db/chat.py::get_chat_sessions_by_user) on + every chat UI load. + +Both created CONCURRENTLY so production deployments don't take a write +lock on chat_message / chat_session during the build. CONCURRENTLY can't +run inside the migration's wrapping transaction, hence +`with op.get_context().autocommit_block()`. + +Revision ID: b5d3f1a9e7c2 +Revises: fd307e9ecc9b +Create Date: 2026-05-01 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "b5d3f1a9e7c2" +down_revision = "fd307e9ecc9b" +branch_labels: None = None +depends_on: None = None + + +def upgrade() -> None: + # Raw SQL to get both CONCURRENTLY and IF NOT EXISTS in one shot — + # `op.create_index` doesn't expose IF NOT EXISTS as a top-level + # kwarg, and passing `if_not_exists=` produces a SAWarning. The + # autocommit_block exits the migration's wrapping transaction so + # CONCURRENTLY is allowed. + with op.get_context().autocommit_block(): + op.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS " + "ix_chat_message_chat_session_id " + "ON chat_message USING btree (chat_session_id)" + ) + op.execute( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS " + "ix_chat_session_user_id " + "ON chat_session USING btree (user_id)" + ) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_chat_session_user_id") + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_chat_message_chat_session_id") diff --git a/backend/alembic/versions/c8a4e2f9d1b3_analytics_daily_rollup.py b/backend/alembic/versions/c8a4e2f9d1b3_analytics_daily_rollup.py new file mode 100644 index 00000000000..8f2eaa944e7 --- /dev/null +++ b/backend/alembic/versions/c8a4e2f9d1b3_analytics_daily_rollup.py @@ -0,0 +1,90 @@ +"""Daily analytics rollup table + +Persists pre-aggregated daily metrics so the admin analytics page +survives chat retention deletes. See `db/models.py::AnalyticsDailyRollup` +and `db/analytics_rollup.py` for the runtime contract. + +Date is the primary key — one row per UTC day. No FK to chat_message / +chat_session on purpose, so retention sweeps don't cascade into this +table. Indefinite retention. + +Revision ID: c8a4e2f9d1b3 +Revises: b5d3f1a9e7c2 +Create Date: 2026-05-01 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "c8a4e2f9d1b3" +down_revision = "b5d3f1a9e7c2" +branch_labels: None = None +depends_on: None = None + + +def upgrade() -> None: + op.create_table( + "analytics_daily_rollup", + sa.Column("date", sa.Date(), nullable=False), + sa.Column( + "total_queries", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "total_likes", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "total_dislikes", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "total_resolved", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "total_needs_help", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "active_users", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "slackbot_total", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "slackbot_auto_resolved", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "rolled_up_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.PrimaryKeyConstraint("date", name="analytics_daily_rollup_pkey"), + ) + + +def downgrade() -> None: + op.drop_table("analytics_daily_rollup") diff --git a/backend/alembic/versions/fd307e9ecc9b_index_attempt_priority.py b/backend/alembic/versions/fd307e9ecc9b_index_attempt_priority.py new file mode 100644 index 00000000000..061f5583fb3 --- /dev/null +++ b/backend/alembic/versions/fd307e9ecc9b_index_attempt_priority.py @@ -0,0 +1,51 @@ +"""Per-attempt indexing priority + +Adds `indexing_priority` to `index_attempt` so a single queued attempt can +jump the line ahead of other NOT_STARTED attempts (including others for +the same connector / cc-pair) without affecting any persistent connector +config. Default 0; conventional manual ceiling is 10. + +Also adds a helper index on (status, indexing_priority DESC, time_created) +so the scheduler's "pick the next attempt to dispatch" query stays fast +even with many queued rows. + +Revision ID: fd307e9ecc9b +Revises: 9d02a9a5ce39 +Create Date: 2026-05-01 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "fd307e9ecc9b" +down_revision = "9d02a9a5ce39" +branch_labels: None = None +depends_on: None = None + + +def upgrade() -> None: + op.add_column( + "index_attempt", + sa.Column( + "indexing_priority", + sa.Integer(), + nullable=False, + server_default="0", + ), + ) + # Speeds up `get_not_started_index_attempts` which now filters by + # status='NOT_STARTED' and orders by indexing_priority DESC, time_created ASC. + op.create_index( + "ix_index_attempt_status_priority_time", + "index_attempt", + ["status", "indexing_priority", "time_created"], + unique=False, + postgresql_using="btree", + ) + + +def downgrade() -> None: + op.drop_index("ix_index_attempt_status_priority_time", table_name="index_attempt") + op.drop_column("index_attempt", "indexing_priority") diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py index b1d0e7b9bad..cce46a9b9f2 100644 --- a/backend/danswer/background/celery/celery_app.py +++ b/backend/danswer/background/celery/celery_app.py @@ -2,6 +2,7 @@ from typing import cast from celery import Celery # type: ignore +from celery.schedules import crontab # type: ignore from sqlalchemy.orm import Session from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector @@ -18,6 +19,8 @@ from danswer.connectors.models import InputType from danswer.db.connector_credential_pair import get_connector_credential_pair from danswer.db.connector_credential_pair import get_connector_credential_pairs +from danswer.db.connector_credential_pair import release_deletion_lock +from danswer.db.connector_credential_pair import try_acquire_deletion_lock from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed from danswer.db.document import get_documents_for_connector_credential_pair from danswer.db.document import prepare_to_modify_documents @@ -63,38 +66,79 @@ def cleanup_connector_credential_pair_task( or updating the ACL""" engine = get_sqlalchemy_engine() with Session(engine) as db_session: - # validate that the connector / credential pair is deletable - cc_pair = get_connector_credential_pair( - db_session=db_session, - connector_id=connector_id, - credential_id=credential_id, - ) - if not cc_pair: - raise ValueError( - f"Cannot run deletion attempt - connector_credential_pair with Connector ID: " - f"{connector_id} and Credential ID: {credential_id} does not exist." + # Per-cc-pair deletion advisory lock. Without this guard, multiple + # `apply_async` dispatches (e.g. user clicking Delete several times, + # or an upstream caller retrying) all race on `SELECT ... FOR UPDATE + # NOWAIT` over the same documents in `prepare_to_modify_documents`. + # `NOWAIT` aborts on contention, the deletion code retries 10 × 30s + # = 5min, then raises `Failed to acquire locks after 10 attempts`. + # The first task usually succeeds; the others spin uselessly. The + # API-side dedup in `administrative.py` catches the common case, + # but this is the safety net (and the only thing protecting against + # any future caller that bypasses that endpoint). + if not try_acquire_deletion_lock(db_session, connector_id, credential_id): + logger.info( + f"Skipping deletion task for connector_id={connector_id}, " + f"credential_id={credential_id}: another worker is already " + "running a deletion for this cc-pair." ) - - deletion_attempt_disallowed_reason = check_deletion_attempt_is_allowed( - connector_credential_pair=cc_pair, db_session=db_session - ) - if deletion_attempt_disallowed_reason: - raise ValueError(deletion_attempt_disallowed_reason) - + return 0 try: - # The bulk of the work is in here, updates Postgres and Vespa - curr_ind_name, sec_ind_name = get_both_index_names(db_session) - document_index = get_default_document_index( - primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name - ) - return delete_connector_credential_pair( + # validate that the connector / credential pair is deletable + cc_pair = get_connector_credential_pair( db_session=db_session, - document_index=document_index, - cc_pair=cc_pair, + connector_id=connector_id, + credential_id=credential_id, ) - except Exception as e: - logger.exception(f"Failed to run connector_deletion due to {e}") - raise e + if not cc_pair: + raise ValueError( + f"Cannot run deletion attempt - connector_credential_pair with Connector ID: " + f"{connector_id} and Credential ID: {credential_id} does not exist." + ) + + deletion_attempt_disallowed_reason = check_deletion_attempt_is_allowed( + connector_credential_pair=cc_pair, db_session=db_session + ) + if deletion_attempt_disallowed_reason: + raise ValueError(deletion_attempt_disallowed_reason) + + try: + # The bulk of the work is in here, updates Postgres and Vespa + curr_ind_name, sec_ind_name = get_both_index_names(db_session) + document_index = get_default_document_index( + primary_index_name=curr_ind_name, + secondary_index_name=sec_ind_name, + ) + return delete_connector_credential_pair( + db_session=db_session, + document_index=document_index, + cc_pair=cc_pair, + ) + except Exception as e: + logger.exception(f"Failed to run connector_deletion due to {e}") + raise e + finally: + # Robustness: same trap as the indexing cc-pair lock — if the + # session is in an aborted-transaction state, the unlock SQL + # would also raise and the lock would ride the connection back + # into the SA pool, silently blocking every subsequent + # deletion task on this cc-pair. Rollback first, then unlock, + # then commit. If unlock still fails, the lock auto-releases + # when this connection drops out of the pool — bounded latency, + # not infinite leak. + try: + db_session.rollback() + except Exception: + pass + try: + release_deletion_lock(db_session, connector_id, credential_id) + db_session.commit() + except Exception: + logger.exception( + f"Could not release deletion lock for " + f"connector_id={connector_id} credential_id={credential_id}; " + "lock will release when this connection drops out of the SA pool." + ) @build_celery_task_wrapper(name_cc_prune_task) @@ -267,6 +311,56 @@ def check_for_document_sets_sync_task() -> None: ) +@celery_app.task( + name="run_analytics_rollup_task", + soft_time_limit=JOB_TIMEOUT, +) +def run_analytics_rollup_task() -> None: + """Daily rollup of admin analytics into `analytics_daily_rollup`. + + Must run BEFORE `run_retention_policies_task` so the rollup sees live + chat data. Default schedule: 07:30 UTC (retention runs at 08:00 UTC). + See backend/danswer/db/analytics_rollup.py for the pipeline + window + semantics. + """ + from danswer.db.analytics_rollup import run_rollup + + try: + n = run_rollup() + except Exception: + logger.exception( + "Analytics rollup failed; will retry on next schedule. " + "If this is the first run after deploy, ensure the migration " + "has been applied (alembic upgrade head)." + ) + raise + logger.info(f"Analytics rollup: upserted {n} day(s)") + + +@celery_app.task( + name="run_retention_policies_task", + soft_time_limit=JOB_TIMEOUT, +) +def run_retention_policies_task() -> None: + """Daily DB retention sweep. Deletes stale rows from + kombu_message, task_queue_jobs, index_attempt, and chat tables per + the policies defined in `danswer.db.retention`. Configured via + RETENTION_DAYS_* env vars; see backend/danswer/db/retention.py.""" + from danswer.db.retention import run_retention_policies + + try: + results = run_retention_policies() + except Exception: + logger.exception("Retention sweep raised; will retry on next schedule") + raise + total = sum(results.values()) + if total == 0: + logger.info("Retention sweep: nothing to delete this run") + else: + summary = ", ".join(f"{name}={n}" for name, n in results.items() if n > 0) + logger.info(f"Retention sweep: {total} rows deleted ({summary})") + + @celery_app.task( name="check_for_prune_task", soft_time_limit=JOB_TIMEOUT, @@ -311,3 +405,22 @@ def check_for_prune_task() -> None: }, } ) +celery_app.conf.beat_schedule.update( + { + # Daily analytics rollup — pre-aggregates admin metrics so the + # dashboard survives chat retention deletes. Runs 30 min BEFORE + # the retention sweep so chat data is still alive when we read. + # See backend/danswer/db/analytics_rollup.py. + "run-analytics-rollup": { + "task": "run_analytics_rollup_task", + "schedule": crontab(hour=7, minute=30), # 07:30 UTC daily + }, + # Daily DB retention sweep — kombu_message / task_queue_jobs / + # index_attempt / chat. Tunable via RETENTION_DAYS_* env vars. + # See backend/danswer/db/retention.py for the policies. + "run-retention": { + "task": "run_retention_policies_task", + "schedule": crontab(hour=8, minute=0), # 08:00 UTC daily + }, + } +) diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py index fa684f020b6..cf3f3eee993 100644 --- a/backend/danswer/background/indexing/run_indexing.py +++ b/backend/danswer/background/indexing/run_indexing.py @@ -22,6 +22,8 @@ from danswer.db.index_attempt import mark_attempt_failed from danswer.db.index_attempt import mark_attempt_in_progress__no_commit from danswer.db.index_attempt import mark_attempt_succeeded +from danswer.db.index_attempt import release_cc_pair_lock +from danswer.db.index_attempt import try_acquire_cc_pair_lock from danswer.db.index_attempt import update_docs_indexed from danswer.db.models import IndexAttempt from danswer.db.models import IndexingStatus @@ -307,7 +309,30 @@ def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexA def run_indexing_entrypoint(index_attempt_id: int, is_ee: bool = False) -> None: """Entrypoint for indexing run when using dask distributed. Wraps the actual logic in a `try` block so that we can catch any exceptions - and mark the attempt as failed.""" + and mark the attempt as failed. + + With NUM_INDEXING_WORKERS > 1, two workers can in rare cases pick up + attempts for the *same* (connector_id, credential_id) concurrently — + e.g. a manual Re-Index click colliding with an auto-scheduled run. To + prevent racing on Vespa writes / last_successful_index_time / connector + checkpoint state, we acquire a per-cc-pair Postgres advisory lock before + doing real work. If we can't acquire it, the attempt is **reverted to + NOT_STARTED** (status + time_started cleared) and the next scheduler + tick will dispatch it again. We deliberately do NOT mark FAILED — that + polluted the indexing-status table with spurious "skipped_concurrent_*" + rows for routine race conditions. + + The scheduler-side guard in `update.py::kickoff_indexing_jobs` defers + submission whenever an IN_PROGRESS attempt already exists for the same + cc-pair, so worker-side lock contention is now extremely rare — + effectively a safety net for the case where two attempts are + submitted to two workers in the same scheduler tick before either + flips to IN_PROGRESS. + + Per-source-type rate-limit caps are also enforced upstream in the + scheduler — over-cap attempts stay NOT_STARTED until a slot frees. + No source-cap fail-fast logic lives here. + """ try: if is_ee: global_version.set_ee() @@ -321,18 +346,91 @@ def run_indexing_entrypoint(index_attempt_id: int, is_ee: bool = False) -> None: # as in progress attempt = _prepare_index_attempt(db_session, index_attempt_id) - logger.info( - f"Running indexing attempt for connector: '{attempt.connector.name}', " - f"with config: '{attempt.connector.connector_specific_config}', and " - f"with credentials: '{attempt.credential_id}'" - ) + connector_id = attempt.connector_id + credential_id = attempt.credential_id + if connector_id is None or credential_id is None: + # Defensive: should never happen for a NOT_STARTED attempt that + # made it past _prepare_index_attempt, but the model allows + # nullable FKs. Fail fast with a useful message. + mark_attempt_failed( + attempt, + db_session, + failure_reason="connector_id or credential_id is null", + ) + return + + # Per-cc-pair concurrency guard. See module docstring for + # rationale. If we can't acquire the lock another worker + # already holds it for the same (connector, credential) pair — + # revert this attempt to NOT_STARTED so the scheduler picks it + # up again on its next tick (instead of marking it FAILED with + # a `skipped_concurrent_cc_pair_run` reason, which polluted + # the indexing-status table with spurious failure rows). + if not try_acquire_cc_pair_lock(db_session, connector_id, credential_id): + logger.info( + f"Re-queueing indexing attempt {index_attempt_id} for " + f"connector_id={connector_id} credential_id={credential_id}: " + "another worker holds the cc-pair lock. Reverting status " + "to NOT_STARTED — the scheduler will retry on its next tick." + ) + # _prepare_index_attempt already flipped the row to + # IN_PROGRESS (and committed for primary index). Roll it + # back to NOT_STARTED + clear time_started so the next + # pick-up looks like a fresh dispatch. + attempt.status = IndexingStatus.NOT_STARTED + attempt.time_started = None + db_session.add(attempt) + db_session.commit() + return - _run_indexing(db_session, attempt) + try: + priority_str = ( + f", priority: {attempt.indexing_priority}" + if attempt.indexing_priority + else "" + ) + logger.info( + f"Running indexing attempt for connector: '{attempt.connector.name}', " + f"with config: '{attempt.connector.connector_specific_config}', and " + f"with credentials: '{attempt.credential_id}'" + f"{priority_str}" + ) - logger.info( - f"Completed indexing attempt for connector: '{attempt.connector.name}', " - f"with config: '{attempt.connector.connector_specific_config}', and " - f"with credentials: '{attempt.credential_id}'" - ) + _run_indexing(db_session, attempt) + + logger.info( + f"Completed indexing attempt for connector: '{attempt.connector.name}', " + f"with config: '{attempt.connector.connector_specific_config}', and " + f"with credentials: '{attempt.credential_id}'" + ) + finally: + # Robustness: a failing `_run_indexing` (Vespa hiccup, Slack + # API blip, model-server timeout, anything that raises) can + # leave the SQLAlchemy session in an aborted-transaction + # state. The unlock SQL would then ALSO raise, the lock + # would NOT release, and the DB connection would go back + # to the SA pool with the advisory lock still held. The + # next worker that pulls that connection from the pool + # silently inherits the orphan — every subsequent task on + # that cc-pair sees phantom lock contention. Same trap we + # hit with the retention advisory lock. + # + # Fix: rollback to clear the aborted-txn state, THEN try + # the unlock; log-and-continue if even that fails (lock + # auto-releases when this connection drops out of the SA + # pool — bounded latency, not infinite leak). + try: + db_session.rollback() + except Exception: + pass + try: + release_cc_pair_lock(db_session, connector_id, credential_id) + db_session.commit() + except Exception: + logger.exception( + f"Could not release cc-pair lock for " + f"connector_id={connector_id} credential_id={credential_id}; " + "lock will release when this connection drops out of the SA pool." + ) except Exception as e: logger.exception(f"Indexing job with ID '{index_attempt_id}' failed due to {e}") diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py index 9ca65f8b33a..8d6f819fd52 100755 --- a/backend/danswer/background/update.py +++ b/backend/danswer/background/update.py @@ -1,6 +1,7 @@ import logging import time from datetime import datetime +from typing import Any import dask from dask.distributed import Client @@ -16,6 +17,7 @@ from danswer.configs.app_configs import DASK_JOB_CLIENT_ENABLED from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP from danswer.configs.app_configs import NUM_INDEXING_WORKERS +from danswer.configs.indexing_concurrency import PER_SOURCE_CAP from danswer.db.connector import fetch_connectors from danswer.db.embedding_model import get_current_db_embedding_model from danswer.db.embedding_model import get_secondary_db_embedding_model @@ -260,6 +262,91 @@ def cleanup_indexing_jobs( return existing_jobs_copy +_DISPATCH = "dispatch" +_DEFER_CC_PAIR = "defer_cc_pair" +_DEFER_SOURCE_CAP = "defer_source_cap" + + +def _build_running_view( + in_progress_rows: list[IndexAttempt], + dispatched_pre_completion_rows: list[IndexAttempt], + per_source_cap: int, +) -> tuple[dict[str, int], set[tuple[int | None, int | None, int]]]: + """Build the scheduler's running-attempt view. + + Counts BOTH: + - Attempts that have flipped to IN_PROGRESS in the DB. + - Attempts that the scheduler has already dispatched to Dask but + which haven't yet flipped to IN_PROGRESS (still queued in Dask + or with a worker spinning up). + + Counting only the DB view leaks the per-source cap: an attempt + sitting in Dask's queue for a few seconds doesn't register, and + the next scheduler tick can submit a second same-source attempt + past the cap. That was the actual cause of "two slack runs at + once despite cap=1" + "lower-priority attempt running while a + higher-priority attempt sits NOT_STARTED". + """ + running_per_source: dict[str, int] = {} + in_progress_cc_pair_keys: set[tuple[int | None, int | None, int]] = set() + accounted: set[int] = set() + for ip in in_progress_rows: + if ip.id in accounted: + continue + accounted.add(ip.id) + if ip.connector is None: + continue + in_progress_cc_pair_keys.add( + (ip.connector_id, ip.credential_id, ip.embedding_model_id) + ) + if per_source_cap > 0: + key = ip.connector.source.value + running_per_source[key] = running_per_source.get(key, 0) + 1 + for d in dispatched_pre_completion_rows: + if d.id in accounted: + continue + accounted.add(d.id) + if d.connector is None: + continue + in_progress_cc_pair_keys.add( + (d.connector_id, d.credential_id, d.embedding_model_id) + ) + if per_source_cap > 0: + key = d.connector.source.value + running_per_source[key] = running_per_source.get(key, 0) + 1 + return running_per_source, in_progress_cc_pair_keys + + +def _evaluate_dispatch_for_attempt( + attempt: IndexAttempt, + running_per_source: dict[str, int], + in_progress_cc_pair_keys: set[tuple[int | None, int | None, int]], + per_source_cap: int, +) -> str: + """Pure decision: should this attempt dispatch now, or defer? + + Mutates `running_per_source` and `in_progress_cc_pair_keys` only + when returning _DISPATCH, so subsequent same-tick iterations see + this attempt as in-flight. + """ + if attempt.connector is None: + return _DISPATCH # caller handles connector-null path separately + cc_pair_key = ( + attempt.connector_id, + attempt.credential_id, + attempt.embedding_model_id, + ) + if cc_pair_key in in_progress_cc_pair_keys: + return _DEFER_CC_PAIR + if per_source_cap > 0: + source_key = attempt.connector.source.value + if running_per_source.get(source_key, 0) >= per_source_cap: + return _DEFER_SOURCE_CAP + running_per_source[source_key] = running_per_source.get(source_key, 0) + 1 + in_progress_cc_pair_keys.add(cc_pair_key) + return _DISPATCH + + def kickoff_indexing_jobs( existing_jobs: dict[int, Future | SimpleJob], client: Client | SimpleJobClient, @@ -277,6 +364,43 @@ def kickoff_indexing_jobs( if attempt.id not in existing_jobs ] + # One DB read fuels two scheduler-side guards: + # (a) per-source-type concurrency cap — defer when the source + # group is full (`PER_SOURCE_CAP`). + # (b) per-cc-pair collision guard — defer when this exact + # (connector, credential, embedding_model) tuple already + # has an IN_PROGRESS attempt. Common case: a manual + # Re-Index click while an auto-scheduled run is mid-flight. + # Both deferrals leave the row as NOT_STARTED — never FAILED — + # so the indexing-status table doesn't get polluted with + # "skipped_concurrent_*" rows for routine queueing decisions. + in_progress_rows = ( + db_session.query(IndexAttempt) + .filter(IndexAttempt.status == IndexingStatus.IN_PROGRESS) + .all() + ) + accounted_attempt_ids = {ip.id for ip in in_progress_rows} + unaccounted_dispatched_ids = [ + aid for aid in existing_jobs if aid not in accounted_attempt_ids + ] + dispatched_pre_completion_rows: list[IndexAttempt] = [] + if unaccounted_dispatched_ids: + dispatched_pre_completion_rows = ( + db_session.query(IndexAttempt) + .filter(IndexAttempt.id.in_(unaccounted_dispatched_ids)) + .filter( + IndexAttempt.status.notin_( + [IndexingStatus.SUCCESS, IndexingStatus.FAILED] + ) + ) + .all() + ) + running_per_source, in_progress_cc_pair_keys = _build_running_view( + in_progress_rows, + dispatched_pre_completion_rows, + PER_SOURCE_CAP, + ) + logger.info(f"Found {len(new_indexing_attempts)} new indexing tasks.") if not new_indexing_attempts: @@ -307,28 +431,59 @@ def kickoff_indexing_jobs( ) continue + decision = _evaluate_dispatch_for_attempt( + attempt, + running_per_source, + in_progress_cc_pair_keys, + PER_SOURCE_CAP, + ) + if decision == _DEFER_CC_PAIR: + logger.info( + f"Deferring indexing attempt {attempt.id} for connector " + f"'{attempt.connector.name}': cc-pair already has an " + "IN_PROGRESS attempt. Will retry on next scheduler tick." + ) + continue + if decision == _DEFER_SOURCE_CAP: + logger.info( + f"Deferring indexing attempt {attempt.id} for connector " + f"'{attempt.connector.name}' " + f"(source={attempt.connector.source.value}): " + f"cap of {PER_SOURCE_CAP} reached. " + "Will retry on next scheduler tick." + ) + continue + + # Per-attempt indexing priority. SimpleJobClient ignores the kwarg; + # the real Dask Client honors it (higher number = scheduled first). + priority = int(attempt.indexing_priority or 0) + submit_kwargs: dict[str, Any] = {"pure": False} + if isinstance(client, Client): + submit_kwargs["priority"] = priority + if use_secondary_index: run = secondary_client.submit( run_indexing_entrypoint, attempt.id, global_version.get_is_ee_version(), - pure=False, + **submit_kwargs, ) else: run = client.submit( run_indexing_entrypoint, attempt.id, global_version.get_is_ee_version(), - pure=False, + **submit_kwargs, ) if run: secondary_str = "(secondary index) " if use_secondary_index else "" + priority_str = f", priority: {priority}" if priority else "" logger.info( f"Kicked off {secondary_str}" f"indexing attempt for connector: '{attempt.connector.name}', " f"with config: '{attempt.connector.connector_specific_config}', and " - f"with credentials: '{attempt.credential_id}'" + f"with credentials: '{attempt.credential_id}'{priority_str}" ) existing_jobs_copy[attempt.id] = run diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index db445958e61..a47d8205c77 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -74,6 +74,7 @@ class DocumentSource(str, Enum): GMAIL = "gmail" REQUESTTRACKER = "requesttracker" GITHUB = "github" + GITHUB_FILES = "github_files" GITLAB = "gitlab" GURU = "guru" BOOKSTACK = "bookstack" diff --git a/backend/danswer/configs/indexing_concurrency.py b/backend/danswer/configs/indexing_concurrency.py new file mode 100644 index 00000000000..5ddf5e22730 --- /dev/null +++ b/backend/danswer/configs/indexing_concurrency.py @@ -0,0 +1,47 @@ +"""Per-source-type indexing concurrency cap. + +Generic rule: when `NUM_INDEXING_WORKERS > 1`, only `INDEXING_PER_SOURCE_CAP` +indexing attempts run per `DocumentSource` at a time (default 1). So 4 +workers + 4 different source types (e.g. Slack + Github + Confluence + +Jira) means 4× speedup. 4 workers + 4 GitHub cc-pairs (same source type) +means 1 runs and the other 3 stay NOT_STARTED until the running one +finishes; the next 10s scheduler tick reconsiders them. + +Adding a new connector requires nothing here — every `DocumentSource` +gets its own slot pool automatically. Connectors that *share* an external +credential (e.g. github + github_files share a PAT) are not currently +collapsed into a single bucket; if you find that distinction matters in +practice, fold them into a single `DocumentSource` rather than reintroduce +a grouping layer. + +The cap is enforced *upstream* in the scheduler — see +`background/update.py::kickoff_indexing_jobs`. Over-cap NOT_STARTED rows +are simply not submitted to Dask on a given tick; the next tick picks +them up once a slot frees. Workers don't see the cap at all — there's no +fail-fast or `skipped_*` error_msg associated with this cap. + +To disable capping entirely (e.g. you genuinely want N parallel indexers +of the same source type because each cc-pair has its own credential): + + export INDEXING_PER_SOURCE_CAP=0 +""" +from __future__ import annotations + +import os + + +def _resolve_cap() -> int: + raw = os.environ.get("INDEXING_PER_SOURCE_CAP", "").strip() + if not raw: + return 1 + try: + return max(0, int(raw)) + except ValueError: + return 1 + + +# Concurrent attempts per source type. 1 = at most one indexing attempt +# per `DocumentSource` at a time (the generic rule). 0 = uncapped (skip +# the slot logic entirely; rely solely on the per-cc-pair lock + +# NUM_INDEXING_WORKERS). +PER_SOURCE_CAP: int = _resolve_cap() diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 2f885bf2b7d..3c205314503 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -15,6 +15,7 @@ from danswer.connectors.dropbox.connector import DropboxConnector from danswer.connectors.file.connector import LocalFileConnector from danswer.connectors.github.connector import GithubConnector +from danswer.connectors.github_files.connector import GithubFilesConnector from danswer.connectors.gitlab.connector import GitlabConnector from danswer.connectors.gmail.connector import GmailConnector from danswer.connectors.gong.connector import GongConnector @@ -64,6 +65,7 @@ def identify_connector_class( InputType.POLL: SlackPollConnector, }, DocumentSource.GITHUB: GithubConnector, + DocumentSource.GITHUB_FILES: GithubFilesConnector, DocumentSource.GMAIL: GmailConnector, DocumentSource.GITLAB: GitlabConnector, DocumentSource.GOOGLE_DRIVE: GoogleDriveConnector, diff --git a/backend/danswer/connectors/github/connector.py b/backend/danswer/connectors/github/connector.py index 89e5de551f6..9eb58650443 100644 --- a/backend/danswer/connectors/github/connector.py +++ b/backend/danswer/connectors/github/connector.py @@ -7,9 +7,11 @@ from typing import cast from github import Github +from github import GithubException from github import RateLimitExceededException from github import Repository from github.Issue import Issue +from github.NamedUser import NamedUser from github.PaginatedList import PaginatedList from github.PullRequest import PullRequest @@ -20,6 +22,7 @@ from danswer.connectors.interfaces import LoadConnector from danswer.connectors.interfaces import PollConnector from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import BasicExpertInfo from danswer.connectors.models import ConnectorMissingCredentialError from danswer.connectors.models import Document from danswer.connectors.models import Section @@ -31,6 +34,7 @@ _MAX_NUM_RATE_LIMIT_RETRIES = 5 +_ITEMS_PER_PAGE = 100 # GitHub max; defaults to 30 if unset def _sleep_after_rate_limit_exception(github_client: Github) -> None: @@ -38,8 +42,9 @@ def _sleep_after_rate_limit_exception(github_client: Github) -> None: tzinfo=timezone.utc ) - datetime.now(tz=timezone.utc) sleep_time += timedelta(minutes=1) # add an extra minute just to be safe - logger.info(f"Ran into Github rate-limit. Sleeping {sleep_time.seconds} seconds.") - time.sleep(sleep_time.seconds) + sleep_seconds = max(0, int(sleep_time.total_seconds())) + logger.info(f"Ran into Github rate-limit. Sleeping {sleep_seconds} seconds.") + time.sleep(sleep_seconds) def _get_batch_rate_limited( @@ -80,39 +85,141 @@ def _batch_github_objects( yield mini_batch +def _safe_user_login(user: NamedUser | None) -> str | None: + if user is None: + return None + try: + return user.login + except GithubException: + return None + + +def _basic_expert_from_user(user: NamedUser | None) -> BasicExpertInfo | None: + """Build a BasicExpertInfo from a github user, with light tolerance for + missing fields (deleted users, ghost commits).""" + if user is None: + return None + try: + login = user.login + except GithubException: + return None + name: str | None = None + email: str | None = None + try: + name = user.name + except GithubException: + pass + try: + email = user.email + except GithubException: + pass + return BasicExpertInfo(display_name=name or login, email=email) + + +def _utc(dt: datetime | None) -> str | None: + if dt is None: + return None + return dt.replace(tzinfo=timezone.utc).isoformat() + + def _convert_pr_to_document(pull_request: PullRequest) -> Document: + metadata: dict[str, str | list[str]] = { + "object_type": "PullRequest", + "number": str(pull_request.number), + "state": pull_request.state, + "merged": str(pull_request.merged), + "num_commits": str(pull_request.commits), + "num_files_changed": str(pull_request.changed_files), + } + if author := _safe_user_login(pull_request.user): + metadata["author"] = author + if merged_by := _safe_user_login(pull_request.merged_by): + metadata["merged_by"] = merged_by + if assignees := [ + login + for login in (_safe_user_login(a) for a in pull_request.assignees) + if login + ]: + metadata["assignees"] = assignees + if labels := [label.name for label in pull_request.labels]: + metadata["labels"] = labels + if pull_request.base and pull_request.base.repo: + metadata["repo"] = pull_request.base.repo.full_name + if created := _utc(pull_request.created_at): + metadata["created_at"] = created + if closed := _utc(pull_request.closed_at): + metadata["closed_at"] = closed + if merged_at := _utc(pull_request.merged_at): + metadata["merged_at"] = merged_at + + primary_owner = _basic_expert_from_user(pull_request.user) + return Document( id=pull_request.html_url, sections=[Section(link=pull_request.html_url, text=pull_request.body or "")], source=DocumentSource.GITHUB, - semantic_identifier=pull_request.title, + semantic_identifier=f"{pull_request.number}: {pull_request.title}", # updated_at is UTC time but is timezone unaware, explicitly add UTC # as there is logic in indexing to prevent wrong timestamped docs # due to local time discrepancies with UTC doc_updated_at=pull_request.updated_at.replace(tzinfo=timezone.utc), - metadata={ - "merged": str(pull_request.merged), - "state": pull_request.state, - }, + primary_owners=[primary_owner] if primary_owner else None, + metadata=metadata, ) def _fetch_issue_comments(issue: Issue) -> str: - comments = issue.get_comments() - return "\nComment: ".join(comment.body for comment in comments) + try: + comments = issue.get_comments() + return "\nComment: ".join(comment.body for comment in comments if comment.body) + except GithubException as e: + logger.warning(f"Failed to fetch comments for issue #{issue.number}: {e}") + return "" def _convert_issue_to_document(issue: Issue) -> Document: + body = issue.body or "" + comments_text = _fetch_issue_comments(issue) + if comments_text: + full_text = ( + f"{body}\nComment: {comments_text}" if body else f"Comment: {comments_text}" + ) + else: + full_text = body + + metadata: dict[str, str | list[str]] = { + "object_type": "Issue", + "number": str(issue.number), + "state": issue.state, + } + if author := _safe_user_login(issue.user): + metadata["author"] = author + if closer := _safe_user_login(issue.closed_by): + metadata["closed_by"] = closer + if assignees := [ + login for login in (_safe_user_login(a) for a in issue.assignees) if login + ]: + metadata["assignees"] = assignees + if labels := [label.name for label in issue.labels]: + metadata["labels"] = labels + if issue.repository: + metadata["repo"] = issue.repository.full_name + if created := _utc(issue.created_at): + metadata["created_at"] = created + if closed := _utc(issue.closed_at): + metadata["closed_at"] = closed + + primary_owner = _basic_expert_from_user(issue.user) + return Document( id=issue.html_url, - sections=[Section(link=issue.html_url, text=issue.body or "")], + sections=[Section(link=issue.html_url, text=full_text)], source=DocumentSource.GITHUB, - semantic_identifier=issue.title, + semantic_identifier=f"{issue.number}: {issue.title}", # updated_at is UTC time but is timezone unaware doc_updated_at=issue.updated_at.replace(tzinfo=timezone.utc), - metadata={ - "state": issue.state, - }, + primary_owners=[primary_owner] if primary_owner else None, + metadata=metadata, ) @@ -120,14 +227,18 @@ class GithubConnector(LoadConnector, PollConnector): def __init__( self, repo_owner: str, - repo_name: str, + repo_name: str = "", batch_size: int = INDEX_BATCH_SIZE, state_filter: str = "all", include_prs: bool = True, include_issues: bool = False, ) -> None: self.repo_owner = repo_owner - self.repo_name = repo_name + # repo_name semantics: + # "" -> index every repo the owner has access to + # "foo" -> single repo (legacy shape, unchanged) + # "foo,bar,..."-> comma-separated list of repo names + self.repo_name = repo_name or "" self.batch_size = batch_size self.state_filter = state_filter self.include_prs = include_prs @@ -135,17 +246,16 @@ def __init__( self.github_client: Github | None = None def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + token = credentials["github_access_token"] self.github_client = ( - Github( - credentials["github_access_token"], base_url=GITHUB_CONNECTOR_BASE_URL - ) + Github(token, base_url=GITHUB_CONNECTOR_BASE_URL, per_page=_ITEMS_PER_PAGE) if GITHUB_CONNECTOR_BASE_URL - else Github(credentials["github_access_token"]) + else Github(token, per_page=_ITEMS_PER_PAGE) ) return None def _get_github_repo( - self, github_client: Github, attempt_num: int = 0 + self, github_client: Github, full_name: str, attempt_num: int = 0 ) -> Repository.Repository: if attempt_num > _MAX_NUM_RATE_LIMIT_RETRIES: raise RuntimeError( @@ -153,10 +263,48 @@ def _get_github_repo( ) try: - return github_client.get_repo(f"{self.repo_owner}/{self.repo_name}") + return github_client.get_repo(full_name) + except RateLimitExceededException: + _sleep_after_rate_limit_exception(github_client) + return self._get_github_repo(github_client, full_name, attempt_num + 1) + + def _get_all_repos( + self, github_client: Github, attempt_num: int = 0 + ) -> list[Repository.Repository]: + if attempt_num > _MAX_NUM_RATE_LIMIT_RETRIES: + raise RuntimeError( + "Re-tried fetching repos too many times. Something is going wrong with fetching objects from Github" + ) + try: + try: + org = github_client.get_organization(self.repo_owner) + return list(org.get_repos()) + except GithubException: + user = github_client.get_user(self.repo_owner) + return list(user.get_repos()) except RateLimitExceededException: _sleep_after_rate_limit_exception(github_client) - return self._get_github_repo(github_client, attempt_num + 1) + return self._get_all_repos(github_client, attempt_num + 1) + + def _resolve_repos(self) -> list[Repository.Repository]: + if self.github_client is None: + raise ConnectorMissingCredentialError("GitHub") + + if not self.repo_name: + return self._get_all_repos(self.github_client) + + names = [n.strip() for n in self.repo_name.split(",") if n.strip()] + repos: list[Repository.Repository] = [] + for name in names: + try: + repos.append( + self._get_github_repo( + self.github_client, f"{self.repo_owner}/{name}" + ) + ) + except GithubException as e: + logger.warning(f"Could not fetch repo {self.repo_owner}/{name}: {e}") + return repos def _fetch_from_github( self, start: datetime | None = None, end: datetime | None = None @@ -164,47 +312,56 @@ def _fetch_from_github( if self.github_client is None: raise ConnectorMissingCredentialError("GitHub") - repo = self._get_github_repo(self.github_client) - - if self.include_prs: - pull_requests = repo.get_pulls( - state=self.state_filter, sort="updated", direction="desc" - ) - - for pr_batch in _batch_github_objects( - pull_requests, self.github_client, self.batch_size - ): - doc_batch: list[Document] = [] - for pr in pr_batch: - if start is not None and pr.updated_at < start: + for repo in self._resolve_repos(): + logger.info(f"Indexing repo: {repo.full_name}") + + if self.include_prs: + pull_requests = repo.get_pulls( + state=self.state_filter, sort="updated", direction="desc" + ) + + stop_outer = False + for pr_batch in _batch_github_objects( + pull_requests, self.github_client, self.batch_size + ): + doc_batch: list[Document] = [] + for pr in pr_batch: + if start is not None and pr.updated_at < start: + stop_outer = True + break + if end is not None and pr.updated_at > end: + continue + doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr))) + if doc_batch: yield doc_batch - return - if end is not None and pr.updated_at > end: - continue - doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr))) - yield doc_batch - - if self.include_issues: - issues = repo.get_issues( - state=self.state_filter, sort="updated", direction="desc" - ) - - for issue_batch in _batch_github_objects( - issues, self.github_client, self.batch_size - ): - doc_batch = [] - for issue in issue_batch: - issue = cast(Issue, issue) - if start is not None and issue.updated_at < start: + if stop_outer: + break + + if self.include_issues: + issues = repo.get_issues( + state=self.state_filter, sort="updated", direction="desc" + ) + + stop_outer = False + for issue_batch in _batch_github_objects( + issues, self.github_client, self.batch_size + ): + doc_batch = [] + for issue in issue_batch: + issue = cast(Issue, issue) + if start is not None and issue.updated_at < start: + stop_outer = True + break + if end is not None and issue.updated_at > end: + continue + if issue.pull_request is not None: + # PRs are handled separately + continue + doc_batch.append(_convert_issue_to_document(issue)) + if doc_batch: yield doc_batch - return - if end is not None and issue.updated_at > end: - continue - if issue.pull_request is not None: - # PRs are handled separately - continue - doc_batch.append(_convert_issue_to_document(issue)) - yield doc_batch + if stop_outer: + break def load_from_state(self) -> GenerateDocumentsOutput: return self._fetch_from_github() @@ -212,15 +369,17 @@ def load_from_state(self) -> GenerateDocumentsOutput: def poll_source( self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch ) -> GenerateDocumentsOutput: - start_datetime = datetime.utcfromtimestamp(start) - end_datetime = datetime.utcfromtimestamp(end) + start_datetime = datetime.fromtimestamp(start, tz=timezone.utc).replace( + tzinfo=None + ) + end_datetime = datetime.fromtimestamp(end, tz=timezone.utc).replace(tzinfo=None) # Move start time back by 3 hours, since some Issues/PRs are getting dropped # Could be due to delayed processing on GitHub side # The non-updated issues since last poll will be shortcut-ed and not embedded adjusted_start_datetime = start_datetime - timedelta(hours=3) - epoch = datetime.utcfromtimestamp(0) + epoch = datetime(1970, 1, 1) if adjusted_start_datetime < epoch: adjusted_start_datetime = epoch @@ -232,7 +391,7 @@ def poll_source( connector = GithubConnector( repo_owner=os.environ["REPO_OWNER"], - repo_name=os.environ["REPO_NAME"], + repo_name=os.environ.get("REPO_NAME", ""), ) connector.load_credentials( {"github_access_token": os.environ["GITHUB_ACCESS_TOKEN"]} diff --git a/backend/danswer/connectors/github_files/__init__.py b/backend/danswer/connectors/github_files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/connectors/github_files/connector.py b/backend/danswer/connectors/github_files/connector.py new file mode 100644 index 00000000000..0d890923437 --- /dev/null +++ b/backend/danswer/connectors/github_files/connector.py @@ -0,0 +1,290 @@ +"""GitHub Files connector — indexes files (default: JSON) sitting at a fixed +depth under a configurable path prefix. + +Matches the layout + + // + +i.e. exactly one folder under the prefix, file directly inside that folder. +Default settings target a service-catalog layout: + + service-catalog/products//.json + +Anything deeper or shallower is skipped, as are files at intermediate +directories. The connector reuses the existing GitHub access token credential +shape, so users don't need to re-enter their PAT. +""" +import time +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from typing import Any + +from github import Github +from github import GithubException +from github import RateLimitExceededException + +from danswer.configs.app_configs import GITHUB_CONNECTOR_BASE_URL +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.utils.logger import setup_logger + + +logger = setup_logger() + + +_MAX_NUM_RATE_LIMIT_RETRIES = 5 +_ITEMS_PER_PAGE = 100 +_DEFAULT_PATH_PREFIX = "service-catalog/products" +_DEFAULT_FILE_EXTENSION = ".json" + + +def _sleep_after_rate_limit_exception(github_client: Github) -> None: + sleep_time = github_client.get_rate_limit().core.reset.replace( + tzinfo=timezone.utc + ) - datetime.now(tz=timezone.utc) + sleep_time += timedelta(minutes=1) + sleep_seconds = max(0, int(sleep_time.total_seconds())) + logger.info(f"Hit GitHub rate-limit. Sleeping {sleep_seconds}s.") + time.sleep(sleep_seconds) + + +def _retry_on_rate_limit(github_client: Github, fn, *args, **kwargs): + """Run `fn(*args, **kwargs)` retrying on RateLimitExceededException.""" + for attempt in range(_MAX_NUM_RATE_LIMIT_RETRIES + 1): + try: + return fn(*args, **kwargs) + except RateLimitExceededException: + if attempt >= _MAX_NUM_RATE_LIMIT_RETRIES: + raise + _sleep_after_rate_limit_exception(github_client) + raise RuntimeError("unreachable") + + +class GithubFilesConnector(LoadConnector, PollConnector): + def __init__( + self, + repo_owner: str, + repo_name: str, + path_prefix: str = _DEFAULT_PATH_PREFIX, + file_extension: str = _DEFAULT_FILE_EXTENSION, + branch: str = "", + batch_size: int = INDEX_BATCH_SIZE, + ) -> None: + self.repo_owner = repo_owner + self.repo_name = repo_name + self.path_prefix = path_prefix.strip("/") + # Normalize: caller can pass "json" or ".json" + self.file_extension = ( + file_extension if file_extension.startswith(".") else f".{file_extension}" + ).lower() + self.branch = branch or "" # empty -> use repo's default branch + self.batch_size = batch_size + self.github_client: Github | None = None + + def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + token = credentials["github_access_token"] + self.github_client = ( + Github(token, base_url=GITHUB_CONNECTOR_BASE_URL, per_page=_ITEMS_PER_PAGE) + if GITHUB_CONNECTOR_BASE_URL + else Github(token, per_page=_ITEMS_PER_PAGE) + ) + return None + + def _open_repo(self): + if self.github_client is None: + raise ConnectorMissingCredentialError("GitHub") + return _retry_on_rate_limit( + self.github_client, + self.github_client.get_repo, + f"{self.repo_owner}/{self.repo_name}", + ) + + def _resolve_branch(self, repo) -> str: + return self.branch or repo.default_branch + + def _list_matching_paths(self, repo, branch: str) -> list[tuple[str, str]]: + """Walk the git tree once, returning (path, blob_sha) pairs for files + matching `//`.""" + branch_obj = _retry_on_rate_limit(self.github_client, repo.get_branch, branch) + head_sha = branch_obj.commit.sha + tree = _retry_on_rate_limit( + self.github_client, repo.get_git_tree, head_sha, True + ) + + prefix = self.path_prefix + expected_depth = len(prefix.split("/")) + 2 if prefix else 2 + + results: list[tuple[str, str]] = [] + for element in tree.tree: + if element.type != "blob": + continue + path = element.path + if prefix and not path.startswith(prefix + "/"): + continue + parts = path.split("/") + if len(parts) != expected_depth: + continue + if not path.lower().endswith(self.file_extension): + continue + results.append((path, element.sha)) + + if tree.raw_data.get("truncated"): + logger.warning( + "Git tree was truncated by GitHub. Some files in the configured " + "path may have been missed. Consider narrowing the prefix or " + "indexing the repo via the standard GitHub connector instead." + ) + + return results + + def _fetch_blob_text(self, repo, path: str, branch: str) -> str | None: + try: + content = _retry_on_rate_limit( + self.github_client, repo.get_contents, path, branch + ) + except GithubException as e: + logger.warning(f"Failed to fetch {path}: {e}") + return None + try: + return content.decoded_content.decode("utf-8") + except UnicodeDecodeError: + logger.warning(f"Skipping non-UTF8 file: {path}") + return None + + def _convert_to_document( + self, + repo, + path: str, + sha: str, + text: str, + branch: str, + doc_updated_at: datetime, + ) -> Document: + parts = path.split("/") + product_dir = parts[-2] if len(parts) >= 2 else "" + filename = parts[-1] + html_url = f"https://github.com/{repo.full_name}/blob/{branch}/{path}" + # Use the blob SHA in the document id so a content change always + # produces a fresh id and the indexer treats it as a new version, + # while unchanged files stay deduped across runs. + doc_id = f"{html_url}@{sha}" + + return Document( + id=doc_id, + sections=[Section(link=html_url, text=text)], + source=DocumentSource.GITHUB_FILES, + semantic_identifier=f"{product_dir}/{filename}" + if product_dir + else filename, + doc_updated_at=doc_updated_at, + metadata={ + "repo": repo.full_name, + "path": path, + "product": product_dir, + "branch": branch, + "blob_sha": sha, + }, + ) + + def _fetch_documents( + self, + start: datetime | None = None, + end: datetime | None = None, + ) -> GenerateDocumentsOutput: + if self.github_client is None: + raise ConnectorMissingCredentialError("GitHub") + + repo = self._open_repo() + branch = self._resolve_branch(repo) + + # If polling, check whether any commit on this branch (within the + # configured prefix) landed in the window before doing real work. + if start is not None or end is not None: + try: + commits_iter = repo.get_commits( + sha=branch, + path=self.path_prefix or None, + **({"since": start} if start else {}), + **({"until": end} if end else {}), + ) + # Materializing one element triggers the API call without + # paging through everything. + first = next(iter(commits_iter), None) + if first is None: + logger.info( + f"No commits affecting '{self.path_prefix}' on branch " + f"'{branch}' between {start} and {end}; skipping fetch." + ) + return + except GithubException as e: + # Don't fail the run if the commit check 422's (e.g. empty path); + # fall through and just re-fetch. + logger.warning( + f"Commit-window check failed ({e}); re-fetching all matching files." + ) + + matches = self._list_matching_paths(repo, branch) + logger.info( + f"GitHub-Files: found {len(matches)} matching file(s) under " + f"'{self.path_prefix}' on {repo.full_name}@{branch}." + ) + + # Use the most recent commit on the branch as doc_updated_at — gives + # a real timestamp without per-file commit lookups. + try: + head_commit = _retry_on_rate_limit( + self.github_client, repo.get_branch, branch + ).commit + commit_dt = head_commit.commit.author.date + if commit_dt.tzinfo is None: + commit_dt = commit_dt.replace(tzinfo=timezone.utc) + except Exception: + commit_dt = datetime.now(tz=timezone.utc) + + doc_batch: list[Document] = [] + for path, sha in matches: + text = self._fetch_blob_text(repo, path, branch) + if text is None: + continue + doc_batch.append( + self._convert_to_document(repo, path, sha, text, branch, commit_dt) + ) + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + if doc_batch: + yield doc_batch + + def load_from_state(self) -> GenerateDocumentsOutput: + return self._fetch_documents() + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + start_dt = datetime.fromtimestamp(start, tz=timezone.utc) + end_dt = datetime.fromtimestamp(end, tz=timezone.utc) + return self._fetch_documents(start=start_dt, end=end_dt) + + +if __name__ == "__main__": + import os + + connector = GithubFilesConnector( + repo_owner=os.environ["REPO_OWNER"], + repo_name=os.environ["REPO_NAME"], + path_prefix=os.environ.get("PATH_PREFIX", _DEFAULT_PATH_PREFIX), + file_extension=os.environ.get("FILE_EXTENSION", _DEFAULT_FILE_EXTENSION), + branch=os.environ.get("BRANCH", ""), + ) + connector.load_credentials( + {"github_access_token": os.environ["GITHUB_ACCESS_TOKEN"]} + ) + print(next(connector.load_from_state())) diff --git a/backend/danswer/connectors/salesforce/connector.py b/backend/danswer/connectors/salesforce/connector.py index 03326df4efd..81ae624d374 100644 --- a/backend/danswer/connectors/salesforce/connector.py +++ b/backend/danswer/connectors/salesforce/connector.py @@ -1,11 +1,10 @@ import os -from collections.abc import Iterator from datetime import datetime from datetime import timezone from typing import Any +from typing import Tuple -from simple_salesforce import Salesforce -from simple_salesforce import SFType +import requests from danswer.configs.app_configs import INDEX_BATCH_SIZE from danswer.configs.constants import DocumentSource @@ -22,9 +21,97 @@ from danswer.connectors.salesforce.utils import extract_dict_text from danswer.utils.logger import setup_logger -DEFAULT_PARENT_OBJECT_TYPES = ["Account"] -MAX_QUERY_LENGTH = 10000 # max query length is 20,000 characters ID_PREFIX = "SALESFORCE_" +DEFAULT_SF_LOGIN_URL = "https://login.salesforce.com" +SF_API_VERSION = "v59.0" + +# Account fields to index, as (api_path, friendly_label) pairs. +# - api_path is what Salesforce sees in the SELECT clause (SOQL doesn't +# support column aliasing in non-aggregate queries, so renaming has to +# happen client-side). +# - friendly_label is what appears as the key in the indexed text. Use +# dot traversal in the api_path for lookup relationships (e.g. CSM__r.Name) +# and the leaf value gets hoisted to the top level under the friendly +# label, flattening the JSON. +# +# To re-verify against your org's actual fields, run +# `backend/scripts/list_salesforce_account_fields.py`. +ACCOUNT_FIELDS: list[tuple[str, str]] = [ + # Identity / naming + ("Id", "Account Id"), + ("Name", "Account Name"), + ("Legal__c", "Legal Entity Name"), + ("Business_Name__c", "Business Name"), + # Classification / categorization + ("RecordType.Name", "Account Record Type"), + ("Account_Type__c", "Account Type"), + ("Classification__c", "Classification"), + ("Segmentation__c", "Segmentation"), + # Ownership + ("Owner.Name", "Account Owner"), + ("Account_Owner_Email__c", "Account Owner Email"), + # System fields needed for Document construction; their friendly labels + # are kept ending in Id/Date so danswer's SF_JSON_FILTER strips them + # from the searchable text body. + ("LastModifiedDate", "LastModifiedDate"), + ("LastModifiedBy.Name", "Last Modified By"), + # Custom fields — verified against this org's describe + ("Geo__c", "Geo"), + ("Area__c", "Area"), + ("Country__c", "Country"), + ("Ruby_Account__c", "Ruby Account"), + ("Maintenance_Flag__c", "Maintenance Flag"), + ("CSM__r.Name", "CSM"), + ("Support_Technical_Advisor__r.Name", "TAM"), + ("Vertical1__c", "Vertical"), + ("Sub_Vertical__c", "Sub-Vertical"), + ("CSD__r.Name", "CSD"), + ("NumberOfEmployees", "Employees"), + ("Annual_Contract_Value__c", "Annual Contract Value"), + # Requested fields with no match on this org's Account object. If they + # actually exist under different API names, drop them in here: + # BDR — no field found + # Lead Sales Engineer — no field found + # Territory Domain — no field found +] + + +def _resolve_path(record: dict[str, Any], path: str) -> Any: + """Walk a dotted SOQL path through the response JSON. Returns None if any + segment is missing or not a dict.""" + value: Any = record + for part in path.split("."): + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _to_friendly_dict(record: dict[str, Any]) -> dict[str, Any]: + """Re-key a SOQL record using ACCOUNT_FIELDS' friendly labels, flattening + dot-traversal paths so e.g. CSM__r.Name -> {"CSM": "Joe Smith"}.""" + out: dict[str, Any] = {} + for api_path, label in ACCOUNT_FIELDS: + value = _resolve_path(record, api_path) + if value is not None: + out[label] = value + return out + + +def _soql_datetime(dt: datetime) -> str: + """Format a datetime for a SOQL date literal. + + Uses the trailing-Z form (`YYYY-MM-DDThh:mm:ss.000Z`) rather than + `+hh:mm` because the `+` would otherwise survive into the URL query + string and risk being decoded as a space by intermediate parsers, + silently turning the WHERE clause into a no-match. + """ + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + else: + dt = dt.astimezone(timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + logger = setup_logger() @@ -36,62 +123,157 @@ def __init__( requested_objects: list[str] = [], ) -> None: self.batch_size = batch_size - self.sf_client: Salesforce | None = None - self.parent_object_list = ( - [obj.capitalize() for obj in requested_objects] - if requested_objects - else DEFAULT_PARENT_OBJECT_TYPES + # `requested_objects` is accepted for backward compatibility with the + # old multi-object connector configuration shape (e.g. ["Account"]). + # The Account-only connector ignores it; filtering is now driven by + # the env-var filters below. Warn loudly so stale configs don't go + # unnoticed. + if requested_objects: + logger.warning( + "SalesforceConnector: 'requested_objects' is deprecated and " + "ignored (got %r). Use SF_ACCOUNT_NAME_FILTER / " + "SF_MAINTENANCE_FLAG_FILTER env vars to scope the query.", + requested_objects, + ) + # Optional WHERE-clause filters from env vars. Each is read once at + # connector construction; restart the indexer to pick up changes. + # SF_ACCOUNT_NAME_FILTER — substring match (Name LIKE '%value%'). + # Use SF_ACCOUNT_NAME_EXACT=1 for an + # exact = match instead. + # SF_MAINTENANCE_FLAG_FILTER — exact match on the Maintenance_Flag__c + # picklist. Comma-separated for IN(...). + self.account_name_filter: str | None = ( + os.environ.get("SF_ACCOUNT_NAME_FILTER", "").strip() or None + ) + self.account_name_exact: bool = ( + os.environ.get("SF_ACCOUNT_NAME_EXACT", "").strip() == "1" ) + maint_raw = os.environ.get("SF_MAINTENANCE_FLAG_FILTER", "").strip() + self.maintenance_flag_filter: list[str] | None = ( + [v.strip() for v in maint_raw.split(",") if v.strip()] + if maint_raw + else None + ) + self.access_token: str | None = None + self.instance_url: str | None = None + self.headers: dict[str, str] = {} def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: - self.sf_client = Salesforce( + self.access_token, self.instance_url = self._get_access_token( + client_id=credentials["sf_client_id"], + client_secret=credentials["sf_client_secret"], username=credentials["sf_username"], password=credentials["sf_password"], - security_token=credentials["sf_security_token"], + login_url=credentials.get("sf_login_url", DEFAULT_SF_LOGIN_URL), ) - + self.headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } return None - def _get_sf_type_object_json(self, type_name: str) -> Any: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - sf_object = SFType( - type_name, self.sf_client.session_id, self.sf_client.sf_instance + def _get_access_token( + self, + client_id: str, + client_secret: str, + username: str, + password: str, + login_url: str, + ) -> Tuple[str, str]: + response = requests.post( + f"{login_url.rstrip('/')}/services/oauth2/token", + data={ + "grant_type": "password", + "client_id": client_id, + "client_secret": client_secret, + "username": username, + "password": password, + }, + timeout=30, ) - return sf_object.describe() + if response.status_code != 200: + logger.error(f"Salesforce authentication failed: {response.text}") + raise Exception("Failed to authenticate with Salesforce.") + data = response.json() + logger.info("Successfully authenticated with Salesforce.") + return data["access_token"], data["instance_url"] + + def _build_account_query( + self, + start: datetime | None = None, + end: datetime | None = None, + select_fields: list[str] | None = None, + ) -> str: + if select_fields is None: + select_fields = [api_path for api_path, _ in ACCOUNT_FIELDS] + select_clause = ", ".join(select_fields) + clauses: list[str] = [] + + if self.account_name_filter: + safe = self.account_name_filter.replace("'", r"\'") + if self.account_name_exact: + clauses.append(f"Name = '{safe}'") + else: + clauses.append(f"Name LIKE '%{safe}%'") - def _get_name_from_id(self, id: str) -> str: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - try: - user_object_info = self.sf_client.query( - f"SELECT Name FROM User WHERE Id = '{id}'" + if self.maintenance_flag_filter: + quoted = ", ".join( + f"'{v.replace(chr(39), chr(92) + chr(39))}'" + for v in self.maintenance_flag_filter ) - name = user_object_info.get("Records", [{}])[0].get("Name", "Null User") - return name - except Exception: - logger.warning(f"Couldnt find name for object id: {id}") - return "Null User" + clauses.append(f"Maintenance_Flag__c IN ({quoted})") + + if start is not None: + clauses.append(f"LastModifiedDate >= {_soql_datetime(start)}") + if end is not None: + clauses.append(f"LastModifiedDate <= {_soql_datetime(end)}") + + where_clause = f" WHERE {' AND '.join(clauses)}" if clauses else "" + return f"SELECT {select_clause} FROM Account{where_clause}" + + def _execute_soql(self, query: str) -> list[dict[str, Any]]: + if self.access_token is None or self.instance_url is None: + raise ConnectorMissingCredentialError("Salesforce") + + records: list[dict[str, Any]] = [] + url: str | None = f"{self.instance_url}/services/data/{SF_API_VERSION}/query" + params: dict[str, str] | None = {"q": query} + + while url: + resp = requests.get(url, headers=self.headers, params=params, timeout=120) + data = resp.json() + if resp.status_code != 200 or isinstance(data, list): + msg = data if isinstance(data, list) else resp.text + raise Exception(f"Salesforce SOQL error: {msg}") + + records.extend(data.get("records", [])) + + next_path = data.get("nextRecordsUrl") + if data.get("done", True) or not next_path: + break + url = f"{self.instance_url}{next_path}" + params = None # nextRecordsUrl already encodes the query + + return records def _convert_object_instance_to_document( self, object_dict: dict[str, Any] ) -> Document: - if self.sf_client is None: + if self.instance_url is None: raise ConnectorMissingCredentialError("Salesforce") salesforce_id = object_dict["Id"] danswer_salesforce_id = f"{ID_PREFIX}{salesforce_id}" - extracted_link = f"https://{self.sf_client.sf_instance}/{salesforce_id}" + extracted_link = f"{self.instance_url}/{salesforce_id}" extracted_doc_updated_at = time_str_to_utc(object_dict["LastModifiedDate"]) - extracted_object_text = extract_dict_text(object_dict) - extracted_semantic_identifier = object_dict.get("Name", "Unknown Object") - extracted_primary_owners = [ - BasicExpertInfo( - display_name=self._get_name_from_id(object_dict["LastModifiedById"]) - ) - ] + extracted_semantic_identifier = object_dict.get("Name", "Unknown Account") + # Re-key with friendly labels before generating the searchable text. + extracted_object_text = extract_dict_text(_to_friendly_dict(object_dict)) - doc = Document( + owner_name = _resolve_path(object_dict, "LastModifiedBy.Name") or "Unknown" + extracted_primary_owners = [BasicExpertInfo(display_name=owner_name)] + + return Document( id=danswer_salesforce_id, sections=[Section(link=extracted_link, text=extracted_object_text)], source=DocumentSource.SALESFORCE, @@ -100,136 +282,25 @@ def _convert_object_instance_to_document( primary_owners=extracted_primary_owners, metadata={}, ) - return doc - - def _is_valid_child_object(self, child_relationship: dict) -> bool: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - - if not child_relationship["childSObject"]: - return False - if not child_relationship["relationshipName"]: - return False - - sf_type = child_relationship["childSObject"] - object_description = self._get_sf_type_object_json(sf_type) - if not object_description["queryable"]: - return False - - try: - query = f"SELECT Count() FROM {sf_type} LIMIT 1" - result = self.sf_client.query(query) - if result["totalSize"] == 0: - return False - except Exception as e: - logger.warning(f"Object type {sf_type} doesn't support query: {e}") - return False - - if child_relationship["field"]: - if child_relationship["field"] == "RelatedToId": - return False - else: - return False - - return True - - def _get_all_children_of_sf_type(self, sf_type: str) -> list[dict]: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - - object_description = self._get_sf_type_object_json(sf_type) - - children_objects: list[dict] = [] - for child_relationship in object_description["childRelationships"]: - if self._is_valid_child_object(child_relationship): - children_objects.append( - { - "relationship_name": child_relationship["relationshipName"], - "object_type": child_relationship["childSObject"], - } - ) - return children_objects - - def _get_all_fields_for_sf_type(self, sf_type: str) -> list[str]: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - - object_description = self._get_sf_type_object_json(sf_type) - - fields = [ - field.get("name") - for field in object_description["fields"] - if field.get("type", "base64") != "base64" - ] - - return fields - - def _generate_query_per_parent_type(self, parent_sf_type: str) -> Iterator[str]: - """ - This function takes in an object_type and generates query(s) designed to grab - information associated to objects of that type. - It does that by getting all the fields of the parent object type. - Then it gets all the child objects of that object type and all the fields of - those children as well. - """ - parent_fields = self._get_all_fields_for_sf_type(parent_sf_type) - child_sf_types = self._get_all_children_of_sf_type(parent_sf_type) - - query = f"SELECT {', '.join(parent_fields)}" - for child_object_dict in child_sf_types: - fields = self._get_all_fields_for_sf_type(child_object_dict["object_type"]) - query_addition = f", \n(SELECT {', '.join(fields)} FROM {child_object_dict['relationship_name']})" - - if len(query_addition) + len(query) > MAX_QUERY_LENGTH: - query += f"\n FROM {parent_sf_type}" - yield query - query = "SELECT Id" + query_addition - else: - query += query_addition - - query += f"\n FROM {parent_sf_type}" - - yield query def _fetch_from_salesforce( self, start: datetime | None = None, end: datetime | None = None, ) -> GenerateDocumentsOutput: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") + query = self._build_account_query(start=start, end=end) + logger.info(f"Running Account SOQL: {query}") + records = self._execute_soql(query) + logger.info(f"Number of Account records fetched: {len(records)}") doc_batch: list[Document] = [] - for parent_object_type in self.parent_object_list: - logger.debug(f"Processing: {parent_object_type}") - - query_results: dict = {} - for query in self._generate_query_per_parent_type(parent_object_type): - if start is not None and end is not None: - if start and start.tzinfo is None: - start = start.replace(tzinfo=timezone.utc) - if end and end.tzinfo is None: - end = end.replace(tzinfo=timezone.utc) - query += f" WHERE LastModifiedDate > {start.isoformat()} AND LastModifiedDate < {end.isoformat()}" - - query_result = self.sf_client.query_all(query) - - for record_dict in query_result["records"]: - query_results.setdefault(record_dict["Id"], {}).update(record_dict) - - logger.info( - f"Number of {parent_object_type} Objects processed: {len(query_results)}" - ) - - for combined_object_dict in query_results.values(): - doc_batch.append( - self._convert_object_instance_to_document(combined_object_dict) - ) - - if len(doc_batch) > self.batch_size: - yield doc_batch - doc_batch = [] - yield doc_batch + for record in records: + doc_batch.append(self._convert_object_instance_to_document(record)) + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + if doc_batch: + yield doc_batch def load_from_state(self) -> GenerateDocumentsOutput: return self._fetch_from_salesforce() @@ -237,37 +308,30 @@ def load_from_state(self) -> GenerateDocumentsOutput: def poll_source( self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch ) -> GenerateDocumentsOutput: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - start_datetime = datetime.utcfromtimestamp(start) - end_datetime = datetime.utcfromtimestamp(end) + start_datetime = datetime.fromtimestamp(start, tz=timezone.utc) + end_datetime = datetime.fromtimestamp(end, tz=timezone.utc) return self._fetch_from_salesforce(start=start_datetime, end=end_datetime) def retrieve_all_source_ids(self) -> set[str]: - if self.sf_client is None: - raise ConnectorMissingCredentialError("Salesforce") - all_retrieved_ids: set[str] = set() - for parent_object_type in self.parent_object_list: - query = f"SELECT Id FROM {parent_object_type}" - query_result = self.sf_client.query_all(query) - all_retrieved_ids.update( - f"{ID_PREFIX}{instance_dict.get('Id', '')}" - for instance_dict in query_result["records"] - ) - - return all_retrieved_ids + query = self._build_account_query(select_fields=["Id"]) + records = self._execute_soql(query) + return {f"{ID_PREFIX}{r.get('Id', '')}" for r in records if r.get("Id")} if __name__ == "__main__": connector = SalesforceConnector( - requested_objects=os.environ["REQUESTED_OBJECTS"].split(",") + requested_objects=os.environ.get("REQUESTED_OBJECTS", "").split(",") + if os.environ.get("REQUESTED_OBJECTS") + else [] ) connector.load_credentials( { + "sf_client_id": os.environ["SF_CLIENT_ID"], + "sf_client_secret": os.environ["SF_CLIENT_SECRET"], "sf_username": os.environ["SF_USERNAME"], "sf_password": os.environ["SF_PASSWORD"], - "sf_security_token": os.environ["SF_SECURITY_TOKEN"], + "sf_login_url": os.environ.get("SF_LOGIN_URL", DEFAULT_SF_LOGIN_URL), } ) document_batches = connector.load_from_state() diff --git a/backend/danswer/connectors/sfkbarticles/connector.py b/backend/danswer/connectors/sfkbarticles/connector.py index 91bed54617c..aad5ba1d048 100644 --- a/backend/danswer/connectors/sfkbarticles/connector.py +++ b/backend/danswer/connectors/sfkbarticles/connector.py @@ -19,7 +19,7 @@ from danswer.connectors.sfkbarticles.utils import extract_dict_text from danswer.utils.logger import setup_logger -ID_PREFIX = "SALESFORCE_" +ID_PREFIX = "SFKBARTICLES_" AUTH_URL = "https://login.salesforce.com/services/oauth2/token" logger = setup_logger() diff --git a/backend/danswer/danswerbot/slack/blocks.py b/backend/danswer/danswerbot/slack/blocks.py index ca5fcbc20a0..fffe9feb279 100644 --- a/backend/danswer/danswerbot/slack/blocks.py +++ b/backend/danswer/danswerbot/slack/blocks.py @@ -454,7 +454,9 @@ def build_follow_up_block(message_id: int | None) -> ActionsBlock: def build_follow_up_resolved_blocks( - tag_ids: list[str], group_ids: list[str] + tag_ids: list[str], + group_ids: list[str], + message_id: int | None = None, ) -> list[Block]: tag_str = " ".join([f"<@{tag}>" for tag in tag_ids]) if tag_str: @@ -470,14 +472,18 @@ def build_follow_up_resolved_blocks( + "Someone has requested more help.\n\n:point_down:Please mark this resolved after answering!" ) text_block = SectionBlock(text=text) + # Encode `message_id` in the ActionsBlock's `block_id` so the resolved- + # button handler can recover it and record a chat_feedback row marking + # this message as resolved. Mirrors the pattern in `build_follow_up_block`. button_block = ActionsBlock( + block_id=build_feedback_id(message_id) if message_id is not None else None, elements=[ ButtonElement( action_id=FOLLOWUP_BUTTON_RESOLVED_ACTION_ID, style="primary", text="Mark Resolved", ) - ] + ], ) return [text_block, button_block] diff --git a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py index 01ba96dc722..b1ff4ca61c8 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py @@ -273,7 +273,18 @@ def handle_followup_button( if dri_ids: tag_ids.extend(dri_ids) - blocks = build_follow_up_resolved_blocks(tag_ids=tag_ids, group_ids=group_ids) + # Pass the message_id (decoded from the followup button's block_id) + # so the "Mark Resolved" button can attribute its feedback row to the + # right chat_message. + resolved_message_id: int | None = None + if action_id is not None: + try: + resolved_message_id, _, _ = decompose_action_id(action_id) + except ValueError: + resolved_message_id = None + blocks = build_follow_up_resolved_blocks( + tag_ids=tag_ids, group_ids=group_ids, message_id=resolved_message_id + ) # Check for curated response based on user title curated_response_sent = handle_curated_response( @@ -340,6 +351,39 @@ def handle_followup_resolved_button( clicker_name = get_clicker_name(req, client) + # Record a chat_feedback row marking this message as resolved so the + # NPS / analytics pipeline can count "resolved" alongside "like" as + # a positive signal. Best-effort: if the action_id / block_id doesn't + # carry a message_id (e.g. older button payloads from before this + # change), we just log and move on rather than blocking the UX. + action_block_id: str | None = None + if actions := req.payload.get("actions"): + action = cast(dict[str, Any], actions[0]) + action_block_id = cast(str | None, action.get("block_id")) + if action_block_id: + try: + resolved_message_id, _, _ = decompose_action_id(action_block_id) + with Session(get_sqlalchemy_engine()) as db_session: + create_chat_message_feedback( + is_positive=None, + feedback_text="", + chat_message_id=resolved_message_id, + user_id=None, # no "user" for Slack bot for now + db_session=db_session, + predefined_feedback="resolved", + ) + except (ValueError, Exception) as e: # noqa: BLE001 — best effort + logger_base.warning( + f"Could not record 'resolved' feedback (block_id=" + f"{action_block_id!r}): {e}" + ) + else: + logger_base.info( + "Resolved button clicked but the ActionsBlock had no block_id; " + "feedback row not recorded. Older message — expected to phase " + "out as new bot replies use the updated build_follow_up_block." + ) + update_emote_react( emoji=DANSWER_FOLLOWUP_EMOJI, channel=channel_id, diff --git a/backend/danswer/db/analytics.py b/backend/danswer/db/analytics.py new file mode 100644 index 00000000000..4af60b7a80d --- /dev/null +++ b/backend/danswer/db/analytics.py @@ -0,0 +1,306 @@ +"""Analytics aggregate queries for the admin /analytics page. + +Lives in the community (open-source) module, parallel to (not depending +on) `ee/danswer/db/analytics.py`. The shape mirrors upstream Onyx EE so +the same frontend code can drive either backend, but the implementation +imports only from `danswer.*` — never from `ee.*`. + +Three time-series aggregates, all daily-bucketed: + - fetch_query_analytics: total assistant messages + likes + dislikes per day + - fetch_per_user_query_analytics: same, broken out per user (used to + derive distinct-user counts) + - fetch_danswerbot_analytics: Slackbot session count + sessions + flagged as "needs help" (negative or required-followup feedback) per day +""" +import datetime +from collections.abc import Sequence +from uuid import UUID + +from sqlalchemy import case +from sqlalchemy import cast +from sqlalchemy import Date +from sqlalchemy import func +from sqlalchemy import or_ +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.configs.constants import MessageType +from danswer.db.models import ChatMessage +from danswer.db.models import ChatMessageFeedback +from danswer.db.models import ChatSession +from danswer.db.models import Connector +from danswer.db.models import ConnectorCredentialPair +from danswer.db.models import Document + + +def fetch_query_analytics( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, int, int, int, datetime.date]]: + """Daily aggregates of assistant chat messages and their feedback. + + Returns rows of: + (total_queries, total_likes, total_dislikes, + total_resolved, total_needs_help, date) + + where each "query" is one assistant-message reply, and each feedback + bucket counts FEEDBACK ROWS (not messages) — a single message with + two feedback events shows up twice. The frontend's "strict NPS" + treats likes + resolved as promoters and dislikes + needs-help as + detractors: + + NPS_strict = (P - D) / (P + D) * 100 + = ((likes + resolved) - (dislikes + needs_help)) / + ((likes + resolved) + (dislikes + needs_help)) * 100 + + `total_resolved` comes from `predefined_feedback = 'resolved'` rows — + written by the Slackbot's "I'm all set!" / "Mark Resolved" buttons + (see danswerbot/slack/handlers/handle_buttons.py). Older bot replies + don't have these rows yet; the count grows as users click resolved + going forward. + + `total_needs_help` counts `required_followup IS TRUE` rows — the "I + need more help" button in the Slackbot. + """ + stmt = ( + select( + func.count(ChatMessage.id), + func.sum(case((ChatMessageFeedback.is_positive, 1), else_=0)), + func.sum( + case( + (ChatMessageFeedback.is_positive == False, 1), # noqa: E712 + else_=0, + ) + ), + func.sum( + case( + (ChatMessageFeedback.predefined_feedback == "resolved", 1), + else_=0, + ) + ), + func.sum( + case( + (ChatMessageFeedback.required_followup.is_(True), 1), + else_=0, + ) + ), + cast(ChatMessage.time_sent, Date), + ) + .join( + ChatMessageFeedback, + ChatMessageFeedback.chat_message_id == ChatMessage.id, + isouter=True, + ) + .where(ChatMessage.time_sent >= start) + .where(ChatMessage.time_sent <= end) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + .group_by(cast(ChatMessage.time_sent, Date)) + .order_by(cast(ChatMessage.time_sent, Date)) + ) + + return db_session.execute(stmt).all() # type: ignore + + +def fetch_per_user_query_analytics( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, int, datetime.date, UUID]]: + """Same as `fetch_query_analytics` but grouped by user_id too. + + The /analytics/admin/user endpoint folds this down to a count of + distinct users per day. + """ + stmt = ( + select( + func.count(ChatMessage.id), + func.sum(case((ChatMessageFeedback.is_positive, 1), else_=0)), + func.sum( + case( + (ChatMessageFeedback.is_positive == False, 1), # noqa: E712 + else_=0, + ) + ), + cast(ChatMessage.time_sent, Date), + ChatSession.user_id, + ) + .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id) + .where(ChatMessage.time_sent >= start) + .where(ChatMessage.time_sent <= end) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + .group_by(cast(ChatMessage.time_sent, Date), ChatSession.user_id) + .order_by(cast(ChatMessage.time_sent, Date), ChatSession.user_id) + ) + + return db_session.execute(stmt).all() # type: ignore + + +def fetch_danswerbot_analytics( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, datetime.date]]: + """Daily Slackbot stats: total sessions and sessions flagged "needs help". + + Returns rows of (total_sessions, total_negatives, date). The endpoint + layer derives `auto_resolved = max(0, total - total_negatives)`. + + "needs help" = the FIRST AI reply in the session has its LATEST + feedback marked as either negative OR required_followup. Sessions + with no feedback are NOT counted as negative (we want to give the + bot the benefit of the doubt when nobody clicked thumbs-down). + """ + # First AI message per Danswerbot session in the window. + subquery_first_ai_response = ( + db_session.query( + ChatMessage.chat_session_id.label("chat_session_id"), + func.min(ChatMessage.id).label("chat_message_id"), + ) + .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id) + .where( + ChatSession.time_created >= start, + ChatSession.time_created <= end, + ChatSession.danswerbot_flow.is_(True), + ) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + .group_by(ChatMessage.chat_session_id) + .subquery() + ) + + # Most recent feedback row per chat_message that has any feedback. + subquery_last_feedback = ( + db_session.query( + ChatMessageFeedback.chat_message_id.label("chat_message_id"), + func.max(ChatMessageFeedback.id).label("max_feedback_id"), + ) + .group_by(ChatMessageFeedback.chat_message_id) + .subquery() + ) + + results = ( + db_session.query( + func.count(ChatSession.id).label("total_sessions"), + func.sum( + case( + ( + or_( + ChatMessageFeedback.is_positive.is_(False), + ChatMessageFeedback.required_followup, + ), + 1, + ), + else_=0, + ) + ).label("negative_answer"), + cast(ChatSession.time_created, Date).label("session_date"), + ) + .join( + subquery_first_ai_response, + ChatSession.id == subquery_first_ai_response.c.chat_session_id, + ) + .outerjoin( + subquery_last_feedback, + subquery_first_ai_response.c.chat_message_id + == subquery_last_feedback.c.chat_message_id, + ) + .outerjoin( + ChatMessageFeedback, + ChatMessageFeedback.id == subquery_last_feedback.c.max_feedback_id, + ) + .group_by(cast(ChatSession.time_created, Date)) + .order_by(cast(ChatSession.time_created, Date)) + .all() + ) + + return results + + +# --------------------------------------------------------------------------- +# Snapshot helpers (point-in-time, no date range) +# --------------------------------------------------------------------------- + + +def fetch_total_docs_indexed(db_session: Session) -> tuple[int, int]: + """Return `(total_docs_indexed, unique_docs)`. + + `total_docs_indexed` is the sum of the denormalized + `connector_credential_pair.total_docs_indexed` counter — counts the + same physical doc once per cc-pair that indexed it. This matches what + the indexing-status page shows per-row, so the KPI total = sum of + visible per-cc-pair counts. + + `unique_docs` is `count(*) FROM document`, the number of distinct + physical documents in Postgres. Lower than `total_docs_indexed` + when the same document is indexed by multiple cc-pairs. + """ + total_docs_indexed = ( + db_session.execute( + select( + func.coalesce(func.sum(ConnectorCredentialPair.total_docs_indexed), 0) + ) + ).scalar() + or 0 + ) + unique_docs = db_session.execute(select(func.count(Document.id))).scalar() or 0 + return int(total_docs_indexed), int(unique_docs) + + +def fetch_docs_per_source(db_session: Session) -> Sequence[tuple[str, int]]: + """Return rows of `(source, total_docs_indexed)` ordered DESC. + + Sources with zero docs (newly-added connectors that haven't run yet) + are included so operators can see them in the chart and notice they + haven't started indexing. Excluding them is harder to debug. + """ + stmt = ( + select( + Connector.source, + func.coalesce(func.sum(ConnectorCredentialPair.total_docs_indexed), 0), + ) + .join( + ConnectorCredentialPair, + ConnectorCredentialPair.connector_id == Connector.id, + isouter=True, + ) + .group_by(Connector.source) + .order_by( + func.coalesce( + func.sum(ConnectorCredentialPair.total_docs_indexed), 0 + ).desc() + ) + ) + return [(str(src), int(n)) for src, n in db_session.execute(stmt).all()] + + +def fetch_slack_bot_channel_stats(db_session: Session) -> tuple[int, int]: + """Return `(total_configs, distinct_channels_enabled)`. + + `slack_bot_config.channel_config` is a JSONB blob with a + `channel_names: list[str]` key. Each row's config can target many + channels; we unroll all rows' arrays and count distinct channel + names across the whole table. A channel listed in two configs counts + once. + + Uses `jsonb_array_elements_text` lateral expansion in raw SQL — the + SQLAlchemy DSL for jsonb-array unrolling is verbose enough that raw + SQL is the cleaner option here. + """ + total_configs = ( + db_session.execute(text("SELECT count(*) FROM slack_bot_config")).scalar() or 0 + ) + distinct_channels = ( + db_session.execute( + text( + """ + SELECT count(DISTINCT chan) + FROM slack_bot_config, + jsonb_array_elements_text(channel_config -> 'channel_names') + AS chan + """ + ) + ).scalar() + or 0 + ) + return int(total_configs), int(distinct_channels) diff --git a/backend/danswer/db/analytics_rollup.py b/backend/danswer/db/analytics_rollup.py new file mode 100644 index 00000000000..9fa8abc5637 --- /dev/null +++ b/backend/danswer/db/analytics_rollup.py @@ -0,0 +1,549 @@ +"""Daily rollup of admin analytics metrics. + +Why this exists: chat retention deletes `chat_message` / `chat_session` +rows older than RETENTION_DAYS_CHAT (default 30 days). The analytics +endpoints used to read directly from those tables, so any date range +older than the retention window returned zeros — silent data loss for +operators wanting quarterly / yearly trends. + +Solution: pre-aggregate the daily metrics into `analytics_daily_rollup` +BEFORE the retention sweep runs, and have the endpoints read from the +rollup. The rollup table is small (one row per day, ~50 bytes) and never +gets retention-cleaned. + +Pipeline: + - 07:30 UTC: `run_analytics_rollup_task` (Celery beat) recomputes from + `(last_rolled_up_to - ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS)` through + today. The checkpoint `last_rolled_up_to` is stored as a JSON row in + `key_value_store` and advanced to today after a successful run. + Idempotent — safe to re-run; safe across outages (next run catches + up from where the last one stopped). + - 08:00 UTC: `run_retention_policies_task` deletes old chat rows. + +The "late feedback" buffer (default 2 days) catches feedback that +arrives slightly after the answer — most reactions are immediate, but +a Slack helper marking "resolved" can take a day or two. We re-process +those days each run so updated counts win via INSERT…ON CONFLICT DO +UPDATE. + +Safety cap: even with a stale checkpoint (e.g. task offline for weeks), +the task won't try to scan back further than `RETENTION_DAYS_CHAT - 2` +days, since chat data older than retention is gone and re-computing +would zero out historical rows. Days lost to a long outage stay at +their last-known values rather than being clobbered. + +For the initial deploy of this module, run +`backend/scripts/backfill_analytics_rollup.py` once before the next +retention sweep — it walks every historical date that still has chat +data, populates the rollup, AND seeds the checkpoint row. After that, +the daily task takes over. +""" +from __future__ import annotations + +import datetime +import os +from collections.abc import Sequence + +from sqlalchemy import case +from sqlalchemy import cast +from sqlalchemy import Date +from sqlalchemy import func +from sqlalchemy import or_ +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from danswer.configs.constants import MessageType +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.models import AnalyticsDailyRollup +from danswer.db.models import ChatMessage +from danswer.db.models import ChatMessageFeedback +from danswer.db.models import ChatSession +from danswer.utils.logger import setup_logger + +logger = setup_logger() + + +def _env_int(name: str, default: int, minimum: int = 0) -> int: + raw = os.environ.get(name, "").strip() + if not raw: + return default + try: + return max(minimum, int(raw)) + except ValueError: + logger.warning(f"Invalid {name}={raw!r}; using default {default}") + return default + + +# How far back from the checkpoint to re-process on each run. This is the +# late-feedback grace period: most reactions arrive within minutes, but +# a "resolved" mark from a Slack helper can take a day or two. We recompute +# those days each run so the updated counts win via the upsert. +ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS = _env_int("ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS", 2) + +# Checkpoint row in `key_value_store`. Stored as JSON like +# `{"last_rolled_up_to": "2026-05-14"}`. The daily task starts from +# `(last_rolled_up_to - buffer)` and advances the checkpoint to today +# after a successful run. +ROLLUP_CHECKPOINT_KEY = "analytics_rollup_state" + + +# --------------------------------------------------------------------------- +# Compute helpers — read live tables, return the metrics for one UTC day +# --------------------------------------------------------------------------- + + +def _day_bounds( + target_date: datetime.date, +) -> tuple[datetime.datetime, datetime.datetime]: + """[start, end) UTC datetimes for one calendar day.""" + start = datetime.datetime.combine( + target_date, datetime.time.min, tzinfo=datetime.timezone.utc + ) + end = start + datetime.timedelta(days=1) + return start, end + + +def _compute_query_metrics_for_date( + db_session: Session, target_date: datetime.date +) -> dict[str, int]: + """Total assistant replies + feedback breakdown for one UTC day. + + Mirrors `fetch_query_analytics`'s SQL but returns a flat dict for the + single date (so we can upsert directly). + """ + start, end = _day_bounds(target_date) + row = db_session.execute( + select( + func.count(ChatMessage.id), + func.coalesce( + func.sum(case((ChatMessageFeedback.is_positive, 1), else_=0)), 0 + ), + func.coalesce( + func.sum( + case( + (ChatMessageFeedback.is_positive == False, 1), # noqa: E712 + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + (ChatMessageFeedback.predefined_feedback == "resolved", 1), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + (ChatMessageFeedback.required_followup.is_(True), 1), + else_=0, + ) + ), + 0, + ), + ) + .join( + ChatMessageFeedback, + ChatMessageFeedback.chat_message_id == ChatMessage.id, + isouter=True, + ) + .where(ChatMessage.time_sent >= start) + .where(ChatMessage.time_sent < end) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + ).one() + total_queries, likes, dislikes, resolved, needs_help = row + return { + "total_queries": int(total_queries or 0), + "total_likes": int(likes or 0), + "total_dislikes": int(dislikes or 0), + "total_resolved": int(resolved or 0), + "total_needs_help": int(needs_help or 0), + } + + +def _compute_active_users_for_date( + db_session: Session, target_date: datetime.date +) -> int: + """Distinct user count of users who asked at least one question on + `target_date`. Joins chat_message → chat_session for user_id. + + `select_from(ChatMessage)` is explicit because the projection only + references ChatSession.user_id — without it, SA would pick + ChatSession as the implicit FROM and then complain when we try to + join it again. + """ + start, end = _day_bounds(target_date) + n = db_session.execute( + select(func.count(func.distinct(ChatSession.user_id))) + .select_from(ChatMessage) + .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id) + .where(ChatMessage.time_sent >= start) + .where(ChatMessage.time_sent < end) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + .where(ChatSession.user_id.is_not(None)) + ).scalar() + return int(n or 0) + + +def _compute_slackbot_metrics_for_date( + db_session: Session, target_date: datetime.date +) -> tuple[int, int]: + """`(slackbot_total, slackbot_auto_resolved)` for one day. + + Total = Slackbot sessions whose first AI reply landed on `target_date`. + Auto-resolved = total minus sessions whose latest feedback is negative + or required_followup. Mirrors the existing /admin/danswerbot logic. + """ + start, end = _day_bounds(target_date) + + subquery_first_ai_response = ( + db_session.query( + ChatMessage.chat_session_id.label("chat_session_id"), + func.min(ChatMessage.id).label("chat_message_id"), + ) + .select_from(ChatMessage) + .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id) + .where( + ChatSession.time_created >= start, + ChatSession.time_created < end, + ChatSession.danswerbot_flow.is_(True), + ) + .where(ChatMessage.message_type == MessageType.ASSISTANT) + .group_by(ChatMessage.chat_session_id) + .subquery() + ) + + subquery_last_feedback = ( + db_session.query( + ChatMessageFeedback.chat_message_id.label("chat_message_id"), + func.max(ChatMessageFeedback.id).label("max_feedback_id"), + ) + .group_by(ChatMessageFeedback.chat_message_id) + .subquery() + ) + + # `select_from(ChatSession)` is explicit because the outer SELECT lists + # only an aggregate on ChatSession.id plus a sum referencing + # ChatMessageFeedback — SA 2.x can't infer a single left-side FROM + # from those, especially with ChatSession also referenced inside the + # `subquery_first_ai_response`. The EE original got away without this + # because it also selected `cast(ChatSession.time_created, Date)` + # which pinned the FROM. + row = ( + db_session.query( + func.count(ChatSession.id), + func.coalesce( + func.sum( + case( + ( + or_( + ChatMessageFeedback.is_positive.is_(False), + ChatMessageFeedback.required_followup, + ), + 1, + ), + else_=0, + ) + ), + 0, + ), + ) + .select_from(ChatSession) + .join( + subquery_first_ai_response, + ChatSession.id == subquery_first_ai_response.c.chat_session_id, + ) + .outerjoin( + subquery_last_feedback, + subquery_first_ai_response.c.chat_message_id + == subquery_last_feedback.c.chat_message_id, + ) + .outerjoin( + ChatMessageFeedback, + ChatMessageFeedback.id == subquery_last_feedback.c.max_feedback_id, + ) + .one() + ) + total, negatives = int(row[0] or 0), int(row[1] or 0) + return total, max(0, total - negatives) + + +# --------------------------------------------------------------------------- +# Upsert (compute + write one day's row) +# --------------------------------------------------------------------------- + + +def upsert_rollup_for_date( + db_session: Session, target_date: datetime.date +) -> dict[str, int]: + """Compute one day's metrics from live tables and INSERT … ON CONFLICT + DO UPDATE into `analytics_daily_rollup`. Returns the metrics dict.""" + query_metrics = _compute_query_metrics_for_date(db_session, target_date) + active_users = _compute_active_users_for_date(db_session, target_date) + slackbot_total, slackbot_auto_resolved = _compute_slackbot_metrics_for_date( + db_session, target_date + ) + + metrics = { + **query_metrics, + "active_users": active_users, + "slackbot_total": slackbot_total, + "slackbot_auto_resolved": slackbot_auto_resolved, + } + + stmt = pg_insert(AnalyticsDailyRollup.__table__).values(date=target_date, **metrics) + update_cols = {col: stmt.excluded[col] for col in metrics.keys()} + update_cols["rolled_up_at"] = func.now() + stmt = stmt.on_conflict_do_update( + index_elements=[AnalyticsDailyRollup.date], + set_=update_cols, + ) + db_session.execute(stmt) + db_session.commit() + return metrics + + +# --------------------------------------------------------------------------- +# Batch operations — sliding window (daily task) + full backfill +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Checkpoint state in key_value_store +# --------------------------------------------------------------------------- + + +def _get_checkpoint(db_session: Session) -> datetime.date | None: + """Read `last_rolled_up_to` from `key_value_store`. Returns None if + the row is absent or malformed (first run after deploy).""" + payload = db_session.execute( + text("SELECT value FROM key_value_store WHERE key = :k"), + {"k": ROLLUP_CHECKPOINT_KEY}, + ).scalar() + if not payload or not isinstance(payload, dict): + return None + raw = payload.get("last_rolled_up_to") + if not isinstance(raw, str): + return None + try: + return datetime.date.fromisoformat(raw) + except ValueError: + logger.warning( + f"Analytics rollup checkpoint malformed: " + f"{ROLLUP_CHECKPOINT_KEY}.last_rolled_up_to={raw!r}; ignoring." + ) + return None + + +def _set_checkpoint(db_session: Session, last_rolled_up_to: datetime.date) -> None: + """Upsert the checkpoint row in `key_value_store`. Pure SQL upsert + via INSERT…ON CONFLICT — avoids a separate read.""" + payload = {"last_rolled_up_to": last_rolled_up_to.isoformat()} + db_session.execute( + text( + """ + INSERT INTO key_value_store (key, value) + VALUES (:k, CAST(:v AS jsonb)) + ON CONFLICT (key) DO UPDATE + SET value = EXCLUDED.value + """ + ), + {"k": ROLLUP_CHECKPOINT_KEY, "v": _json_dumps(payload)}, + ) + db_session.commit() + + +def _json_dumps(obj: dict) -> str: + # Local import so the analytics_rollup module's import surface + # stays minimal — `json` is stdlib but we only need it here. + import json + + return json.dumps(obj) + + +def _safe_lookback_floor(today: datetime.date) -> datetime.date: + """Earliest date the rollup task is allowed to recompute, honoring + chat retention. We refuse to scan beyond `(today - (RETENTION_DAYS_CHAT - 2))` + because chat data older than that is already deleted; re-processing + those days would zero out historical totals. + + Imports `RETENTION_DAYS_CHAT` lazily to avoid pulling the retention + module's side effects at import time on this hot path. + """ + from danswer.db.retention import RETENTION_DAYS_CHAT + + if RETENTION_DAYS_CHAT <= 0: + # Chat retention disabled — chat data lives forever, so no floor + # needed. Cap at "a year ago" just to keep first-run windows sane. + return today - datetime.timedelta(days=365) + return today - datetime.timedelta(days=max(2, RETENTION_DAYS_CHAT - 2)) + + +def run_rollup(today: datetime.date | None = None) -> int: + """Run the daily rollup. Recomputes from + `(last_rolled_up_to - ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS)` through + today, then advances the checkpoint to today. Idempotent. + + First-run handling (no checkpoint yet): defaults to `today - buffer`, + just refreshing the most recent few days. The backfill CLI is the + intended way to populate historical rows the first time. + + Outage handling: if the task hasn't run for N days, the next run's + window is `[checkpoint - buffer, today]` and naturally catches up, + capped by `_safe_lookback_floor` so we never re-process beyond + retention. Returns the number of dates upserted. + """ + today = today or datetime.datetime.now(tz=datetime.timezone.utc).date() + engine = get_sqlalchemy_engine() + with Session(engine) as db_session: + checkpoint = _get_checkpoint(db_session) + if checkpoint is None: + start = today - datetime.timedelta(days=ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS) + logger.info( + "Analytics rollup: no checkpoint found (first run after " + f"deploy?); processing last {ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS}+1 " + "days. Run backfill_analytics_rollup.py to populate history." + ) + else: + start = checkpoint - datetime.timedelta( + days=ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS + ) + + floor = _safe_lookback_floor(today) + if start < floor: + logger.warning( + f"Analytics rollup: checkpoint {checkpoint} would push the " + f"window back to {start}, but retention has likely deleted " + f"chat data older than {floor}. Capping the window at " + f"{floor} — older days keep their last-known rollup values." + ) + start = floor + + n = 0 + current = start + while current <= today: + upsert_rollup_for_date(db_session, current) + current += datetime.timedelta(days=1) + n += 1 + + _set_checkpoint(db_session, today) + logger.info( + f"Analytics rollup: processed {n} day(s) [{start} → {today}], " + f"checkpoint advanced to {today}" + ) + return n + + +def backfill_all_rollups(start_date: datetime.date, end_date: datetime.date) -> int: + """Walk every date in [start_date, end_date] and upsert. Used by the + one-time backfill CLI before retention starts deleting chat data. + + Sets the checkpoint to `end_date` on completion so the next daily + run's window starts from `(end_date - buffer)` instead of the + pre-deploy default. + """ + if start_date > end_date: + raise ValueError( + f"backfill_all_rollups: start_date {start_date} > end_date {end_date}" + ) + engine = get_sqlalchemy_engine() + n = 0 + with Session(engine) as db_session: + current = start_date + while current <= end_date: + upsert_rollup_for_date(db_session, current) + current += datetime.timedelta(days=1) + n += 1 + if n % 30 == 0: + logger.info( + f"backfill_all_rollups: processed {n} days " + f"(through {current - datetime.timedelta(days=1)})" + ) + _set_checkpoint(db_session, end_date) + return n + + +# --------------------------------------------------------------------------- +# Read-from-rollup helpers — what the API endpoints call +# --------------------------------------------------------------------------- + + +def fetch_query_analytics_from_rollup( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, int, int, int, datetime.date]]: + """Same shape as `db.analytics.fetch_query_analytics` but reads from + the rollup table — survives chat retention deletes. + + Returns rows of `(total_queries, total_likes, total_dislikes, + total_resolved, total_needs_help, date)` for each day in + [start.date(), end.date()] that has a rollup row. Days with no rollup + yet (e.g. today, before 07:30 UTC) are absent — caller should treat + missing dates as zero / not-yet-rolled-up. + """ + stmt = ( + select( + AnalyticsDailyRollup.total_queries, + AnalyticsDailyRollup.total_likes, + AnalyticsDailyRollup.total_dislikes, + AnalyticsDailyRollup.total_resolved, + AnalyticsDailyRollup.total_needs_help, + AnalyticsDailyRollup.date, + ) + .where(AnalyticsDailyRollup.date >= cast(start, Date)) + .where(AnalyticsDailyRollup.date <= cast(end, Date)) + .order_by(AnalyticsDailyRollup.date) + ) + return db_session.execute(stmt).all() # type: ignore + + +def fetch_user_analytics_from_rollup( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, datetime.date]]: + """Distinct-user count per day from the rollup. Returns + `(active_users, date)` tuples ordered by date.""" + stmt = ( + select(AnalyticsDailyRollup.active_users, AnalyticsDailyRollup.date) + .where(AnalyticsDailyRollup.date >= cast(start, Date)) + .where(AnalyticsDailyRollup.date <= cast(end, Date)) + .order_by(AnalyticsDailyRollup.date) + ) + return db_session.execute(stmt).all() # type: ignore + + +def fetch_danswerbot_analytics_from_rollup( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, datetime.date]]: + """Slackbot daily stats from the rollup. Returns + `(slackbot_total, slackbot_auto_resolved, date)` tuples.""" + stmt = ( + select( + AnalyticsDailyRollup.slackbot_total, + AnalyticsDailyRollup.slackbot_auto_resolved, + AnalyticsDailyRollup.date, + ) + .where(AnalyticsDailyRollup.date >= cast(start, Date)) + .where(AnalyticsDailyRollup.date <= cast(end, Date)) + .order_by(AnalyticsDailyRollup.date) + ) + return db_session.execute(stmt).all() # type: ignore + + +def get_earliest_chat_date(db_session: Session) -> datetime.date | None: + """Earliest `chat_session.time_created::date`, or None if the table + is empty. Used by the backfill CLI as the default --start.""" + earliest = db_session.execute( + select(func.min(cast(ChatSession.time_created, Date))) + ).scalar() + return earliest if isinstance(earliest, datetime.date) else None diff --git a/backend/danswer/db/connector_credential_pair.py b/backend/danswer/db/connector_credential_pair.py index 3d6740b1915..4fa2f8a0a09 100644 --- a/backend/danswer/db/connector_credential_pair.py +++ b/backend/danswer/db/connector_credential_pair.py @@ -4,6 +4,7 @@ from sqlalchemy import delete from sqlalchemy import desc from sqlalchemy import select +from sqlalchemy import text from sqlalchemy.orm import Session from danswer.db.connector import fetch_connector_by_id @@ -20,6 +21,61 @@ logger = setup_logger() +# Per-cc-pair *deletion* advisory lock. Distinct namespace from the +# indexing per-cc-pair lock (`b"INDX"` in db/index_attempt.py) so the +# two never collide — a deletion sweep should serialize against other +# deletions, not against indexing. The high 32 bits are `b"DELE"`. +# +# The bug this prevents: the API endpoint `/admin/deletion-attempt` +# has no in-flight dedup, so each click of "Delete connector" queues a +# fresh `cleanup_connector_credential_pair_task`. Each task does a +# `SELECT ... FOR UPDATE NOWAIT` over the cc-pair's documents in +# 1000-doc batches. Two-or-more concurrent tasks all see the lock +# contention, retry 10 × 30s = 5min each, then raise. The user's +# logs showed six task IDs all failing within 35ms because they had +# all been retrying for 5min and timed out together. +_DELETION_LOCK_KEY_OFFSET = 0x44454C45_00000000 # b"DELE" in the high bits + + +def _deletion_lock_key(connector_id: int, credential_id: int) -> int: + """Stable 64-bit advisory-lock key for deletion of (connector_id, + credential_id). Same hashing scheme as the indexing lock — only the + namespace prefix differs, so the two locks are independent.""" + h = (connector_id * 0x9E3779B1 ^ credential_id) & 0xFFFFFFFF + raw = _DELETION_LOCK_KEY_OFFSET | h + if raw >= 1 << 63: + raw -= 1 << 64 + return raw + + +def try_acquire_deletion_lock( + db_session: Session, connector_id: int, credential_id: int +) -> bool: + """Non-blocking attempt to acquire the deletion lock for this cc-pair. + + Returns True if acquired (caller must eventually call + `release_deletion_lock`). Returns False if another worker is + already running a deletion for this cc-pair. + + The lock is session-scoped so a crashed worker won't strand it + forever — Postgres releases on connection drop. + """ + key = _deletion_lock_key(connector_id, credential_id) + row = db_session.execute( + text("SELECT pg_try_advisory_lock(:k)"), {"k": key} + ).scalar() + return bool(row) + + +def release_deletion_lock( + db_session: Session, connector_id: int, credential_id: int +) -> None: + """Release the deletion lock for this cc-pair. Safe to call even if + we don't hold it — Postgres returns false but doesn't raise.""" + key = _deletion_lock_key(connector_id, credential_id) + db_session.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": key}) + + def get_connector_credential_pairs( db_session: Session, include_disabled: bool = True ) -> list[ConnectorCredentialPair]: diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py index 51c41c71986..6bb188360d2 100644 --- a/backend/danswer/db/index_attempt.py +++ b/backend/danswer/db/index_attempt.py @@ -7,6 +7,7 @@ from sqlalchemy import func from sqlalchemy import or_ from sqlalchemy import select +from sqlalchemy import text from sqlalchemy import update from sqlalchemy.orm import joinedload from sqlalchemy.orm import Session @@ -20,6 +21,77 @@ from danswer.utils.telemetry import optional_telemetry from danswer.utils.telemetry import RecordType + +# --------------------------------------------------------------------------- +# Per-cc-pair indexing lock (Postgres advisory lock) +# --------------------------------------------------------------------------- +# +# When NUM_INDEXING_WORKERS > 1, two indexing attempts for the same +# (connector_id, credential_id) could otherwise be assigned to two Dask +# workers and run concurrently — racing on Vespa writes, on +# `last_successful_index_time`, and on connector-side checkpoint state. +# +# Upstream Onyx prevents this with per-cc-pair Redis fences. We don't +# have Redis (the broker is Postgres), but advisory locks give the same +# semantics: a lock acquired by one session is invisible to other +# sessions until released or the holding session disconnects. +# +# `pg_try_advisory_lock(int8)` is the non-blocking form — we want a +# fast-fail "someone else has it" signal, not to wait. Lock IDs are 64- +# bit; we encode the cc-pair as `(offset || hash(connector_id, credential_id))` +# so they don't collide with the retention sweep's lock id. +_INDEXING_LOCK_KEY_OFFSET = 0x494E4458_00000000 # b"INDX" in the high bits + + +def _cc_pair_lock_key(connector_id: int, credential_id: int) -> int: + """Stable 64-bit advisory-lock key for (connector_id, credential_id). + + The high 32 bits are a fixed offset (`b"INDX"`) so this lock id can + never collide with the retention sweep's lock id (`b"RETENTIO"`) or + any other future advisory lock. The low 32 bits are a hash of the + cc-pair tuple — collisions there mean two unrelated cc-pairs would + share a lock, but with 32 bits of space and typical cc-pair counts in + the hundreds the birthday bound is well below 1%. + """ + h = (connector_id * 0x9E3779B1 ^ credential_id) & 0xFFFFFFFF + # Combine into a signed 64-bit int (Postgres bigint range). + raw = _INDEXING_LOCK_KEY_OFFSET | h + if raw >= 1 << 63: + raw -= 1 << 64 + return raw + + +def try_acquire_cc_pair_lock( + db_session: Session, connector_id: int, credential_id: int +) -> bool: + """Non-blocking attempt to acquire the indexing lock for this cc-pair. + + Returns True if the lock was acquired (caller is now responsible for + eventually calling `release_cc_pair_lock`). Returns False if another + session already holds it. + + The lock is *session-scoped*: it survives commits but is automatically + released when the database session disconnects. So even if a worker + process crashes mid-run without calling release, the lock won't be + permanently stuck once the connection times out. + """ + key = _cc_pair_lock_key(connector_id, credential_id) + row = db_session.execute( + text("SELECT pg_try_advisory_lock(:k)"), {"k": key} + ).scalar() + return bool(row) + + +def release_cc_pair_lock( + db_session: Session, connector_id: int, credential_id: int +) -> None: + """Release the indexing lock for this cc-pair. Safe to call even if + we don't currently hold the lock — Postgres returns false but doesn't + raise.""" + key = _cc_pair_lock_key(connector_id, credential_id) + db_session.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": key}) + + logger = setup_logger() @@ -36,6 +108,7 @@ def create_index_attempt( embedding_model_id: int, db_session: Session, from_beginning: bool = False, + indexing_priority: int = 0, ) -> int: new_attempt = IndexAttempt( connector_id=connector_id, @@ -43,6 +116,7 @@ def create_index_attempt( embedding_model_id=embedding_model_id, from_beginning=from_beginning, status=IndexingStatus.NOT_STARTED, + indexing_priority=max(0, min(int(indexing_priority), 100)), ) db_session.add(new_attempt) db_session.commit() @@ -50,6 +124,26 @@ def create_index_attempt( return new_attempt.id +def update_index_attempt_priority( + index_attempt_id: int, + indexing_priority: int, + db_session: Session, +) -> IndexAttempt | None: + """Update the priority of a NOT_STARTED attempt. Returns None if the + attempt doesn't exist or is no longer in NOT_STARTED — once an attempt + has been dispatched the priority can't change its scheduling decision.""" + attempt = db_session.execute( + select(IndexAttempt).where(IndexAttempt.id == index_attempt_id) + ).scalar_one_or_none() + if attempt is None: + return None + if attempt.status != IndexingStatus.NOT_STARTED: + return None + attempt.indexing_priority = max(0, min(int(indexing_priority), 100)) + db_session.commit() + return attempt + + def get_inprogress_index_attempts( connector_id: int | None, db_session: Session, @@ -65,9 +159,17 @@ def get_inprogress_index_attempts( def get_not_started_index_attempts(db_session: Session) -> list[IndexAttempt]: """This eagerly loads the connector and credential so that the db_session can be expired - before running long-living indexing jobs, which causes increasing memory usage""" + before running long-living indexing jobs, which causes increasing memory usage. + + Higher-priority attempts come first; within the same priority band the + oldest attempt wins (FIFO) so we don't starve normal-priority queues. + """ stmt = select(IndexAttempt) stmt = stmt.where(IndexAttempt.status == IndexingStatus.NOT_STARTED) + stmt = stmt.order_by( + desc(IndexAttempt.indexing_priority), + IndexAttempt.time_created.asc(), + ) stmt = stmt.options( joinedload(IndexAttempt.connector), joinedload(IndexAttempt.credential) ) diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index e1578284bda..58a8f32a8e9 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -12,6 +12,7 @@ from fastapi_users_db_sqlalchemy import SQLAlchemyBaseUserTableUUID from fastapi_users_db_sqlalchemy.access_token import SQLAlchemyBaseAccessTokenTableUUID from sqlalchemy import Boolean +from sqlalchemy import Date from sqlalchemy import DateTime from sqlalchemy import Enum from sqlalchemy import Float @@ -506,6 +507,14 @@ class IndexAttempt(Base): ForeignKey("embedding_model.id"), nullable=False, ) + # Higher values dispatch to Dask first when this attempt is in NOT_STARTED. + # 0 is the default (set by the auto-scheduler in update_loop). Manual + # triggers can supply a higher value (steps of 10, conventional ceiling + # 100) to jump the queue without affecting any other attempts on the + # same cc-pair. + indexing_priority: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) time_created: Mapped[datetime.datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), @@ -1434,3 +1443,59 @@ class UsageReport(Base): requestor = relationship("User") file = relationship("PGFileStore") + + +class AnalyticsDailyRollup(Base): + """Pre-aggregated daily analytics so the admin dashboard survives chat + retention deletes. + + `chat_message` / `chat_session` rows older than RETENTION_DAYS_CHAT + (default 30d) are purged by the daily retention sweep. The analytics + endpoints used to read directly from those tables, so any date range + older than ~30 days returned zeros. This rollup table is computed + BEFORE the retention sweep each day (Celery beat at 07:30 UTC, sweep + at 08:00 UTC) and persisted indefinitely. The endpoints now read from + here. + + One row per UTC date. Idempotent upserts let the rollup task re-run + over a sliding window (default last 7 days, configurable via + `ANALYTICS_ROLLUP_LOOKBACK_DAYS`) so feedback that arrives a few days + after the answer still gets counted. The window MUST be shorter than + `RETENTION_DAYS_CHAT` — otherwise the rollup would re-compute days + whose source rows have already been deleted, zeroing out historical + totals. + """ + + __tablename__ = "analytics_daily_rollup" + + date: Mapped[datetime.date] = mapped_column(Date, primary_key=True) + total_queries: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + total_likes: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + total_dislikes: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + total_resolved: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + total_needs_help: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + active_users: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + slackbot_total: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + slackbot_auto_resolved: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + rolled_up_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/danswer/db/retention.py b/backend/danswer/db/retention.py new file mode 100644 index 00000000000..e5cf43699d4 --- /dev/null +++ b/backend/danswer/db/retention.py @@ -0,0 +1,818 @@ +"""DB retention / cleanup policies. + +Periodic deletion of stale rows from tables that grow without bound. Each +policy declares a retention window (in days) configurable via env var +plus the SQL or function that performs the delete. The executor runs +every policy under a single Postgres advisory lock so two runs can't +overlap, and deletes in batches so a multi-million-row first run can't +hold table locks for minutes. + +This is the lean equivalent of the per-table `check_for_*_cleanup` Celery +tasks in upstream Onyx, adapted for this fork's Postgres broker (which +materializes Kombu's queue as SQL tables — Onyx uses Redis and doesn't +have to worry about kombu_message bloat). + +Policies registered here: + - kombu_message : Celery's broker storage when SQLAlchemy is the broker. + - task_queue_jobs : This fork's Celery task tracking (TaskQueueState). + - index_attempt : Indexing run history (DISABLED by default; opt in + by setting RETENTION_DAYS_INDEX_ATTEMPT to a positive + integer). + - permission_sync_run : Permission sync run history (terminal rows only — + `in_progress` rows are kept regardless of age). + - usage_reports : UsageReport rows + their associated file_store rows + and LO blobs (FK-safe deletion). + - chat : chat_session + chat_message + chat-attached + file_store rows + LO blobs + orphaned search_doc + rows (mirrors db/chat.py UI-driven delete path). + +Run from the daily beat task `run_retention_policies_task` or one-shot +via `backend/scripts/cleanup_stale_db.py`. + +NOTE on disk reclamation: Postgres `DELETE` only marks rows dead — disk +space is reclaimed by autovacuum (or manual `VACUUM`) at some later +point. For typical daily sweeps this is a non-issue; autovacuum keeps up. +For the FIRST run against accumulated bloat (e.g. millions of orphaned +kombu_message rows), expect: + 1. Disk usage stays the same immediately after the DELETE. + 2. autovacuum will reclaim space within minutes-to-hours. + 3. If you need to reclaim immediately (e.g. low disk pressure): + VACUUM (VERBOSE, ANALYZE) kombu_message; + `VACUUM FULL` is also possible but takes an AccessExclusiveLock and + blocks all reads/writes — avoid unless absolutely necessary. +""" +from __future__ import annotations + +import os +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from datetime import timedelta +from datetime import timezone + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.pg_file_store import delete_lobj_by_name +from danswer.utils.logger import setup_logger + +logger = setup_logger() + + +# --------------------------------------------------------------------------- +# Defaults — overridable by env var +# --------------------------------------------------------------------------- + + +def _env_int(name: str, default: int, minimum: int = 0) -> int: + raw = os.environ.get(name, "").strip() + if not raw: + return default + try: + v = int(raw) + return max(minimum, v) + except ValueError: + logger.warning(f"Invalid {name}={raw!r}; using default {default}") + return default + + +RETENTION_DAYS_KOMBU = _env_int("RETENTION_DAYS_KOMBU", 7) +RETENTION_DAYS_TASK_QUEUE = _env_int("RETENTION_DAYS_TASK_QUEUE", 30) +# index_attempt rows are retained indefinitely by default — they're the +# debug history for every indexing run, including failures, and operators +# tend to want them around for postmortems on slow connectors. The +# `index_attempt` policy stays in the registry so you can opt into time- +# based + keep-last-N pruning by setting RETENTION_DAYS_INDEX_ATTEMPT to a +# positive integer; the executor short-circuits when days <= 0. +RETENTION_DAYS_INDEX_ATTEMPT = _env_int("RETENTION_DAYS_INDEX_ATTEMPT", 0) +RETENTION_DAYS_CHAT = _env_int("RETENTION_DAYS_CHAT", 30) +RETENTION_DAYS_USAGE_REPORTS = _env_int("RETENTION_DAYS_USAGE_REPORTS", 90) +RETENTION_DAYS_PERMISSION_SYNC = _env_int("RETENTION_DAYS_PERMISSION_SYNC", 30) +RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS = _env_int( + "RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS", 20 +) + +# Batch size for incremental DELETEs. Each batch is a separate transaction +# so locks release between iterations and concurrent writers (Celery +# enqueue, indexer status updates) aren't starved. 5000 is a reasonable +# default — large enough to be efficient, small enough that any single +# DELETE finishes in well under a second on a modern Postgres. +RETENTION_BATCH_SIZE = _env_int("RETENTION_BATCH_SIZE", 5000, minimum=100) + +# Hard ceiling on number of batches per policy per run. Stops a runaway +# DELETE from spinning forever if `criterion < cutoff` somehow keeps +# matching new rows (e.g. clock skew, very large windows). At default +# settings, 200 * 5000 = 1M rows per policy per run; the next daily run +# picks up where this one stopped. +RETENTION_MAX_BATCHES = _env_int("RETENTION_MAX_BATCHES", 200, minimum=1) + +# Postgres advisory lock id, arbitrary 64-bit int. Keeps two concurrent +# retention runs from racing. Picked from /dev/urandom; no domain meaning. +_RETENTION_ADVISORY_LOCK_ID = 0x52455445_4E54494F # b"RETENTIO" packed + + +# --------------------------------------------------------------------------- +# Policy framework +# --------------------------------------------------------------------------- + +# A policy function takes (db_session, cutoff, batch_size, dry_run) and +# returns the number of rows it deleted (or *would* delete in dry-run). +# Function-based policies are responsible for their own batching + commit +# loop. SQL-based policies use the framework's batching helper. +PolicyFn = Callable[[Session, datetime, int, bool], int] + + +@dataclass +class RetentionPolicy: + name: str + days: int + + # SQL-based policy: provide both a `count_sql` (used in dry-run, no + # locks) and a `delete_sql` that selects up to `:batch_size` rows by + # primary key and deletes them. Both should reference `:cutoff`. The + # framework loops `delete_sql` until rowcount == 0 (or + # RETENTION_MAX_BATCHES, whichever comes first). + count_sql: str | None = None + delete_sql: str | None = None + + # Function-based policy: for multi-table deletes that need explicit + # FK ordering. The function handles its own batching + commits. + fn: PolicyFn | None = None + + # Optional table to ANALYZE after a substantial purge (>= one full + # batch). Postgres planner stats can drift after deleting millions of + # rows; ANALYZE refreshes them so subsequent queries pick good plans. + # Only meaningful for SQL-based policies — function policies that + # touch multiple tables can do their own ANALYZE if needed. + analyze_table: str | None = None + + enabled: bool = True + + def __post_init__(self) -> None: + sql_set = self.count_sql is not None and self.delete_sql is not None + fn_set = self.fn is not None + if sql_set == fn_set: + raise ValueError( + f"RetentionPolicy {self.name!r} must set EITHER both " + "(count_sql + delete_sql) OR fn — not both / neither." + ) + + +# --------------------------------------------------------------------------- +# Per-table policies +# --------------------------------------------------------------------------- + +# Kombu's queue storage. The `visible=true` filter is the safety net — +# we never delete a message a worker is currently holding (those have +# `visible=false` until the worker acks or the visibility timeout +# expires). Old + visible = orphaned message that no consumer will ever +# pick up; safe to drop. Batched on `id` (kombu_message PK). +# +# `visible = true` is repeated in the OUTER DELETE WHERE — not just the +# inner SELECT — so a worker leasing a row between the snapshot and the +# DELETE (flipping visible to false) is protected. Without the outer +# filter, the DELETE would match by id alone and could nuke an in-flight +# message. Belt and suspenders. +# +# `kombu_message.timestamp` is tz-naive UTC by Kombu's transport schema; +# the tz-aware `:cutoff` we bind has its tz stripped at psycopg2 bind +# time. Both sides represent UTC, so the comparison is correct in +# practice — but if the Kombu schema ever changes, revisit this. +_KOMBU_COUNT_SQL = """ + SELECT count(*) FROM kombu_message + WHERE timestamp < :cutoff AND visible = true +""" +_KOMBU_DELETE_SQL = """ + DELETE FROM kombu_message + WHERE id IN ( + SELECT id FROM kombu_message + WHERE timestamp < :cutoff AND visible = true + ORDER BY id + LIMIT :batch_size + ) + AND visible = true +""" + +# Our own task tracking table. Only delete rows in terminal status — +# never drop a PENDING / STARTED row, those may be alive even if old +# (e.g. a stuck deletion task from a dead worker that we still need to +# notice). +_TASK_QUEUE_COUNT_SQL = """ + SELECT count(*) FROM task_queue_jobs + WHERE register_time < :cutoff + AND status IN ('SUCCESS', 'FAILURE') +""" +_TASK_QUEUE_DELETE_SQL = """ + DELETE FROM task_queue_jobs + WHERE id IN ( + SELECT id FROM task_queue_jobs + WHERE register_time < :cutoff + AND status IN ('SUCCESS', 'FAILURE') + ORDER BY id + LIMIT :batch_size + ) +""" + +# Index attempt history. Keep the last N per (connector, credential, +# embedding_model) so paused-but-not-deleted connectors don't lose all +# their debug history. Only TERMINAL rows are eligible. Time bound is +# secondary — if there are still 20 newer attempts, even a 90-day-old +# row is kept. Batched by limiting the outer DELETE. +# +# Status filter uses UPPERCASE values: the column type is +# `Enum(IndexingStatus, native_enum=False)` which by default stores the +# enum NAME (uppercase 'SUCCESS' / 'FAILED'), not its `.value` (lowercase). +# Verified via `SELECT status FROM index_attempt LIMIT 1`. To store the +# values instead you'd need `values_callable=lambda x: [e.value for e in x]` +# in the column declaration — but the schema doesn't, so uppercase wins. +# There is no CANCELED status in this fork's enum (only in upstream Onyx). +# Don't add it back without checking `db/enums.py`. +_INDEX_ATTEMPT_COUNT_SQL = """ + SELECT count(*) FROM ( + SELECT + id, + row_number() OVER ( + PARTITION BY connector_id, credential_id, embedding_model_id + ORDER BY time_created DESC + ) AS rn + FROM index_attempt + WHERE status IN ('SUCCESS', 'FAILED') + AND time_created < :cutoff + ) ranked + WHERE rn > :keep_n +""" +_INDEX_ATTEMPT_DELETE_SQL = """ + DELETE FROM index_attempt + WHERE id IN ( + SELECT id FROM ( + SELECT + id, + row_number() OVER ( + PARTITION BY connector_id, credential_id, embedding_model_id + ORDER BY time_created DESC + ) AS rn + FROM index_attempt + WHERE status IN ('SUCCESS', 'FAILED') + AND time_created < :cutoff + ) ranked + WHERE rn > :keep_n + LIMIT :batch_size + ) +""" + + +# Permission sync run history. Only delete TERMINAL rows (`success`, +# `failed`); never drop `in_progress` rows even if old, since those may +# represent stuck syncs an operator still needs to notice. +_PERMISSION_SYNC_COUNT_SQL = """ + SELECT count(*) FROM permission_sync_run + WHERE updated_at < :cutoff + AND status IN ('success', 'failed') +""" +_PERMISSION_SYNC_DELETE_SQL = """ + DELETE FROM permission_sync_run + WHERE id IN ( + SELECT id FROM permission_sync_run + WHERE updated_at < :cutoff + AND status IN ('success', 'failed') + ORDER BY id + LIMIT :batch_size + ) +""" + + +def _delete_old_usage_reports( + db_session: Session, + cutoff: datetime, + batch_size: int, + dry_run: bool, +) -> int: + """Delete UsageReport rows older than `cutoff` along with their + associated `file_store` row + LO blob. + + `usage_reports.report_name` is a FK to `file_store.file_name`, so we + delete the UsageReport row first (releasing the FK) and only then + drop the file_store row + unlink the LO. Otherwise the file_store + delete would fail with FK violation while the report still references + it. + + `delete_lobj_by_name` commits per-file; we keep batches small so the + overall sweep makes incremental progress even if interrupted. + """ + rows = db_session.execute( + text( + "SELECT id, report_name FROM usage_reports " + "WHERE time_created < :cutoff ORDER BY id" + ), + {"cutoff": cutoff}, + ).all() + if not rows: + return 0 + + if dry_run: + logger.info( + f"usage_reports dry-run: {len(rows)} reports would be deleted " + "(plus their file_store rows and LO blobs)" + ) + return len(rows) + + total = 0 + for batch_start in range(0, len(rows), batch_size): + batch = rows[batch_start : batch_start + batch_size] + batch_ids = [r[0] for r in batch] + batch_names = [r[1] for r in batch] + + # 1. Drop the usage_reports rows so the FK on report_name is + # released. Commit immediately so the LO+file_store deletes + # below run in their own short transactions. + result = db_session.execute( + text("DELETE FROM usage_reports WHERE id = ANY(:ids)"), + {"ids": batch_ids}, + ) + total += int(result.rowcount or 0) + db_session.commit() + + # 2. Drop each LO blob + file_store row. Per-file because + # `delete_lobj_by_name` does its own lookup + commit; if a row + # is already gone (concurrent admin delete, etc.) it logs and + # moves on rather than blowing up the sweep. + for name in batch_names: + try: + delete_lobj_by_name(name, db_session) + except Exception as e: + logger.warning( + f"usage_reports: failed to delete file_store/{name}: " + f"{e}; usage_reports row already gone, file_store row " + "may now be orphaned. Continuing." + ) + + return total + + +def _delete_old_chat( + db_session: Session, + cutoff: datetime, + batch_size: int, + dry_run: bool, +) -> int: + """Delete chat sessions whose newest message is older than `cutoff`, + along with their messages and the FK-dependent rows. + + Order matters because two FKs on `chat_message` have no ON DELETE + CASCADE / SET NULL: `chat_message__search_doc.chat_message_id` and + `tool_call.message_id`. We delete those first, then messages, then + sessions. `document_retrieval_feedback` and `chat_feedback` self- + handle via `ON DELETE SET NULL`. + + "Newest message older than cutoff" is the right boundary because + `chat_session.time_updated` doesn't get bumped when a new message is + inserted (the FK is one-way; `onupdate=func.now()` only fires on + UPDATEs to chat_session itself). + + Batching: we identify session IDs once (the JOIN is the expensive + part), then delete the dependent + parent rows in chunks of + `batch_size`, committing between chunks. For dry-run we just count. + """ + # 1. Find session IDs whose newest message (or session creation, for + # sessions with no messages) is older than the cutoff. + session_id_rows = db_session.execute( + text( + """ + SELECT cs.id + FROM chat_session cs + LEFT JOIN chat_message cm ON cm.chat_session_id = cs.id + GROUP BY cs.id + HAVING COALESCE(MAX(cm.time_sent), cs.time_created) < :cutoff + """ + ), + {"cutoff": cutoff}, + ).all() + session_ids = [r[0] for r in session_id_rows] + if not session_ids: + return 0 + + if dry_run: + # Count messages in those sessions (so the user sees both + # numbers); return session count as the headline. + msg_count = ( + db_session.execute( + text( + "SELECT count(*) FROM chat_message " + "WHERE chat_session_id = ANY(:ids)" + ), + {"ids": session_ids}, + ).scalar() + or 0 + ) + # Count search_doc rows that would become orphans (or are already). + # The query mirrors the orphan-cleanup DELETE at the end of the + # real path, but bounded to a counter only. + orphan_count = ( + db_session.execute( + text( + "SELECT count(*) FROM search_doc sd " + "LEFT JOIN chat_message__search_doc cmsd " + " ON cmsd.search_doc_id = sd.id " + "WHERE cmsd.chat_message_id IS NULL" + ) + ).scalar() + or 0 + ) + logger.info( + f"chat dry-run: {len(session_ids)} sessions would be deleted " + f"(plus {msg_count} messages, " + "chat_message__search_doc / tool_call rows, " + "any chat_message.files LO blobs, and NULLed feedback FKs); " + f"+ {orphan_count} currently-orphan search_doc rows would be " + "swept up at the end." + ) + return len(session_ids) + + total_sessions_deleted = 0 + # Process sessions in batches so each transaction stays small. + for batch_start in range(0, len(session_ids), batch_size): + batch_session_ids = session_ids[batch_start : batch_start + batch_size] + + # FK race guard: chat_message.chat_session_id has no `ondelete=` + # in the schema (defaults to NO ACTION). Without locking, a new + # chat_message INSERT on one of these sessions between our + # message-DELETE and session-DELETE would cause the session + # DELETE to fail with FK violation, rolling back the whole batch. + # `SELECT ... FOR UPDATE` takes a row-level FOR UPDATE lock, + # which conflicts with the KEY SHARE lock that any FK-validating + # INSERT into chat_message must take on its parent chat_session + # row — so concurrent inserts block until our batch commits. + # Sessions older than RETENTION_DAYS_CHAT (default 30d) almost + # never see new activity, so the lock wait is effectively zero. + db_session.execute( + text("SELECT id FROM chat_session " "WHERE id = ANY(:ids) FOR UPDATE"), + {"ids": batch_session_ids}, + ) + + # Find this batch's message IDs. + message_id_rows = db_session.execute( + text("SELECT id FROM chat_message " "WHERE chat_session_id = ANY(:ids)"), + {"ids": batch_session_ids}, + ).all() + message_ids = [r[0] for r in message_id_rows] + + # Capture file blob names referenced by these messages BEFORE + # we delete them. The `files` JSONB column holds entries like + # `{"id": "", ...}`; each one is backed + # by a `file_store` row + a Postgres LO blob. We unlink them + # AFTER the batch commit (delete_lobj_by_name commits internally, + # which would otherwise release our FOR UPDATE locks mid-batch). + # Mirrors db/chat.py::delete_messages_and_files_from_chat_session. + blob_names: list[str] = [] + if message_ids: + files_rows = db_session.execute( + text( + "SELECT files FROM chat_message " + "WHERE id = ANY(:ids) AND files IS NOT NULL" + ), + {"ids": message_ids}, + ).all() + for (files_json,) in files_rows: + for file_info in files_json or []: + if not isinstance(file_info, dict): + continue + name = file_info.get("id") + if name: + blob_names.append(name) + + if message_ids: + # 2a. Clean join table — no ondelete on chat_message_id. + db_session.execute( + text( + "DELETE FROM chat_message__search_doc " + "WHERE chat_message_id = ANY(:ids)" + ), + {"ids": message_ids}, + ) + # 2b. Clean tool_call — no ondelete on message_id. + db_session.execute( + text("DELETE FROM tool_call WHERE message_id = ANY(:ids)"), + {"ids": message_ids}, + ) + # 2c. Delete chat_message — document_retrieval_feedback and + # chat_feedback FKs auto-NULL via ON DELETE SET NULL. + db_session.execute( + text("DELETE FROM chat_message WHERE id = ANY(:ids)"), + {"ids": message_ids}, + ) + + # 3. Delete sessions in this batch. + result = db_session.execute( + text("DELETE FROM chat_session WHERE id = ANY(:ids)"), + {"ids": batch_session_ids}, + ) + total_sessions_deleted += int(result.rowcount or 0) + + # Commit per-batch so row-level locks release and other writers + # can make progress. The retention advisory lock is session- + # scoped in Postgres, so it survives the commit; the FOR UPDATE + # locks taken above are released. + db_session.commit() + + # POST-COMMIT: unlink file blobs. Sessions are now deleted, so + # FK constraints prevent any new chat_message rows from referring + # to them — safe to do this outside the batch transaction. + # delete_lobj_by_name swallows missing-file cases internally. + for name in blob_names: + try: + delete_lobj_by_name(name, db_session) + except Exception as e: + logger.warning( + f"chat retention: failed to unlink file blob {name!r}: " + f"{e}. file_store row may now be orphaned." + ) + + # Clean up search_doc rows that are no longer referenced by any + # chat_message__search_doc. Mirrors db/chat.py::delete_orphaned_search_docs + # (called by every UI-driven chat delete) — without this, retention + # silently leaks search_doc rows. Batched + capped at the same + # safety ceilings as SQL policies. + total_orphans = 0 + orphan_hit_ceiling = False + for _ in range(RETENTION_MAX_BATCHES): + orphan_result = db_session.execute( + text( + "DELETE FROM search_doc WHERE id IN (" + " SELECT sd.id FROM search_doc sd " + " LEFT JOIN chat_message__search_doc cmsd " + " ON cmsd.search_doc_id = sd.id " + " WHERE cmsd.chat_message_id IS NULL " + " ORDER BY sd.id " + " LIMIT :batch_size" + ")" + ), + {"batch_size": batch_size}, + ) + n = int(orphan_result.rowcount or 0) + db_session.commit() + total_orphans += n + if n < batch_size: + break + else: + orphan_hit_ceiling = True + + if total_orphans > 0: + logger.info( + f"chat retention: cleaned up {total_orphans} orphan search_doc rows" + ) + if orphan_hit_ceiling: + logger.warning( + f"chat retention: orphan search_doc cleanup hit " + f"RETENTION_MAX_BATCHES ({RETENTION_MAX_BATCHES}); " + f"cleaned {total_orphans} this run, remainder picks up next sweep." + ) + + return total_sessions_deleted + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + +RETENTION_POLICIES: dict[str, RetentionPolicy] = { + "kombu_message": RetentionPolicy( + name="kombu_message", + days=RETENTION_DAYS_KOMBU, + count_sql=_KOMBU_COUNT_SQL, + delete_sql=_KOMBU_DELETE_SQL, + analyze_table="kombu_message", + ), + "task_queue_jobs": RetentionPolicy( + name="task_queue_jobs", + days=RETENTION_DAYS_TASK_QUEUE, + count_sql=_TASK_QUEUE_COUNT_SQL, + delete_sql=_TASK_QUEUE_DELETE_SQL, + analyze_table="task_queue_jobs", + ), + "index_attempt": RetentionPolicy( + name="index_attempt", + days=RETENTION_DAYS_INDEX_ATTEMPT, + count_sql=_INDEX_ATTEMPT_COUNT_SQL, + delete_sql=_INDEX_ATTEMPT_DELETE_SQL, + analyze_table="index_attempt", + ), + "permission_sync_run": RetentionPolicy( + name="permission_sync_run", + days=RETENTION_DAYS_PERMISSION_SYNC, + count_sql=_PERMISSION_SYNC_COUNT_SQL, + delete_sql=_PERMISSION_SYNC_DELETE_SQL, + analyze_table="permission_sync_run", + ), + "usage_reports": RetentionPolicy( + name="usage_reports", + days=RETENTION_DAYS_USAGE_REPORTS, + fn=_delete_old_usage_reports, + ), + "chat": RetentionPolicy( + name="chat", + days=RETENTION_DAYS_CHAT, + fn=_delete_old_chat, + ), +} + + +# --------------------------------------------------------------------------- +# Executor +# --------------------------------------------------------------------------- + + +def _try_advisory_lock(db_session: Session) -> bool: + """Postgres advisory lock — non-blocking. Returns True if acquired.""" + row = db_session.execute( + text("SELECT pg_try_advisory_lock(:lock_id)"), + {"lock_id": _RETENTION_ADVISORY_LOCK_ID}, + ).scalar() + return bool(row) + + +def _release_advisory_lock(db_session: Session) -> None: + db_session.execute( + text("SELECT pg_advisory_unlock(:lock_id)"), + {"lock_id": _RETENTION_ADVISORY_LOCK_ID}, + ) + + +def run_retention_policies( + dry_run: bool = False, + only: list[str] | None = None, +) -> dict[str, int]: + """Execute every enabled policy. Returns {policy_name: rows_deleted}. + + `dry_run`: prints the row counts that *would* be deleted via cheap + SELECT count(*) queries — does NOT scan or lock for the DELETE side. + Useful for the first run after a long period of accumulated bloat. + + `only`: optional list of policy names to run (others skipped). Used + by `backend/scripts/cleanup_stale_db.py --policy=...`. + + Each policy runs in its own batched loop with per-batch commits, so + a multi-million-row first run won't hold a table lock for minutes. + The advisory lock is session-scoped and survives mid-loop commits. + """ + results: dict[str, int] = {} + engine = get_sqlalchemy_engine() + with Session(engine) as db_session: + if not _try_advisory_lock(db_session): + logger.warning("Retention: advisory lock held by another run; skipping.") + return results + try: + now = datetime.now(tz=timezone.utc) + for name, policy in RETENTION_POLICIES.items(): + if only and name not in only: + continue + if not policy.enabled: + logger.info(f"Retention: {name} disabled, skipping.") + continue + if policy.days <= 0: + logger.info( + f"Retention: {name} retention=0d, treating as disabled " + "(set the corresponding env var to a positive integer " + "to enable)." + ) + continue + + cutoff = now - timedelta(days=policy.days) + started = time.monotonic() + try: + deleted = _run_one(db_session, policy, cutoff, dry_run) + except Exception as e: + logger.exception( + f"Retention: {name} failed after " + f"{time.monotonic() - started:.2f}s: {e}. " + "Rolling back this policy and continuing with the rest." + ) + db_session.rollback() + continue + + elapsed = time.monotonic() - started + results[name] = deleted + if dry_run: + logger.info( + f"Retention [DRY RUN]: {name}: would delete {deleted} " + f"rows older than {policy.days}d " + f"(cutoff {cutoff.isoformat()}, took {elapsed:.2f}s)" + ) + elif deleted > 0: + logger.info( + f"Retention: {name}: deleted {deleted} rows older " + f"than {policy.days}d " + f"(cutoff {cutoff.isoformat()}, took {elapsed:.2f}s)" + ) + else: + # No-op runs are common in steady state — keep them at + # debug level so daily logs stay quiet. The summary line + # in the wrapping Celery task still announces the + # overall result. + logger.debug( + f"Retention: {name}: nothing eligible for deletion " + f"(took {elapsed:.2f}s)" + ) + finally: + # Rollback first — if any policy errored out and we're still + # inside its transaction, the next SQL would raise "current + # transaction is aborted, commands ignored until end of + # transaction block", and the unlock would never run. The + # advisory lock would then sit on this connection forever + # (or until the connection drops out of the pool), blocking + # every future retention run with "advisory lock held by + # another run; skipping". Rollback first puts the session + # back into a clean state so the unlock can run. + try: + db_session.rollback() + except Exception: + pass + try: + _release_advisory_lock(db_session) + db_session.commit() + except Exception: + logger.exception( + "Retention: advisory unlock failed; the lock will be " + "released when this DB connection drops out of the " + "SQLAlchemy pool. Until then, subsequent retention " + "runs on this same connection will skip." + ) + return results + + +def _run_one( + db_session: Session, + policy: RetentionPolicy, + cutoff: datetime, + dry_run: bool, +) -> int: + """Execute one policy. SQL policies loop their delete_sql in batches of + RETENTION_BATCH_SIZE, committing between batches. Function policies + handle their own batching.""" + # Function-based policies (chat) — they handle batching + commits + # internally and obey dry-run themselves. + if policy.fn is not None: + return policy.fn(db_session, cutoff, RETENTION_BATCH_SIZE, dry_run) + + assert policy.count_sql is not None and policy.delete_sql is not None + + params: dict = {"cutoff": cutoff, "batch_size": RETENTION_BATCH_SIZE} + if policy.name == "index_attempt": + params["keep_n"] = RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS + + # Dry-run: cheap SELECT count, no DELETE scan, no locks. + if dry_run: + count_params = {k: v for k, v in params.items() if k != "batch_size"} + n = db_session.execute(text(policy.count_sql), count_params).scalar() or 0 + return int(n) + + # Real run: loop DELETE … LIMIT :batch_size until empty (or the safety + # ceiling RETENTION_MAX_BATCHES is hit). Commit per batch so locks + # release and concurrent writers aren't blocked. + total = 0 + hit_ceiling = False + for i in range(RETENTION_MAX_BATCHES): + result = db_session.execute(text(policy.delete_sql), params) + n = int(result.rowcount or 0) + db_session.commit() + total += n + if n < RETENTION_BATCH_SIZE: + # Last batch was partial → no more rows match → done. + break + logger.debug( + f"Retention: {policy.name}: batch {i + 1} deleted {n} rows " + f"(running total: {total})" + ) + else: + hit_ceiling = True + + if hit_ceiling: + logger.warning( + f"Retention: {policy.name}: hit RETENTION_MAX_BATCHES " + f"({RETENTION_MAX_BATCHES}); deleted {total} rows this run, " + "remainder will be picked up by the next scheduled sweep. " + "If this happens persistently, raise RETENTION_MAX_BATCHES " + "or run cleanup_stale_db.py manually." + ) + + # Refresh planner stats after substantial purges. Threshold = one full + # batch — small steady-state cleanups don't need it. ANALYZE doesn't + # take heavy locks (just SHARE UPDATE EXCLUSIVE on stats), but it is + # I/O, so we gate on volume. + if policy.analyze_table and total >= RETENTION_BATCH_SIZE: + try: + db_session.execute(text(f"ANALYZE {policy.analyze_table}")) + db_session.commit() + logger.info( + f"Retention: {policy.name}: ran ANALYZE " + f"{policy.analyze_table} after deleting {total} rows" + ) + except Exception as e: + logger.warning( + f"Retention: {policy.name}: ANALYZE {policy.analyze_table} " + f"failed: {e}. Stats will refresh on next autovacuum cycle." + ) + db_session.rollback() + + return total diff --git a/backend/danswer/db/tasks.py b/backend/danswer/db/tasks.py index 23a7edc9882..58bd605aa36 100644 --- a/backend/danswer/db/tasks.py +++ b/backend/danswer/db/tasks.py @@ -26,6 +26,39 @@ def get_latest_task( return latest_task +def get_latest_tasks_by_names( + task_names: list[str], + db_session: Session, +) -> dict[str, TaskQueueState]: + """Bulk equivalent of `get_latest_task` for many task names at once. + + Returns a dict keyed by task_name pointing at the most recent + TaskQueueState row for that name. Names with no matching rows are + omitted from the result. One round-trip regardless of N. + """ + if not task_names: + return {} + + # First find the max id per task_name (small subquery), then join back to + # fetch the full row. This is the standard "latest-per-group" pattern in + # Postgres, fully covered by an index on (task_name, id DESC). + latest_ids_subq = ( + select( + TaskQueueState.task_name, + func.max(TaskQueueState.id).label("max_id"), + ) + .where(TaskQueueState.task_name.in_(task_names)) + .group_by(TaskQueueState.task_name) + .subquery() + ) + stmt = select(TaskQueueState).join( + latest_ids_subq, + TaskQueueState.id == latest_ids_subq.c.max_id, + ) + rows = db_session.execute(stmt).scalars().all() + return {row.task_name: row for row in rows} + + def get_latest_task_by_type( task_name: str, db_session: Session, diff --git a/backend/danswer/main.py b/backend/danswer/main.py index ffbb491da26..54a9dbd45bb 100644 --- a/backend/danswer/main.py +++ b/backend/danswer/main.py @@ -52,6 +52,7 @@ from danswer.llm.llm_initialization import load_llm_providers from danswer.search.retrieval.search_runner import download_nltk_data from danswer.search.search_nlp_models import warm_up_encoders +from danswer.server.analytics.api import router as analytics_router from danswer.server.auth_check import check_router_auth from danswer.server.danswer_api.ingestion import router as danswer_api_router from danswer.server.documents.cc_pair import router as cc_pair_router @@ -290,6 +291,7 @@ def get_application() -> FastAPI: include_router_with_global_prefix_prepended( application, token_rate_limit_settings_router ) + include_router_with_global_prefix_prepended(application, analytics_router) if AUTH_TYPE == AuthType.DISABLED: # Server logs this during auth setup verification step diff --git a/backend/danswer/server/analytics/__init__.py b/backend/danswer/server/analytics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/server/analytics/api.py b/backend/danswer/server/analytics/api.py new file mode 100644 index 00000000000..dbd773a5a25 --- /dev/null +++ b/backend/danswer/server/analytics/api.py @@ -0,0 +1,188 @@ +"""Admin analytics endpoints (community / open-source). + +Lives in `danswer.server.analytics`, parallel to (not depending on) the +EE module at `ee.danswer.server.analytics`. Same routes (`/analytics/...`) +and same response shapes — the frontend can target either backend +without code changes. + +If the EE build also registers its analytics router, FastAPI ends up +with duplicate routes; the community one is registered first by +`danswer.main:get_application` and wins. Removing the EE registration +is the cleanest way to avoid that ambiguity, but it's not required for +correctness. +""" +import datetime + +from fastapi import APIRouter +from fastapi import Depends +from pydantic import BaseModel +from sqlalchemy.orm import Session + +import danswer.db.models as db_models +from danswer.auth.users import current_admin_user +from danswer.db.analytics import fetch_docs_per_source +from danswer.db.analytics import fetch_slack_bot_channel_stats +from danswer.db.analytics import fetch_total_docs_indexed +from danswer.db.analytics_rollup import fetch_danswerbot_analytics_from_rollup +from danswer.db.analytics_rollup import fetch_query_analytics_from_rollup +from danswer.db.analytics_rollup import fetch_user_analytics_from_rollup +from danswer.db.engine import get_session + +router = APIRouter(prefix="/analytics") + + +class QueryAnalyticsResponse(BaseModel): + total_queries: int + total_likes: int + total_dislikes: int + # Slackbot resolved-button presses (predefined_feedback='resolved'). + # Counted as a positive signal alongside likes for "strict NPS". + total_resolved: int + # Slackbot "I need more help" presses (required_followup=true). Counted + # as a negative signal alongside dislikes for "strict NPS". + total_needs_help: int + date: datetime.date + + +@router.get("/admin/query") +def get_query_analytics( + start: datetime.datetime | None = None, + end: datetime.datetime | None = None, + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[QueryAnalyticsResponse]: + """Daily query volume + feedback breakdown. Default window: last 30 days. + + Reads from `analytics_daily_rollup`, populated nightly at 07:30 UTC. + Days that haven't been rolled up yet (e.g. today before 07:30) are + simply absent from the response. + """ + daily_query_usage_info = fetch_query_analytics_from_rollup( + start=start or (datetime.datetime.utcnow() - datetime.timedelta(days=30)), + end=end or datetime.datetime.utcnow(), + db_session=db_session, + ) + return [ + QueryAnalyticsResponse( + total_queries=total_queries, + total_likes=total_likes, + total_dislikes=total_dislikes, + total_resolved=total_resolved, + total_needs_help=total_needs_help, + date=date, + ) + for ( + total_queries, + total_likes, + total_dislikes, + total_resolved, + total_needs_help, + date, + ) in daily_query_usage_info + ] + + +class UserAnalyticsResponse(BaseModel): + total_active_users: int + date: datetime.date + + +@router.get("/admin/user") +def get_user_analytics( + start: datetime.datetime | None = None, + end: datetime.datetime | None = None, + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[UserAnalyticsResponse]: + """Distinct active users per day, served from `analytics_daily_rollup`.""" + rows = fetch_user_analytics_from_rollup( + start=start or (datetime.datetime.utcnow() - datetime.timedelta(days=30)), + end=end or datetime.datetime.utcnow(), + db_session=db_session, + ) + return [ + UserAnalyticsResponse(total_active_users=int(active_users), date=date) + for active_users, date in rows + ] + + +class DanswerbotAnalyticsResponse(BaseModel): + total_queries: int + auto_resolved: int + date: datetime.date + + +@router.get("/admin/danswerbot") +def get_danswerbot_analytics( + start: datetime.datetime | None = None, + end: datetime.datetime | None = None, + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[DanswerbotAnalyticsResponse]: + """Slackbot daily stats from `analytics_daily_rollup`. The rollup + already stores `slackbot_auto_resolved` (clamped to ≥0 at write + time) so we pass it straight through.""" + rows = fetch_danswerbot_analytics_from_rollup( + start=start or (datetime.datetime.utcnow() - datetime.timedelta(days=30)), + end=end or datetime.datetime.utcnow(), + db_session=db_session, + ) + return [ + DanswerbotAnalyticsResponse( + total_queries=int(total_queries), + auto_resolved=int(auto_resolved), + date=date, + ) + for total_queries, auto_resolved, date in rows + ] + + +# --------------------------------------------------------------------------- +# Snapshot endpoints (no date range — current state) +# --------------------------------------------------------------------------- + + +class TotalDocsResponse(BaseModel): + total_docs_indexed: int + unique_docs: int + + +@router.get("/admin/total-docs") +def get_total_docs( + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> TotalDocsResponse: + total, unique = fetch_total_docs_indexed(db_session) + return TotalDocsResponse(total_docs_indexed=total, unique_docs=unique) + + +class DocsPerSourceRow(BaseModel): + source: str + docs_indexed: int + + +@router.get("/admin/docs-per-source") +def get_docs_per_source( + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[DocsPerSourceRow]: + return [ + DocsPerSourceRow(source=src, docs_indexed=n) + for src, n in fetch_docs_per_source(db_session) + ] + + +class SlackChannelsResponse(BaseModel): + total_configs: int + enabled_channels: int + + +@router.get("/admin/slack-channels") +def get_slack_channels( + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> SlackChannelsResponse: + total_configs, enabled_channels = fetch_slack_bot_channel_stats(db_session) + return SlackChannelsResponse( + total_configs=total_configs, enabled_channels=enabled_channels + ) diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py index 6ca9d23ab90..78e4df06a45 100644 --- a/backend/danswer/server/documents/connector.py +++ b/backend/danswer/server/documents/connector.py @@ -14,7 +14,7 @@ from danswer.auth.api_key import validate_api_key from danswer.auth.users import current_admin_user from danswer.auth.users import current_user -from danswer.background.celery.celery_utils import get_deletion_status +from danswer.background.task_utils import name_cc_cleanup_task from danswer.configs.app_configs import ENABLED_CONNECTOR_TYPES from danswer.configs.constants import DocumentSource from danswer.configs.constants import FileOrigin @@ -57,7 +57,6 @@ from danswer.db.credentials import delete_gmail_service_account_credentials from danswer.db.credentials import delete_google_drive_service_account_credentials from danswer.db.credentials import fetch_credential_by_id -from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed from danswer.db.document import get_document_cnts_for_cc_pairs from danswer.db.embedding_model import get_current_db_embedding_model from danswer.db.engine import get_session @@ -66,7 +65,10 @@ from danswer.db.index_attempt import create_index_attempt from danswer.db.index_attempt import get_index_attempts_for_cc_pair from danswer.db.index_attempt import get_latest_index_attempts +from danswer.db.index_attempt import update_index_attempt_priority +from danswer.db.models import IndexingStatus from danswer.db.models import User +from danswer.db.tasks import get_latest_tasks_by_names from danswer.dynamic_configs.interface import ConfigNotFoundError from danswer.file_store.file_store import get_default_file_store from danswer.server.documents.models import AuthStatus @@ -76,6 +78,7 @@ from danswer.server.documents.models import ConnectorIndexingStatus from danswer.server.documents.models import ConnectorSnapshot from danswer.server.documents.models import CredentialSnapshot +from danswer.server.documents.models import DeletionAttemptSnapshot from danswer.server.documents.models import FileUploadResponse from danswer.server.documents.models import GDriveCallback from danswer.server.documents.models import GmailCallback @@ -85,6 +88,7 @@ from danswer.server.documents.models import IndexAttemptSnapshot from danswer.server.documents.models import ObjectCreationIdResponse from danswer.server.documents.models import RunConnectorRequest +from danswer.server.documents.models import UpdateIndexAttemptPriorityRequest from danswer.server.models import StatusResponse _GMAIL_CREDENTIAL_ID_COOKIE_NAME = "gmail_credential_id" @@ -368,13 +372,26 @@ def upload_files( @router.get("/admin/connector/indexing-status") def get_connector_indexing_status( secondary_index: bool = False, + # Optional server-side filter on `connector.disabled`: + # `disabled=false` → only enabled connectors (default page view) + # `disabled=true` → only disabled connectors + # omitted → all connectors + # Drives the network/serialization win on environments with hundreds + # of cc-pairs where most are intentionally paused (e.g. one-off + # historical web-scrape connectors). + disabled: bool | None = None, _: User = Depends(current_admin_user), db_session: Session = Depends(get_session), ) -> list[ConnectorIndexingStatus]: indexing_statuses: list[ConnectorIndexingStatus] = [] - # TODO: make this one query cc_pairs = get_connector_credential_pairs(db_session) + if disabled is not None: + cc_pairs = [ + cc_pair + for cc_pair in cc_pairs + if cc_pair.connector is not None and cc_pair.connector.disabled == disabled + ] cc_pair_identifiers = [ ConnectorCredentialPairIdentifier( connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id @@ -401,6 +418,19 @@ def get_connector_indexing_status( for connector_id, credential_id, cnt in document_count_info } + # Bulk-fetch latest cleanup-task row per cc_pair in one query, replacing + # the previous per-row `get_deletion_status` round-trips. Goes from + # O(N) round-trips to 1. + cleanup_task_names = [ + name_cc_cleanup_task( + connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id + ) + for cc_pair in cc_pairs + ] + cleanup_task_by_name = get_latest_tasks_by_names( + task_names=cleanup_task_names, db_session=db_session + ) + for cc_pair in cc_pairs: # TODO remove this to enable ingestion API if cc_pair.name == "DefaultCCPair": @@ -411,6 +441,31 @@ def get_connector_indexing_status( latest_index_attempt = cc_pair_to_latest_index_attempt.get( (connector.id, credential.id) ) + + # Compute is_deletable inline using data we already have, instead of + # re-querying the DB per row via `check_deletion_attempt_is_allowed`. + # Mirrors the logic of that helper with `allow_scheduled=True`: + # - connector must be disabled (paused) + # - latest index attempt must not be IN_PROGRESS + is_deletable = bool(connector.disabled) and ( + latest_index_attempt is None + or latest_index_attempt.status != IndexingStatus.IN_PROGRESS + ) + + # Build deletion_attempt snapshot from the bulk-fetched task row. + cleanup_task_row = cleanup_task_by_name.get( + name_cc_cleanup_task(connector_id=connector.id, credential_id=credential.id) + ) + deletion_attempt = ( + DeletionAttemptSnapshot( + connector_id=connector.id, + credential_id=credential.id, + status=cleanup_task_row.status, + ) + if cleanup_task_row is not None + else None + ) + indexing_statuses.append( ConnectorIndexingStatus( cc_pair_id=cc_pair.id, @@ -434,18 +489,8 @@ def get_connector_indexing_status( ) if latest_index_attempt else None, - deletion_attempt=get_deletion_status( - connector_id=connector.id, - credential_id=credential.id, - db_session=db_session, - ), - is_deletable=check_deletion_attempt_is_allowed( - connector_credential_pair=cc_pair, - db_session=db_session, - # allow scheduled indexing attempts here, since on deletion request we will cancel them - allow_scheduled=True, - ) - is None, + deletion_attempt=deletion_attempt, + is_deletable=is_deletable, ) ) @@ -592,6 +637,7 @@ def connector_run_once( credential_id=credential_id, embedding_model_id=embedding_model.id, from_beginning=run_info.from_beginning, + indexing_priority=run_info.indexing_priority, db_session=db_session, ) for credential_id in credential_ids @@ -611,6 +657,40 @@ def connector_run_once( ) +@router.patch("/admin/index-attempt/{index_attempt_id}/priority") +def update_index_attempt_priority_route( + index_attempt_id: int, + payload: UpdateIndexAttemptPriorityRequest, + _: User = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> StatusResponse[int]: + """Bumps the priority of an existing NOT_STARTED index attempt. Once + the attempt is in flight (IN_PROGRESS / SUCCESS / FAILED), priority no + longer affects scheduling so we refuse the update — the request is a + no-op and the caller can re-trigger if they need a fresh, higher- + priority run.""" + updated = update_index_attempt_priority( + index_attempt_id=index_attempt_id, + indexing_priority=payload.indexing_priority, + db_session=db_session, + ) + if updated is None: + raise HTTPException( + status_code=400, + detail=( + f"Index attempt {index_attempt_id} not found, or no longer " + "in NOT_STARTED state — priority can only be changed before " + "the attempt is dispatched." + ), + ) + return StatusResponse( + success=True, + message=f"Updated priority of index attempt {index_attempt_id} " + f"to {updated.indexing_priority}", + data=updated.indexing_priority, + ) + + """Endpoints for basic users""" diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py index e02132e741a..9726958b350 100644 --- a/backend/danswer/server/documents/models.py +++ b/backend/danswer/server/documents/models.py @@ -36,6 +36,7 @@ class IndexAttemptSnapshot(BaseModel): full_exception_trace: str | None time_started: str | None time_updated: str + indexing_priority: int = 0 @classmethod def from_index_attempt_db_model( @@ -53,6 +54,7 @@ def from_index_attempt_db_model( if index_attempt.time_started else None, time_updated=index_attempt.time_updated.isoformat(), + indexing_priority=getattr(index_attempt, "indexing_priority", 0) or 0, ) @@ -197,6 +199,14 @@ class RunConnectorRequest(BaseModel): connector_id: int credential_ids: list[int] | None from_beginning: bool = False + # Optional priority for the resulting index attempt(s). Higher values + # are dispatched first by the indexing scheduler. 0 = normal (default + # for auto-scheduled runs), conventional ceiling = 10. + indexing_priority: int = 0 + + +class UpdateIndexAttemptPriorityRequest(BaseModel): + indexing_priority: int """Connectors Models""" diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py index 84d0b390f57..65c8e2d68b0 100644 --- a/backend/danswer/server/manage/administrative.py +++ b/backend/danswer/server/manage/administrative.py @@ -10,6 +10,7 @@ from danswer.auth.api_key import validate_api_key from danswer.auth.users import current_admin_user +from danswer.background.task_utils import name_cc_cleanup_task from danswer.configs.app_configs import GENERATIVE_MODEL_ACCESS_CHECK_FREQ from danswer.configs.constants import DocumentSource from danswer.db.connector_credential_pair import get_connector_credential_pair @@ -20,6 +21,8 @@ from danswer.db.feedback import update_document_hidden from danswer.db.index_attempt import cancel_indexing_attempts_for_connector from danswer.db.models import User +from danswer.db.tasks import check_task_is_live_and_not_timed_out +from danswer.db.tasks import get_latest_task from danswer.document_index.document_index_utils import get_both_index_names from danswer.document_index.factory import get_default_document_index from danswer.dynamic_configs.factory import get_dynamic_config_store @@ -180,6 +183,30 @@ def create_deletion_attempt_for_connector_id( detail=deletion_attempt_disallowed_reason, ) + # Dedup: refuse if a deletion task for this same cc-pair is already + # in flight. The bug we're guarding against — repeated clicks on + # "Delete connector" each call apply_async, which spawns parallel + # workers that all race on `SELECT ... FOR UPDATE NOWAIT` over the + # same documents in `prepare_to_modify_documents`. They retry 10 × + # 30s and all raise "Failed to acquire locks after 10 attempts". + # The worker-side advisory lock in `cleanup_connector_credential_pair_task` + # is the safety net; this 409 is the user-friendly path that avoids + # ever submitting the duplicate work. + cleanup_task_name = name_cc_cleanup_task( + connector_id=connector_id, credential_id=credential_id + ) + latest_cleanup = get_latest_task(task_name=cleanup_task_name, db_session=db_session) + if latest_cleanup and check_task_is_live_and_not_timed_out( + latest_cleanup, db_session + ): + raise HTTPException( + status_code=409, + detail=( + "A deletion is already in progress for this connector. " + "Wait for it to complete before retrying." + ), + ) + cleanup_connector_credential_pair_task.apply_async( kwargs=dict(connector_id=connector_id, credential_id=credential_id), ) diff --git a/backend/scripts/backfill_analytics_rollup.py b/backend/scripts/backfill_analytics_rollup.py new file mode 100644 index 00000000000..61571674436 --- /dev/null +++ b/backend/scripts/backfill_analytics_rollup.py @@ -0,0 +1,94 @@ +"""Populate `analytics_daily_rollup` from existing chat data. + +Run this ONCE after deploying the rollup feature, before the next chat +retention sweep deletes any old data. After this completes, the daily +Celery beat task (`run_analytics_rollup_task`) keeps the table fresh. + +Usage: + + cd backend + PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py + PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py --start=2024-01-01 + PYTHONPATH=$(pwd) python scripts/backfill_analytics_rollup.py --start=2024-01-01 --end=2024-12-31 + +Defaults: + --start: earliest `chat_session.time_created` (or today if no chats). + --end: today (UTC). + +Idempotent — re-running re-upserts the same dates with the same values. +""" +from __future__ import annotations + +import argparse +import datetime +import sys + +from sqlalchemy.orm import Session + +from danswer.db.analytics_rollup import backfill_all_rollups +from danswer.db.analytics_rollup import get_earliest_chat_date +from danswer.db.engine import get_sqlalchemy_engine + + +def _parse_date(s: str) -> datetime.date: + try: + return datetime.date.fromisoformat(s) + except ValueError as e: + raise argparse.ArgumentTypeError( + f"invalid date {s!r}; expected YYYY-MM-DD" + ) from e + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--start", + type=_parse_date, + default=None, + help="Start date YYYY-MM-DD. Defaults to earliest chat_session.", + ) + parser.add_argument( + "--end", + type=_parse_date, + default=None, + help="End date YYYY-MM-DD (inclusive). Defaults to today (UTC).", + ) + args = parser.parse_args() + + engine = get_sqlalchemy_engine() + today = datetime.datetime.now(tz=datetime.timezone.utc).date() + end = args.end or today + + if args.start: + start = args.start + else: + with Session(engine) as db_session: + earliest = get_earliest_chat_date(db_session) + if earliest is None: + print( + "No chat_session rows yet — nothing to backfill. The daily " + "Celery task will start populating from now on.", + file=sys.stderr, + ) + return 0 + start = earliest + + if start > end: + print( + f"--start {start} is after --end {end}; nothing to do.", + file=sys.stderr, + ) + return 2 + + span_days = (end - start).days + 1 + print( + f"Backfilling analytics_daily_rollup for {span_days} day(s) " + f"({start} → {end})…" + ) + n = backfill_all_rollups(start, end) + print(f"Done. {n} day(s) upserted.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/cleanup_stale_db.py b/backend/scripts/cleanup_stale_db.py new file mode 100644 index 00000000000..ac9c6dabd7a --- /dev/null +++ b/backend/scripts/cleanup_stale_db.py @@ -0,0 +1,83 @@ +"""Run the DB retention policies once, manually. + +Useful for the first cleanup against a long-accumulated backlog (e.g. +the kombu_queue_message bloat the Celery beat-vs-dead-worker race +created), without waiting for the daily 08:00 UTC beat tick. + +Usage: + + cd backend + PYTHONPATH=$(pwd) python scripts/cleanup_stale_db.py # run all policies + PYTHONPATH=$(pwd) python scripts/cleanup_stale_db.py --dry-run # preview row counts + PYTHONPATH=$(pwd) python scripts/cleanup_stale_db.py --policy=kombu_message + PYTHONPATH=$(pwd) python scripts/cleanup_stale_db.py --policy=chat,index_attempt + +Honors the same RETENTION_DAYS_* env vars as the periodic task. To +override for one run without editing your shell profile: + + RETENTION_DAYS_KOMBU=3 python scripts/cleanup_stale_db.py --policy=kombu_message +""" +from __future__ import annotations + +import argparse +import sys + +from danswer.db.retention import RETENTION_POLICIES +from danswer.db.retention import run_retention_policies + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview the row counts that would be deleted; rolls back instead " + "of committing.", + ) + parser.add_argument( + "--policy", + type=str, + default=None, + help="Comma-separated subset of policy names to run. Available: " + + ", ".join(RETENTION_POLICIES.keys()), + ) + args = parser.parse_args() + + only: list[str] | None = None + if args.policy: + only = [p.strip() for p in args.policy.split(",") if p.strip()] + unknown = set(only) - set(RETENTION_POLICIES.keys()) + if unknown: + print( + f"Unknown policy/policies: {sorted(unknown)}. " + f"Available: {sorted(RETENTION_POLICIES.keys())}", + file=sys.stderr, + ) + return 2 + + label = "DRY RUN" if args.dry_run else "EXECUTE" + scope = ",".join(only) if only else "all" + print(f"[{label}] Running retention policies: {scope}") + + results = run_retention_policies(dry_run=args.dry_run, only=only) + + if not results: + print("(no policies ran — see logs)") + return 0 + + width = max(len(k) for k in results.keys()) + print() + print( + f"{'policy':<{width}} rows {'(would delete)' if args.dry_run else 'deleted'}" + ) + print(f"{'-' * width} -----") + for name, n in sorted(results.items(), key=lambda kv: -kv[1]): + print(f"{name:<{width}} {n}") + print() + if args.dry_run: + print("No changes committed. Re-run without --dry-run to actually delete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py index adbb5d22090..42eb6b1de68 100644 --- a/backend/scripts/dev_run_background_jobs.py +++ b/backend/scripts/dev_run_background_jobs.py @@ -18,21 +18,24 @@ def monitor_process(process_name: str, process: subprocess.Popen) -> None: def run_jobs(exclude_indexing: bool) -> None: + # NOTE: --autoscale only works with the prefork pool (it calls pool.grow/ + # pool.shrink, which the threads pool doesn't implement). Either drop + # autoscale and rely on a fixed concurrency (current choice), or switch + # to "--pool=prefork --autoscale=3,10". cmd_worker = [ "celery", "-A", - "ee.danswer.background.celery", + "ee.danswer.background.celery.celery_app", "worker", "--pool=threads", - "--autoscale=3,10", + "--concurrency=10", "--loglevel=INFO", - "--concurrency=1", ] cmd_beat = [ "celery", "-A", - "ee.danswer.background.celery", + "ee.danswer.background.celery.celery_app", "beat", "--loglevel=INFO", ] diff --git a/backend/scripts/dump_salesforce_account.py b/backend/scripts/dump_salesforce_account.py new file mode 100644 index 00000000000..c2bf51bb6dd --- /dev/null +++ b/backend/scripts/dump_salesforce_account.py @@ -0,0 +1,147 @@ +"""Dump every field of one or more Account records as pretty JSON. + +Authenticates with the OAuth Username-Password flow and runs +`SELECT FIELDS(ALL) FROM Account [WHERE …] LIMIT N`. Useful when curating +the connector's ACCOUNT_FIELDS list, inspecting custom-field values, or +verifying that a record matches the filter you intend to apply. + +`FIELDS(ALL)` has a hard SOQL limit of <= 200 rows per call (Salesforce +constraint, not ours), so this script is intentionally for spot-checking, +not bulk export. + +Usage: + + SF_CLIENT_ID=... SF_CLIENT_SECRET=... \\ + SF_USERNAME=... SF_PASSWORD='...' \\ + [SF_LOGIN_URL=https://test.salesforce.com] \\ + [SF_LIMIT=1] # default 1; max 200 + [SF_NAME_LIKE=acme] # optional Name LIKE '%...%' + [SF_ACCOUNT_ID=0011x...] # optional single Account by Id + [SF_KEEP_ATTRIBUTES=1] # keep Salesforce 'attributes' block (default: stripped) + python backend/scripts/dump_salesforce_account.py +""" +import json +import os +import sys +from typing import Any + +import requests + +DEFAULT_LOGIN_URL = "https://login.salesforce.com" +SF_API_VERSION = "v59.0" +FIELDS_ALL_MAX_LIMIT = 200 + + +def _get_access_token( + client_id: str, + client_secret: str, + username: str, + password: str, + login_url: str, +) -> tuple[str, str]: + resp = requests.post( + f"{login_url.rstrip('/')}/services/oauth2/token", + data={ + "grant_type": "password", + "client_id": client_id, + "client_secret": client_secret, + "username": username, + "password": password, + }, + timeout=30, + ) + if resp.status_code != 200: + try: + err = resp.json() + except ValueError: + err = resp.text + raise SystemExit( + f"Salesforce OAuth failed (HTTP {resp.status_code}) at {login_url}\n" + f" response: {err}" + ) + body = resp.json() + return body["access_token"], body["instance_url"] + + +def _build_query() -> str: + clauses: list[str] = [] + + name_like = os.environ.get("SF_NAME_LIKE", "").strip() + account_id = os.environ.get("SF_ACCOUNT_ID", "").strip() + + if account_id: + safe = account_id.replace("'", r"\'") + clauses.append(f"Id = '{safe}'") + if name_like: + safe = name_like.replace("'", r"\'") + clauses.append(f"Name LIKE '%{safe}%'") + + where = f" WHERE {' AND '.join(clauses)}" if clauses else "" + + try: + limit = int(os.environ.get("SF_LIMIT", "1")) + except ValueError: + limit = 1 + # FIELDS(ALL) requires an explicit LIMIT and Salesforce enforces a 200 cap. + limit = max(1, min(limit, FIELDS_ALL_MAX_LIMIT)) + + return f"SELECT FIELDS(ALL) FROM Account{where} LIMIT {limit}" + + +def _execute_soql( + instance_url: str, access_token: str, query: str +) -> list[dict[str, Any]]: + headers = {"Authorization": f"Bearer {access_token}"} + resp = requests.get( + f"{instance_url}/services/data/{SF_API_VERSION}/query", + headers=headers, + params={"q": query}, + timeout=120, + ) + data = resp.json() + if resp.status_code != 200 or isinstance(data, list): + msg = data if isinstance(data, list) else resp.text + raise SystemExit(f"Salesforce SOQL error: {msg}") + return data.get("records", []) + + +def _strip_attributes(value: Any) -> Any: + """Recursively remove Salesforce's noisy 'attributes' blocks.""" + if isinstance(value, dict): + return {k: _strip_attributes(v) for k, v in value.items() if k != "attributes"} + if isinstance(value, list): + return [_strip_attributes(v) for v in value] + return value + + +def main() -> int: + try: + client_id = os.environ["SF_CLIENT_ID"] + client_secret = os.environ["SF_CLIENT_SECRET"] + username = os.environ["SF_USERNAME"] + password = os.environ["SF_PASSWORD"] + except KeyError as e: + print(f"Missing required env var: {e}", file=sys.stderr) + print(__doc__, file=sys.stderr) + return 2 + + login_url = os.environ.get("SF_LOGIN_URL", DEFAULT_LOGIN_URL) + keep_attrs = os.environ.get("SF_KEEP_ATTRIBUTES", "").strip() == "1" + + access_token, instance_url = _get_access_token( + client_id, client_secret, username, password, login_url + ) + + query = _build_query() + print(f"# SOQL: {query}\n", file=sys.stderr) + + records = _execute_soql(instance_url, access_token, query) + print(f"# Fetched {len(records)} record(s)\n", file=sys.stderr) + + payload = records if keep_attrs else _strip_attributes(records) + print(json.dumps(payload, indent=2, sort_keys=True, default=str)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/list_salesforce_account_fields.py b/backend/scripts/list_salesforce_account_fields.py new file mode 100644 index 00000000000..418b4595760 --- /dev/null +++ b/backend/scripts/list_salesforce_account_fields.py @@ -0,0 +1,128 @@ +"""List Account fields available on the connected Salesforce org. + +Authenticates with the OAuth Username-Password flow, calls the Account +`describe` endpoint, and prints every field's API name plus its display +label. Useful when curating ACCOUNT_FIELDS in +`danswer/connectors/salesforce/connector.py` — run this against your org +and cross-reference the output with the labels you want to index. + +Usage: + + SF_CLIENT_ID=... SF_CLIENT_SECRET=... \\ + SF_USERNAME=... SF_PASSWORD=... \\ + [SF_LOGIN_URL=https://test.salesforce.com] \\ + [SF_FILTER=label_or_api_substring] \\ + [SF_ONLY_CUSTOM=1] \\ + python backend/scripts/list_salesforce_account_fields.py +""" +import os +import sys + +import requests + +DEFAULT_LOGIN_URL = "https://login.salesforce.com" +SF_API_VERSION = "v59.0" + + +def _get_access_token( + client_id: str, + client_secret: str, + username: str, + password: str, + login_url: str, +) -> tuple[str, str]: + resp = requests.post( + f"{login_url.rstrip('/')}/services/oauth2/token", + data={ + "grant_type": "password", + "client_id": client_id, + "client_secret": client_secret, + "username": username, + "password": password, + }, + timeout=30, + ) + if resp.status_code != 200: + try: + err = resp.json() + except ValueError: + err = resp.text + raise SystemExit( + f"Salesforce OAuth failed (HTTP {resp.status_code}) at {login_url}\n" + f" response: {err}\n" + "Common causes:\n" + " * invalid_client_id / invalid_client: wrong sf_client_id or sf_client_secret\n" + " * invalid_grant + 'authentication failure': wrong username/password,\n" + " or password needs the security token appended (password+token)\n" + " * invalid_grant + 'user hasn't approved this consumer': Connected App\n" + " 'Permitted Users' must be 'All users may self-authorize'\n" + " * sandbox accounts: set SF_LOGIN_URL=https://test.salesforce.com\n" + ) + body = resp.json() + return body["access_token"], body["instance_url"] + + +def main() -> int: + try: + client_id = os.environ["SF_CLIENT_ID"] + client_secret = os.environ["SF_CLIENT_SECRET"] + username = os.environ["SF_USERNAME"] + password = os.environ["SF_PASSWORD"] + except KeyError as e: + print(f"Missing required env var: {e}", file=sys.stderr) + print(__doc__, file=sys.stderr) + return 2 + + login_url = os.environ.get("SF_LOGIN_URL", DEFAULT_LOGIN_URL) + only_custom = os.environ.get("SF_ONLY_CUSTOM") == "1" + needle = os.environ.get("SF_FILTER", "").lower() + + access_token, instance_url = _get_access_token( + client_id, client_secret, username, password, login_url + ) + + desc = requests.get( + f"{instance_url}/services/data/{SF_API_VERSION}/sobjects/Account/describe", + headers={"Authorization": f"Bearer {access_token}"}, + timeout=60, + ) + desc.raise_for_status() + fields = desc.json()["fields"] + + rows: list[tuple[str, str, str, str]] = [] + for field in fields: + if only_custom and not field.get("custom"): + continue + api_name = field.get("name", "") + label = field.get("label", "") + ftype = field.get("type", "") + ref = ",".join(field.get("referenceTo") or []) + if needle and needle not in api_name.lower() and needle not in label.lower(): + continue + rows.append((api_name, label, ftype, ref)) + + rows.sort(key=lambda r: r[0].lower()) + + name_w = max((len(r[0]) for r in rows), default=10) + label_w = max((len(r[1]) for r in rows), default=10) + type_w = max((len(r[2]) for r in rows), default=8) + + header = f"{'API Name'.ljust(name_w)} {'Label'.ljust(label_w)} {'Type'.ljust(type_w)} References" + print(header) + print("-" * len(header)) + for api_name, label, ftype, ref in rows: + print( + f"{api_name.ljust(name_w)} {label.ljust(label_w)} {ftype.ljust(type_w)} {ref}" + ) + + print(f"\n{len(rows)} field(s){' (custom only)' if only_custom else ''}.") + if rows and any(r[2] == "reference" for r in rows): + print( + "Note: 'reference' fields are lookups — index them as " + "__r.Name (or .Id) rather than the raw API name." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/preview_salesforce_accounts.py b/backend/scripts/preview_salesforce_accounts.py new file mode 100644 index 00000000000..837d6685c08 --- /dev/null +++ b/backend/scripts/preview_salesforce_accounts.py @@ -0,0 +1,231 @@ +"""Preview what the SalesforceConnector would actually index. + +Authenticates with the OAuth Username-Password flow, runs the same SOQL +query the connector would build (using ACCOUNT_FIELDS), and prints each +record with the friendly labels — exactly as it would land in the indexed +text body. Use this to sanity-check field values before kicking off a real +indexing run. + +Usage: + + SF_CLIENT_ID=... SF_CLIENT_SECRET=... \\ + SF_USERNAME=... SF_PASSWORD='...' \\ + [SF_LOGIN_URL=https://test.salesforce.com] \\ + [SF_LIMIT=5] # default 5; set to 0 for unlimited + [SF_NAME_LIKE=acme] # filter Account.Name LIKE '%acme%' + [SF_ACCOUNT_ID=0011x...] # fetch one specific Account by Id + [SF_MAINTENANCE_FLAG_FILTER=Active] # exact-match Maintenance_Flag__c IN(...); + # comma-separate for multiple values + [SF_FORMAT=text|json] # default text + python backend/scripts/preview_salesforce_accounts.py +""" +import json +import os +import sys +from typing import Any + +import requests + +DEFAULT_LOGIN_URL = "https://login.salesforce.com" +SF_API_VERSION = "v59.0" + +# Account fields to fetch as (api_path, friendly_label) pairs. +# +# Kept in sync manually with +# `backend/danswer/connectors/salesforce/connector.py::ACCOUNT_FIELDS`. +# This script is intentionally standalone (no danswer imports) so it can +# be run from anywhere with just `requests` installed; if the connector's +# field list changes, mirror it here. +ACCOUNT_FIELDS: list[tuple[str, str]] = [ + ("Id", "Account Id"), + ("Name", "Account Name"), + ("Legal__c", "Legal Entity Name"), + ("Business_Name__c", "Business Name"), + ("RecordType.Name", "Account Record Type"), + ("Account_Type__c", "Account Type"), + ("Classification__c", "Classification"), + ("Segmentation__c", "Segmentation"), + ("Owner.Name", "Account Owner"), + ("Account_Owner_Email__c", "Account Owner Email"), + ("LastModifiedDate", "LastModifiedDate"), + ("LastModifiedBy.Name", "Last Modified By"), + ("Geo__c", "Geo"), + ("Area__c", "Area"), + ("Country__c", "Country"), + ("BillingCity", "Billing City"), + ("BillingCountry", "Billing Country"), + ("Ruby_Account__c", "Ruby Account"), + ("Maintenance_Flag__c", "Maintenance Flag"), + ("CSM__r.Name", "CSM"), + ("Support_Technical_Advisor__r.Name", "TAM"), + ("Vertical1__c", "Vertical"), + ("Sub_Vertical__c", "Sub-Vertical"), + ("CSD__r.Name", "CSD"), + ("NumberOfEmployees", "Employees"), + ("Annual_Contract_Value__c", "Annual Contract Value"), + ("Annual_Revenue_Local_Currency__c", "Annual Revenue (Local Currency)"), +] + + +def _resolve_path(record: dict[str, Any], path: str) -> Any: + """Walk a dotted SOQL path through the response JSON. Returns None if + any segment is missing or not a dict.""" + value: Any = record + for part in path.split("."): + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _to_friendly_dict(record: dict[str, Any]) -> dict[str, Any]: + """Re-key a SOQL record using ACCOUNT_FIELDS' friendly labels, flattening + dot-traversal paths so e.g. CSM__r.Name -> {"CSM": "Joe Smith"}.""" + out: dict[str, Any] = {} + for api_path, label in ACCOUNT_FIELDS: + value = _resolve_path(record, api_path) + if value is not None: + out[label] = value + return out + + +def _get_access_token( + client_id: str, + client_secret: str, + username: str, + password: str, + login_url: str, +) -> tuple[str, str]: + resp = requests.post( + f"{login_url.rstrip('/')}/services/oauth2/token", + data={ + "grant_type": "password", + "client_id": client_id, + "client_secret": client_secret, + "username": username, + "password": password, + }, + timeout=30, + ) + if resp.status_code != 200: + try: + err = resp.json() + except ValueError: + err = resp.text + raise SystemExit( + f"Salesforce OAuth failed (HTTP {resp.status_code}) at {login_url}\n" + f" response: {err}" + ) + body = resp.json() + return body["access_token"], body["instance_url"] + + +def _build_query() -> str: + select_clause = ", ".join(api_path for api_path, _ in ACCOUNT_FIELDS) + clauses: list[str] = [] + + maintenance_flag = os.environ.get("SF_MAINTENANCE_FLAG_FILTER", "").strip() + name_like = os.environ.get("SF_NAME_LIKE", "").strip() + account_id = os.environ.get("SF_ACCOUNT_ID", "").strip() + + if account_id: + clauses.append(f"Id = '{account_id}'") + if name_like: + # escape single quotes inside the LIKE pattern + safe = name_like.replace("'", r"\'") + clauses.append(f"Name LIKE '%{safe}%'") + if maintenance_flag: + values = [v.strip() for v in maintenance_flag.split(",") if v.strip()] + if values: + quoted = ", ".join( + f"'{v.replace(chr(39), chr(92) + chr(39))}'" for v in values + ) + clauses.append(f"Maintenance_Flag__c IN ({quoted})") + + where = f" WHERE {' AND '.join(clauses)}" if clauses else "" + query = f"SELECT {select_clause} FROM Account{where}" + + try: + limit = int(os.environ.get("SF_LIMIT", "5")) + except ValueError: + limit = 5 + if limit > 0: + query += f" LIMIT {limit}" + return query + + +def _execute_soql( + instance_url: str, access_token: str, query: str +) -> list[dict[str, Any]]: + headers = {"Authorization": f"Bearer {access_token}"} + records: list[dict[str, Any]] = [] + url: str | None = f"{instance_url}/services/data/{SF_API_VERSION}/query" + params: dict[str, str] | None = {"q": query} + + while url: + resp = requests.get(url, headers=headers, params=params, timeout=120) + data = resp.json() + if resp.status_code != 200 or isinstance(data, list): + msg = data if isinstance(data, list) else resp.text + raise SystemExit(f"Salesforce SOQL error: {msg}") + records.extend(data.get("records", [])) + next_path = data.get("nextRecordsUrl") + if data.get("done", True) or not next_path: + break + url = f"{instance_url}{next_path}" + params = None + return records + + +def _print_text(records: list[dict[str, Any]]) -> None: + if not records: + print("(no records)") + return + label_w = max(len(label) for _, label in ACCOUNT_FIELDS) + for i, record in enumerate(records, start=1): + friendly = _to_friendly_dict(record) + print(f"--- Record {i} ---") + for _, label in ACCOUNT_FIELDS: + value = friendly.get(label, "") + print(f" {label.ljust(label_w)} {value}") + print() + + +def _print_json(records: list[dict[str, Any]]) -> None: + out = [_to_friendly_dict(r) for r in records] + print(json.dumps(out, indent=2, default=str)) + + +def main() -> int: + try: + client_id = os.environ["SF_CLIENT_ID"] + client_secret = os.environ["SF_CLIENT_SECRET"] + username = os.environ["SF_USERNAME"] + password = os.environ["SF_PASSWORD"] + except KeyError as e: + print(f"Missing required env var: {e}", file=sys.stderr) + print(__doc__, file=sys.stderr) + return 2 + + login_url = os.environ.get("SF_LOGIN_URL", DEFAULT_LOGIN_URL) + output_format = os.environ.get("SF_FORMAT", "text").lower() + + access_token, instance_url = _get_access_token( + client_id, client_secret, username, password, login_url + ) + + query = _build_query() + print(f"# SOQL: {query}\n", file=sys.stderr) + + records = _execute_soql(instance_url, access_token, query) + print(f"# Fetched {len(records)} record(s)\n", file=sys.stderr) + + if output_format == "json": + _print_json(records) + else: + _print_text(records) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/seed_test_data.py b/backend/scripts/seed_test_data.py new file mode 100644 index 00000000000..e72b8384870 --- /dev/null +++ b/backend/scripts/seed_test_data.py @@ -0,0 +1,874 @@ +"""Auto-generate tagged test data for analytics + retention smoke tests. + +DESTRUCTIVE: writes (and on --clean, deletes) rows in your DB. Run only +against a dev / staging DB. The script prints the DB host and asks for +confirmation unless --yes is passed. + +Tagging: every row this script creates has a `__test_seed__` marker in a +queryable column (description / name / file_name / etc.) so --clean can +find and remove only its own data without touching real rows. + +Usage examples: + + cd backend + PYTHONPATH=$(pwd) python scripts/seed_test_data.py --help + PYTHONPATH=$(pwd) python scripts/seed_test_data.py --days=60 --chats-per-day=20 + PYTHONPATH=$(pwd) python scripts/seed_test_data.py --clean --yes +""" +from __future__ import annotations + +import argparse +import datetime +import random +import sys +import uuid +from dataclasses import dataclass + +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.auth.schemas import UserRole +from danswer.configs.constants import DocumentSource +from danswer.configs.constants import MessageType +from danswer.connectors.models import InputType +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.models import ChatMessage +from danswer.db.models import ChatMessage__SearchDoc +from danswer.db.models import ChatMessageFeedback +from danswer.db.models import ChatSession +from danswer.db.models import ChatSessionSharedStatus +from danswer.db.models import Connector +from danswer.db.models import ConnectorCredentialPair +from danswer.db.models import Credential +from danswer.db.models import Document +from danswer.db.models import DocumentByConnectorCredentialPair +from danswer.db.models import EmbeddingModel +from danswer.db.models import IndexAttempt +from danswer.db.models import IndexingStatus +from danswer.db.models import PermissionSyncJobType +from danswer.db.models import PermissionSyncStatus +from danswer.db.models import Persona +from danswer.db.models import SearchDoc +from danswer.db.models import SlackBotConfig +from danswer.db.models import SlackBotResponseType +from danswer.db.models import User + + +SEED_PREFIX = "__test_seed__" + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + + +@dataclass +class SeedConfig: + days: int + chats_per_day: int + slackbot_share: float + feedback_rate: float + like_share: float + resolved_share: float + needs_help_share: float + users_count: int + connectors_count: int + docs_per_connector: int + with_search_docs: bool + with_old_data: bool + seed: int + + +SOURCES = [ + DocumentSource.GITHUB, + DocumentSource.SLACK, + DocumentSource.JIRA, + DocumentSource.CONFLUENCE, + DocumentSource.NOTION, + DocumentSource.WEB, +] + + +# --------------------------------------------------------------------------- +# Safety: confirm before writing +# --------------------------------------------------------------------------- + + +def confirm_destructive(skip: bool) -> None: + engine = get_sqlalchemy_engine() + url = engine.url + safe_url = f"{url.drivername}://{url.username}@{url.host}:{url.port}/{url.database}" + if skip: + print(f"[--yes] Proceeding against {safe_url}") + return + print(f"This will WRITE / DELETE tagged ({SEED_PREFIX!r}) rows in:") + print(f" {safe_url}") + answer = input("Type 'yes' to continue: ") + if answer.strip().lower() != "yes": + print("Aborted.") + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Lookups for required-NOT-NULL FKs +# --------------------------------------------------------------------------- + + +def lookup_default_persona(db: Session) -> Persona: + p = db.execute(select(Persona).order_by(Persona.id).limit(1)).scalar_one_or_none() + if p is None: + sys.exit( + "No persona found. Bootstrap your DB first (run `alembic upgrade head` " + "and start the API server once so the default personas get loaded)." + ) + return p + + +def lookup_default_embedding_model(db: Session) -> EmbeddingModel: + m = db.execute( + select(EmbeddingModel).order_by(EmbeddingModel.id).limit(1) + ).scalar_one_or_none() + if m is None: + sys.exit( + "No embedding_model found. Bootstrap the DB first (start the API " + "server once to load the default embedding model)." + ) + return m + + +# --------------------------------------------------------------------------- +# Seeders — each returns the rows it created (or counts) for downstream wiring +# --------------------------------------------------------------------------- + + +def seed_users(db: Session, n: int) -> list[User]: + users: list[User] = [] + for i in range(n): + u = User( + id=uuid.uuid4(), + email=f"{SEED_PREFIX}user-{i}-{uuid.uuid4().hex[:6]}@example.test", + hashed_password="x" * 60, + is_active=True, + is_superuser=False, + is_verified=True, + role=UserRole.BASIC, + ) + db.add(u) + users.append(u) + db.commit() + return users + + +def seed_connectors( + db: Session, count: int, docs_per_connector: int +) -> list[tuple[Connector, Credential, ConnectorCredentialPair]]: + out: list[tuple[Connector, Credential, ConnectorCredentialPair]] = [] + for i in range(count): + source = SOURCES[i % len(SOURCES)] + connector = Connector( + name=f"{SEED_PREFIX}connector-{source.value}-{i}", + source=source, + input_type=InputType.POLL, + connector_specific_config={"_test_seed": True}, + refresh_freq=600, + disabled=False, + ) + credential = Credential(admin_public=True, credential_json={}) + db.add_all([connector, credential]) + db.flush() + ccp = ConnectorCredentialPair( + connector_id=connector.id, + credential_id=credential.id, + name=f"{SEED_PREFIX}ccp-{source.value}-{i}", + is_public=True, + total_docs_indexed=docs_per_connector, + ) + db.add(ccp) + out.append((connector, credential, ccp)) + db.commit() + return out + + +def seed_documents( + db: Session, + pairs: list[tuple[Connector, Credential, ConnectorCredentialPair]], + docs_per_connector: int, +) -> int: + """One row per (connector, doc-index). Also writes the join table so + `document_by_connector_credential_pair` is populated.""" + n = 0 + for conn, cred, _ccp in pairs: + for d in range(docs_per_connector): + doc_id = f"{SEED_PREFIX}doc-{conn.id}-{d}" + # Documents are deduped by id — one shared row across cc-pairs in + # real life. For the seeder we only have one ccp per source so + # simple insert is fine. + doc = Document( + id=doc_id, + boost=0, + hidden=False, + semantic_id=f"semantic-{doc_id}", + link=None, + from_ingestion_api=False, + ) + db.add(doc) + db.add( + DocumentByConnectorCredentialPair( + id=doc_id, + connector_id=conn.id, + credential_id=cred.id, + ) + ) + n += 1 + db.commit() # commit per cc-pair to keep transactions reasonable + return n + + +def seed_chat_thread( + db: Session, + user: User, + persona_id: int, + when: datetime.datetime, + danswerbot: bool, + feedback_kind: str | None, +) -> tuple[ChatSession, ChatMessage]: + """Create one chat session + one assistant message + optional feedback. + `feedback_kind` ∈ {None, 'like', 'dislike', 'resolved', 'needs_help'}. + """ + session = ChatSession( + user_id=user.id, + persona_id=persona_id, + description=f"{SEED_PREFIX}session-{uuid.uuid4().hex[:8]}", + deleted=False, + one_shot=False, + shared_status=ChatSessionSharedStatus.PRIVATE, + danswerbot_flow=danswerbot, + time_created=when, + time_updated=when, + ) + db.add(session) + db.flush() + + assistant_msg = ChatMessage( + chat_session_id=session.id, + message="Test assistant response", + message_type=MessageType.ASSISTANT, + token_count=10, + time_sent=when, + ) + db.add(assistant_msg) + db.flush() + + if feedback_kind == "like": + db.add( + ChatMessageFeedback( + chat_message_id=assistant_msg.id, + is_positive=True, + ) + ) + elif feedback_kind == "dislike": + db.add( + ChatMessageFeedback( + chat_message_id=assistant_msg.id, + is_positive=False, + ) + ) + elif feedback_kind == "resolved": + db.add( + ChatMessageFeedback( + chat_message_id=assistant_msg.id, + is_positive=None, + predefined_feedback="resolved", + ) + ) + elif feedback_kind == "needs_help": + db.add( + ChatMessageFeedback( + chat_message_id=assistant_msg.id, + is_positive=None, + required_followup=True, + ) + ) + + return session, assistant_msg + + +def _pick_feedback_kind(rng: random.Random, cfg: SeedConfig) -> str | None: + """Return one of {None, 'like', 'dislike', 'resolved', 'needs_help'} + according to the configured rates.""" + if rng.random() >= cfg.feedback_rate: + return None + # Within feedback, allocate by share. Likes + resolved + needs_help may + # not sum to 1 — the remainder maps to dislikes. + r = rng.random() + if r < cfg.like_share: + return "like" + elif r < cfg.like_share + cfg.resolved_share: + return "resolved" + elif r < cfg.like_share + cfg.resolved_share + cfg.needs_help_share: + return "needs_help" + return "dislike" + + +def seed_chat_data( + db: Session, + cfg: SeedConfig, + users: list[User], + persona_id: int, + rng: random.Random, + days_offset: int = 0, +) -> tuple[int, list[ChatMessage]]: + """Seed chats across `cfg.days` days starting `days_offset` days ago. + + Returns (total_threads_created, list_of_assistant_messages_for_linking). + The message list is used by --with-search-docs to attach search_doc + rows. + """ + now = datetime.datetime.now(tz=datetime.timezone.utc) + total = 0 + msgs: list[ChatMessage] = [] + for day_idx in range(cfg.days): + when_day = now - datetime.timedelta(days=days_offset + day_idx) + for c in range(cfg.chats_per_day): + # Spread within the day (random hour/minute) so time-bucket + # boundaries don't clip artificially. + when = when_day.replace( + hour=rng.randint(0, 23), + minute=rng.randint(0, 59), + second=rng.randint(0, 59), + microsecond=0, + ) + user = rng.choice(users) + danswerbot = rng.random() < cfg.slackbot_share + kind = _pick_feedback_kind(rng, cfg) + _session, msg = seed_chat_thread( + db, + user=user, + persona_id=persona_id, + when=when, + danswerbot=danswerbot, + feedback_kind=kind, + ) + msgs.append(msg) + total += 1 + db.commit() + return total, msgs + + +def seed_old_chat_for_retention( + db: Session, + users: list[User], + persona_id: int, + rng: random.Random, + count: int, +) -> int: + """Backdated chats (35–90 days old) so the retention sweep deletes them. + + Used to verify chat retention actually removes old data while leaving + fresh data alone. + """ + n = 0 + now = datetime.datetime.now(tz=datetime.timezone.utc) + for _ in range(count): + days_old = rng.randint(35, 90) + when = now - datetime.timedelta( + days=days_old, + hours=rng.randint(0, 23), + minutes=rng.randint(0, 59), + ) + user = rng.choice(users) + seed_chat_thread( + db, + user=user, + persona_id=persona_id, + when=when, + danswerbot=rng.random() < 0.5, + feedback_kind=rng.choice([None, "like", "dislike", "resolved"]), + ) + n += 1 + db.commit() + return n + + +def seed_search_docs_for_messages( + db: Session, msgs: list[ChatMessage], per_message: int = 3 +) -> int: + """Create search_doc rows and link them to chat messages via the + join table. Used to verify orphan-search_doc cleanup at the end of + chat retention.""" + created = 0 + for msg in msgs[:200]: # cap at 200 messages to keep totals sane + for _ in range(per_message): + sd = SearchDoc( + document_id=f"{SEED_PREFIX}doc-link-{uuid.uuid4().hex[:8]}", + chunk_ind=0, + semantic_id=f"{SEED_PREFIX}sd-{uuid.uuid4().hex[:8]}", + link=None, + blurb="seeded blurb", + boost=0, + source_type=DocumentSource.WEB.value, + hidden=False, + score=0.5, + match_highlights=[], + doc_metadata={}, + ) + db.add(sd) + db.flush() + db.add(ChatMessage__SearchDoc(chat_message_id=msg.id, search_doc_id=sd.id)) + created += 1 + db.commit() + return created + + +def seed_slack_bot_configs( + db: Session, count: int, persona_id: int, rng: random.Random +) -> int: + """Create slack_bot_config rows with seeded `channel_names` arrays. + + Used to verify the slack-channels analytics endpoint and the + `jsonb_array_elements_text` distinct-channel count. + """ + for i in range(count): + n_channels = rng.randint(2, 5) + channels = [f"{SEED_PREFIX}-channel-{i}-{j}" for j in range(n_channels)] + db.add( + SlackBotConfig( + persona_id=persona_id, + channel_config={"channel_names": channels}, + response_type=SlackBotResponseType.CITATIONS, + ) + ) + db.commit() + return count + + +def seed_index_attempts( + db: Session, + pairs: list[tuple[Connector, Credential, ConnectorCredentialPair]], + embedding_model: EmbeddingModel, + count_per_pair: int, + with_old: bool, + rng: random.Random, +) -> int: + """Mix of statuses + ages. With --with-old-data, half are backdated + 35-90 days so the optional index_attempt retention can hit them.""" + statuses = [IndexingStatus.SUCCESS, IndexingStatus.FAILED] + n = 0 + now = datetime.datetime.now(tz=datetime.timezone.utc) + for conn, cred, _ccp in pairs: + for i in range(count_per_pair): + backdate = with_old and rng.random() < 0.5 + days_old = rng.randint(35, 90) if backdate else rng.randint(0, 5) + when = now - datetime.timedelta(days=days_old, hours=rng.randint(0, 23)) + attempt = IndexAttempt( + connector_id=conn.id, + credential_id=cred.id, + embedding_model_id=embedding_model.id, + from_beginning=False, + status=rng.choice(statuses), + error_msg=None if rng.random() < 0.7 else "test failure", + new_docs_indexed=rng.randint(0, 100), + total_docs_indexed=rng.randint(0, 1000), + docs_removed_from_index=0, + time_created=when, + time_updated=when, + time_started=when, + ) + db.add(attempt) + n += 1 + db.commit() + return n + + +def seed_permission_syncs( + db: Session, + pairs: list[tuple[Connector, Credential, ConnectorCredentialPair]], + count: int, + with_old: bool, + rng: random.Random, +) -> int: + """Mix of fresh + old (35-90d) terminal-state permission_sync_run rows. + + Uses raw `text()` SQL because the `PermissionSyncRun.update_type` and + `.status` columns are declared as `Enum(...)` without `native_enum= + False` in the model — SA 2.x's bulk-insert path then emits + `::permissionsyncjobtype` casts in the SQL, but the underlying column + is plain `varchar`. Going through raw SQL with the enum's `.value` + string sidesteps the type-cast machinery cleanly. + """ + if not pairs: + return 0 + now = datetime.datetime.now(tz=datetime.timezone.utc) + statuses = [ + PermissionSyncStatus.SUCCESS.value, + PermissionSyncStatus.FAILED.value, + ] + job_types = [ + PermissionSyncJobType.USER_LEVEL.value, + PermissionSyncJobType.GROUP_LEVEL.value, + ] + for i in range(count): + backdate = with_old and rng.random() < 0.5 + days_old = rng.randint(35, 90) if backdate else rng.randint(0, 5) + when = now - datetime.timedelta(days=days_old) + conn, _cred, ccp = rng.choice(pairs) + db.execute( + text( + """ + INSERT INTO permission_sync_run + (source_type, update_type, cc_pair_id, status, + error_msg, updated_at) + VALUES + (:source_type, :update_type, :cc_pair_id, :status, + :error_msg, :updated_at) + """ + ), + { + "source_type": conn.source.value, + "update_type": rng.choice(job_types), + "cc_pair_id": ccp.id, + "status": rng.choice(statuses), + "error_msg": None, + "updated_at": when, + }, + ) + db.commit() + return count + + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + + +def clean_seeded_data(db: Session) -> dict[str, int]: + """Find and delete every row this script created, in FK-safe order. + + Tagged via SEED_PREFIX in description / name / file_name etc. Tables + touched: chat_message__search_doc, search_doc, chat_message, + chat_session, chat_message_feedback (cascades via ON DELETE SET NULL), + permission_sync_run, slack_bot_config, index_attempt, + document_by_connector_credential_pair, document, connector, + credential, connector_credential_pair, user. + + Per-table delete with explicit FK ordering — never `DROP`. + """ + counts: dict[str, int] = {} + + # 0. chat_message__search_doc → search_doc rows we created + counts["chat_message__search_doc"] = ( + db.execute( + text( + """ + DELETE FROM chat_message__search_doc + WHERE search_doc_id IN ( + SELECT id FROM search_doc WHERE semantic_id LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + counts["search_doc"] = ( + db.execute( + text("DELETE FROM search_doc WHERE semantic_id LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + db.commit() + + # 1. chat_message__search_doc → chat_message (where session is tagged) + db.execute( + text( + """ + DELETE FROM chat_message__search_doc + WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ) + + # 2. tool_call → chat_message + db.execute( + text( + """ + DELETE FROM tool_call + WHERE message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ) + + # 3. chat_message_feedback (FK has ON DELETE SET NULL but we want to + # drop the rows we created, identifiable via the tagged session). + db.execute( + text( + """ + DELETE FROM chat_feedback + WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ) + + # 4. chat_message + counts["chat_message"] = ( + db.execute( + text( + """ + DELETE FROM chat_message + WHERE chat_session_id IN ( + SELECT id FROM chat_session WHERE description LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + + # 5. chat_session + counts["chat_session"] = ( + db.execute( + text("DELETE FROM chat_session WHERE description LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + db.commit() + + # 6. permission_sync_run linked to seeded ccps + counts["permission_sync_run"] = ( + db.execute( + text( + """ + DELETE FROM permission_sync_run + WHERE cc_pair_id IN ( + SELECT id FROM connector_credential_pair WHERE name LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + + # 7. slack_bot_config — tagged via channel_config JSON + counts["slack_bot_config"] = ( + db.execute( + text( + """ + DELETE FROM slack_bot_config + WHERE EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(channel_config -> 'channel_names') c + WHERE c LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + + # 8. index_attempt linked to seeded connectors + counts["index_attempt"] = ( + db.execute( + text( + """ + DELETE FROM index_attempt + WHERE connector_id IN ( + SELECT id FROM connector WHERE name LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + + # 9. document_by_connector_credential_pair + document + counts["document_by_connector_credential_pair"] = ( + db.execute( + text("DELETE FROM document_by_connector_credential_pair WHERE id LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + counts["document"] = ( + db.execute( + text("DELETE FROM document WHERE id LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + db.commit() + + # 10. connector_credential_pair → connector + credential + counts["connector_credential_pair"] = ( + db.execute( + text("DELETE FROM connector_credential_pair WHERE name LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + counts["connector"] = ( + db.execute( + text("DELETE FROM connector WHERE name LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + # Credentials don't have a "name" column; we created admin_public=true + # rows linked to seeded cc_pairs. Find via the JSON marker we put in + # connector_specific_config? Cleaner: orphan credentials with no + # matching cc_pair and matching empty credential_json. Skip for now + # (low volume; manual cleanup possible). + db.commit() + + # 11. analytics_daily_rollup is owned by the rollup pipeline, not seeded + # data. Don't touch it here — the orchestrator script clears the + # checkpoint when needed. + + # 12. users (last — many FKs reference user.id) + counts["user"] = ( + db.execute( + text("""DELETE FROM "user" WHERE email LIKE :p"""), + {"p": f"{SEED_PREFIX}%"}, + ).rowcount + or 0 + ) + db.commit() + return counts + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--clean", action="store_true", help="Wipe seeded data and exit." + ) + parser.add_argument( + "--yes", action="store_true", help="Skip the confirmation prompt." + ) + parser.add_argument("--days", type=int, default=60) + parser.add_argument("--chats-per-day", type=int, default=20) + parser.add_argument("--slackbot-share", type=float, default=0.7) + parser.add_argument("--feedback-rate", type=float, default=0.6) + parser.add_argument("--like-share", type=float, default=0.5) + parser.add_argument("--resolved-share", type=float, default=0.2) + parser.add_argument("--needs-help-share", type=float, default=0.1) + parser.add_argument("--users", type=int, default=25, dest="users_count") + parser.add_argument("--connectors", type=int, default=4, dest="connectors_count") + parser.add_argument( + "--docs-per-connector", type=int, default=200, dest="docs_per_connector" + ) + parser.add_argument("--with-search-docs", action="store_true") + parser.add_argument("--with-old-data", action="store_true") + parser.add_argument("--seed", type=int, default=42, help="RNG seed.") + args = parser.parse_args() + + cfg = SeedConfig( + days=args.days, + chats_per_day=args.chats_per_day, + slackbot_share=args.slackbot_share, + feedback_rate=args.feedback_rate, + like_share=args.like_share, + resolved_share=args.resolved_share, + needs_help_share=args.needs_help_share, + users_count=args.users_count, + connectors_count=args.connectors_count, + docs_per_connector=args.docs_per_connector, + with_search_docs=args.with_search_docs, + with_old_data=args.with_old_data, + seed=args.seed, + ) + + confirm_destructive(skip=args.yes) + + engine = get_sqlalchemy_engine() + rng = random.Random(cfg.seed) + + if args.clean: + with Session(engine) as db: + counts = clean_seeded_data(db) + for table, n in counts.items(): + if n: + print(f" cleaned {n:>8d} rows from {table}") + print("Done.") + return 0 + + print(f"Seeding with config: {cfg}") + with Session(engine) as db: + persona = lookup_default_persona(db) + embedding_model = lookup_default_embedding_model(db) + + print( + f" using persona id={persona.id}, embedding_model id={embedding_model.id}" + ) + + users = seed_users(db, cfg.users_count) + print(f" ✓ {len(users)} users") + + pairs = seed_connectors(db, cfg.connectors_count, cfg.docs_per_connector) + print(f" ✓ {len(pairs)} connectors + cc_pairs") + + n_docs = seed_documents(db, pairs, cfg.docs_per_connector) + print(f" ✓ {n_docs} documents (+ join rows)") + + n_threads, msgs = seed_chat_data(db, cfg, users, persona.id, rng, days_offset=0) + print(f" ✓ {n_threads} chat threads (last {cfg.days} days)") + + if cfg.with_old_data: + n_old = seed_old_chat_for_retention( + db, users, persona.id, rng, count=cfg.chats_per_day * 2 + ) + print(f" ✓ {n_old} old chat threads (35-90 days old, for retention)") + + if cfg.with_search_docs: + n_sd = seed_search_docs_for_messages(db, msgs) + print(f" ✓ {n_sd} search_doc + chat_message__search_doc rows") + + n_sb = seed_slack_bot_configs(db, count=3, persona_id=persona.id, rng=rng) + print(f" ✓ {n_sb} slack_bot_config rows") + + n_ia = seed_index_attempts( + db, + pairs, + embedding_model, + count_per_pair=4, + with_old=cfg.with_old_data, + rng=rng, + ) + print(f" ✓ {n_ia} index_attempts") + + n_ps = seed_permission_syncs( + db, pairs, count=8, with_old=cfg.with_old_data, rng=rng + ) + print(f" ✓ {n_ps} permission_sync_run rows") + + print("Seeding complete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/test_analytics_e2e.py b/backend/scripts/test_analytics_e2e.py new file mode 100644 index 00000000000..b4b83da4c44 --- /dev/null +++ b/backend/scripts/test_analytics_e2e.py @@ -0,0 +1,511 @@ +"""End-to-end smoke test for the analytics + retention pipeline. + +DESTRUCTIVE: writes/deletes data in your DB. Run only against a dev DB. +The script confirms once at the start (skip via --yes). + +Phases: + 1. Clean any prior seed → re-seed 60 days of data + 35-90d "old" chats + 2. Backfill rollup, assert table populated + checkpoint advanced + 3. Read endpoints (via direct fn calls), assert counts match seeded data + 4. Dry-run retention, assert reported counts make sense + 5. Real retention, assert old chat data gone, fresh data alive + 6. Re-hit endpoints, assert rollup data SURVIVED retention deletes + 7. Re-run rollup, assert idempotency (checkpoint advances, recent days + re-processed, old days untouched) + 8. Final cleanup of seeded data + the rollup checkpoint + +Each phase prints PASS/FAIL with context. Exits 0 on full success, non- +zero on the first failed assertion. Re-run safe — idempotent. + +Usage: + cd backend + PYTHONPATH=$(pwd) python scripts/test_analytics_e2e.py [--yes] [--keep-data] +""" +from __future__ import annotations + +import argparse +import datetime +import subprocess +import sys +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.db.engine import get_sqlalchemy_engine + +# Reach into the seed module via direct import — keeps the orchestrator +# self-contained and avoids reflection / subprocess overhead. +THIS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(THIS_DIR)) +from seed_test_data import SEED_PREFIX # noqa: E402 +from seed_test_data import clean_seeded_data # noqa: E402 + + +# --------------------------------------------------------------------------- +# Tiny test harness +# --------------------------------------------------------------------------- + + +_FAILED = 0 + + +def section(name: str) -> None: + print(f"\n=== {name} ===") + + +def passed(msg: str) -> None: + print(f" ✓ {msg}") + + +def failed(msg: str, detail: str | None = None) -> None: + global _FAILED + _FAILED += 1 + print(f" ✗ {msg}") + if detail: + for line in detail.splitlines(): + print(f" {line}") + + +def assert_eq(actual, expected, label: str) -> None: + if actual == expected: + passed(f"{label} — {actual}") + else: + failed(label, f"expected={expected}, actual={actual}") + + +def assert_ge(actual, threshold: int, label: str) -> None: + n = int(actual or 0) + if n >= threshold: + passed(f"{label} — {n} (≥ {threshold})") + else: + failed(label, f"expected ≥ {threshold}, actual={n}") + + +def assert_true(cond: bool, label: str, detail: str = "") -> None: + if cond: + passed(label) + else: + failed(label, detail or None) + + +def run_script(args: list[str]) -> None: + """Invoke another script in the same env. Inherits PYTHONPATH.""" + print(f" $ python {' '.join(args)}") + result = subprocess.run( + [sys.executable] + args, + cwd=str(THIS_DIR.parent), # backend/ + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(" (stdout)") + for line in (result.stdout or "").splitlines()[-20:]: + print(f" {line}") + print(" (stderr)") + for line in (result.stderr or "").splitlines()[-20:]: + print(f" {line}") + failed(f"subprocess failed with rc={result.returncode}") + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Phase implementations +# --------------------------------------------------------------------------- + + +def phase_migrate() -> None: + """Idempotent — applies any pending migrations including + analytics_daily_rollup. Required for Phase 2 onward.""" + section("Phase 0 — alembic upgrade head (idempotent)") + print(f" $ {sys.executable} -m alembic upgrade head") + result = subprocess.run( + [sys.executable, "-m", "alembic", "upgrade", "head"], + cwd=str(THIS_DIR.parent), + capture_output=True, + text=True, + ) + if result.returncode != 0: + for line in (result.stdout or "").splitlines()[-20:]: + print(f" {line}") + for line in (result.stderr or "").splitlines()[-20:]: + print(f" {line}") + failed(f"alembic upgrade failed (rc={result.returncode})") + sys.exit(1) + # Print just the final "Running upgrade ..." lines for context. + tail = [ + ln + for ln in (result.stdout or "").splitlines() + if "Running upgrade" in ln or "INFO" in ln + ][-5:] + for line in tail: + print(f" {line}") + passed("migrations up to date") + + +def phase_seed() -> None: + section("Phase 1 — clean + seed 60 days + old data") + run_script(["scripts/seed_test_data.py", "--clean", "--yes"]) + run_script( + [ + "scripts/seed_test_data.py", + "--yes", + "--days=60", + "--chats-per-day=10", + "--slackbot-share=0.7", + "--feedback-rate=0.6", + "--like-share=0.5", + "--resolved-share=0.2", + "--needs-help-share=0.1", + "--users=15", + "--connectors=4", + "--docs-per-connector=50", + "--with-old-data", + "--with-search-docs", + "--seed=42", + ] + ) + + with Session(get_sqlalchemy_engine()) as db: + n_sessions = db.execute( + text("SELECT count(*) FROM chat_session WHERE description LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + n_messages = db.execute( + text( + """ + SELECT count(*) FROM chat_message + WHERE chat_session_id IN ( + SELECT id FROM chat_session WHERE description LIKE :p + ) + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + n_old = db.execute( + text( + """ + SELECT count(*) FROM chat_session + WHERE description LIKE :p + AND time_created < now() - interval '31 days' + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + n_search_docs = db.execute( + text("SELECT count(*) FROM search_doc WHERE semantic_id LIKE :p"), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + + assert_ge(n_sessions, 60 * 10, "seeded chat sessions (60d × 10/day)") + assert_ge(n_messages, 60 * 10, "seeded chat messages") + assert_ge(n_old, 1, "seeded OLD (>31d) chat sessions for retention") + assert_ge(n_search_docs, 1, "seeded search_doc rows") + + +def phase_clear_rollup_state() -> None: + """Wipe the rollup checkpoint + table BEFORE backfill so we exercise + the from-scratch path on every run.""" + section("Phase 2a — clear rollup state (checkpoint + table)") + with Session(get_sqlalchemy_engine()) as db: + db.execute( + text("DELETE FROM key_value_store WHERE key = 'analytics_rollup_state'") + ) + n_pre = db.execute(text("SELECT count(*) FROM analytics_daily_rollup")).scalar() + db.execute(text("TRUNCATE TABLE analytics_daily_rollup")) + db.commit() + passed(f"cleared {n_pre} prior rollup rows + checkpoint") + + +def phase_backfill_and_verify_rollup() -> None: + section("Phase 2b — backfill rollup + verify table + checkpoint") + run_script(["scripts/backfill_analytics_rollup.py"]) + + with Session(get_sqlalchemy_engine()) as db: + rows = db.execute( + text( + """ + SELECT count(*), + sum(total_queries), + sum(total_likes), + sum(total_dislikes), + sum(total_resolved), + sum(total_needs_help), + sum(slackbot_total) + FROM analytics_daily_rollup + """ + ) + ).one() + n_days, q, likes, dislikes, resolved, needs_help, slackbot = rows + checkpoint_payload = db.execute( + text( + "SELECT value FROM key_value_store " + "WHERE key = 'analytics_rollup_state'" + ) + ).scalar() + + assert_ge(int(n_days), 60, "analytics_daily_rollup row count after backfill") + assert_ge(int(q or 0), 60 * 10, "sum(total_queries) across rollup") + assert_ge(int(likes or 0), 1, "sum(total_likes) across rollup") + assert_ge(int(slackbot or 0), 1, "sum(slackbot_total) across rollup") + assert_true( + bool(checkpoint_payload) + and isinstance(checkpoint_payload, dict) + and "last_rolled_up_to" in checkpoint_payload, + "checkpoint row exists with last_rolled_up_to", + f"actual: {checkpoint_payload!r}", + ) + + +def phase_verify_endpoints() -> None: + section("Phase 3 — read-from-rollup helpers return seeded data") + from danswer.db.analytics_rollup import ( + fetch_danswerbot_analytics_from_rollup, + fetch_query_analytics_from_rollup, + fetch_user_analytics_from_rollup, + ) + + end = datetime.datetime.now(tz=datetime.timezone.utc) + start = end - datetime.timedelta(days=70) + with Session(get_sqlalchemy_engine()) as db: + q_rows = list(fetch_query_analytics_from_rollup(start, end, db)) + u_rows = list(fetch_user_analytics_from_rollup(start, end, db)) + b_rows = list(fetch_danswerbot_analytics_from_rollup(start, end, db)) + + assert_ge(len(q_rows), 60, "fetch_query_analytics_from_rollup row count") + assert_ge(len(u_rows), 60, "fetch_user_analytics_from_rollup row count") + assert_ge(len(b_rows), 60, "fetch_danswerbot_analytics_from_rollup row count") + + sum(int(r[0]) for r in q_rows) + total_likes = sum(int(r[1]) for r in q_rows) + total_resolved = sum(int(r[3]) for r in q_rows) + + # NPS strict denominator should be > 0 with 60 days × 10 chats × 60% feedback + promoters = total_likes + total_resolved + detractors = sum(int(r[2]) for r in q_rows) + sum(int(r[4]) for r in q_rows) + nps_denom = promoters + detractors + assert_ge(nps_denom, 1, "NPS-strict denominator (promoters + detractors)") + + nps = round((promoters - detractors) / nps_denom * 100) if nps_denom else None + if nps is not None: + passed( + f"NPS-strict computed = {nps:+d} (promoters={promoters}, detractors={detractors})" + ) + else: + failed("NPS-strict undefined (no feedback)") + + +def phase_dry_run_retention() -> None: + section("Phase 4 — retention dry-run reports something to clean") + # Don't assert specific numbers — just that the dry-run completes + # without error and the chat policy reports >0 rows. + run_script(["scripts/cleanup_stale_db.py", "--dry-run", "--policy=chat"]) + + +def phase_real_retention() -> None: + section("Phase 5 — run retention, verify old chats deleted") + + # Freeze a "fresh" boundary BEFORE the retention runs. Use a buffer of + # 28 days (retention uses 30) so chats right at the 30-day cutoff + # don't flicker between "fresh" and "deleted" due to clock drift + # during the retention sweep. The chats we seeded are spread across + # 60 days; the 28-day window is well-separated from the deletion edge. + fresh_boundary_iso = ( + datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=28) + ).isoformat() + + with Session(get_sqlalchemy_engine()) as db: + n_old_before = db.execute( + text( + """ + SELECT count(*) FROM chat_session + WHERE description LIKE :p + AND time_created < now() - interval '31 days' + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + n_fresh_before = db.execute( + text( + """ + SELECT count(*) FROM chat_session + WHERE description LIKE :p + AND time_created >= CAST(:b AS timestamptz) + """ + ), + {"p": f"{SEED_PREFIX}%", "b": fresh_boundary_iso}, + ).scalar() + + assert_ge(int(n_old_before or 0), 1, "old chats present before retention") + + run_script(["scripts/cleanup_stale_db.py", "--policy=chat"]) + + with Session(get_sqlalchemy_engine()) as db: + n_old_after = db.execute( + text( + """ + SELECT count(*) FROM chat_session + WHERE description LIKE :p + AND time_created < now() - interval '31 days' + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + n_fresh_after = db.execute( + text( + """ + SELECT count(*) FROM chat_session + WHERE description LIKE :p + AND time_created >= CAST(:b AS timestamptz) + """ + ), + {"p": f"{SEED_PREFIX}%", "b": fresh_boundary_iso}, + ).scalar() + n_orphan_search_doc = db.execute( + text( + """ + SELECT count(*) FROM search_doc sd + LEFT JOIN chat_message__search_doc cmsd + ON cmsd.search_doc_id = sd.id + WHERE sd.semantic_id LIKE :p + AND cmsd.chat_message_id IS NULL + """ + ), + {"p": f"{SEED_PREFIX}%"}, + ).scalar() + + assert_eq(int(n_old_after or 0), 0, "old chats deleted by retention") + assert_eq( + int(n_fresh_after or 0), + int(n_fresh_before or 0), + "fresh chats untouched by retention (frozen 28-day boundary)", + ) + # Orphan search_doc cleanup is a side-effect of chat retention; should + # be 0 immediately after retention. (Some seeded SDs were linked to + # OLD messages that just got deleted → the join goes empty → those SDs + # are orphans → retention's orphan sweep deletes them.) + assert_eq( + int(n_orphan_search_doc or 0), 0, "orphan search_doc cleanup after retention" + ) + + +def phase_rollup_survived_retention() -> None: + section("Phase 6 — rollup data SURVIVED retention deletes") + with Session(get_sqlalchemy_engine()) as db: + n_days = db.execute( + text("SELECT count(*) FROM analytics_daily_rollup") + ).scalar() + # The rollup should still cover 60 days even though chat data older + # than 30 days is now gone. This is the whole point of the rollup. + assert_ge(int(n_days or 0), 60, "rollup row count survives retention") + + +def phase_rollup_idempotent() -> None: + section("Phase 7 — re-run rollup is idempotent + advances checkpoint") + from danswer.db.analytics_rollup import run_rollup + + with Session(get_sqlalchemy_engine()) as db: + before = db.execute( + text( + "SELECT value FROM key_value_store " + "WHERE key = 'analytics_rollup_state'" + ) + ).scalar() + + n = run_rollup() + assert_ge(n, 1, "run_rollup processed ≥1 day") + + with Session(get_sqlalchemy_engine()) as db: + after = db.execute( + text( + "SELECT value FROM key_value_store " + "WHERE key = 'analytics_rollup_state'" + ) + ).scalar() + n_days = db.execute( + text("SELECT count(*) FROM analytics_daily_rollup") + ).scalar() + + assert_true( + bool(after) and isinstance(after, dict), + "checkpoint row still present after re-run", + ) + today_iso = datetime.datetime.now(tz=datetime.timezone.utc).date().isoformat() + if after and isinstance(after, dict): + assert_eq( + after.get("last_rolled_up_to"), today_iso, "checkpoint advanced to today" + ) + assert_ge(int(n_days or 0), 60, "rollup row count unchanged by re-run") + print(f" (info) checkpoint before={before}, after={after}") + + +def phase_final_cleanup(keep_data: bool) -> None: + section("Phase 8 — final cleanup") + if keep_data: + passed("--keep-data passed; leaving seeded rows + rollup behind") + return + with Session(get_sqlalchemy_engine()) as db: + counts = clean_seeded_data(db) + # Also drop the rollup rows + checkpoint for a clean slate. + db.execute(text("TRUNCATE TABLE analytics_daily_rollup")) + db.execute( + text("DELETE FROM key_value_store WHERE key = 'analytics_rollup_state'") + ) + db.commit() + nonzero = {k: v for k, v in counts.items() if v} + passed(f"removed seeded rows: {nonzero}") + passed("cleared analytics_daily_rollup + checkpoint") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", action="store_true", help="Skip the destructive-op confirmation." + ) + parser.add_argument( + "--keep-data", + action="store_true", + help="Don't clean seeded data + rollup at the end (debugging).", + ) + args = parser.parse_args() + + engine = get_sqlalchemy_engine() + safe_url = ( + f"{engine.url.drivername}://{engine.url.username}@" + f"{engine.url.host}:{engine.url.port}/{engine.url.database}" + ) + print(f"Target DB: {safe_url}") + if not args.yes: + ans = input( + "This will create + delete data in the target DB. Type 'yes' to continue: " + ) + if ans.strip().lower() != "yes": + print("Aborted.") + return 1 + + phase_migrate() + phase_seed() + phase_clear_rollup_state() + phase_backfill_and_verify_rollup() + phase_verify_endpoints() + phase_dry_run_retention() + phase_real_retention() + phase_rollup_survived_retention() + phase_rollup_idempotent() + phase_final_cleanup(keep_data=args.keep_data) + + print() + if _FAILED: + print(f"❌ {_FAILED} assertion(s) failed.") + return 1 + print("✅ All phases passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/test_celery_jobs_smoke.py b/backend/scripts/test_celery_jobs_smoke.py new file mode 100644 index 00000000000..8708233a311 --- /dev/null +++ b/backend/scripts/test_celery_jobs_smoke.py @@ -0,0 +1,407 @@ +"""Force-trigger the daily Celery tasks against fresh dummy data. + +Proves the broker → worker pipeline is alive end-to-end — the same path +beat uses for the 07:30 UTC rollup and 08:00 UTC retention fires. If +this script passes, you can trust the daily schedule will too (modulo +beat actually firing, which the schedule check covers separately). + +Steps: + 1. Seed minimal data: + - 5 chat_session + chat_message pairs backdated 35-90 days + (eligible for the chat retention sweep) + - 5 chat_session + chat_message pairs in the last 6 days + (visible in the rollup window) + - 1 connector + cc-pair (so the rollup has source variety) + 2. Snapshot before-state (chat counts, rollup row count) + 3. Fire `run_analytics_rollup_task.delay()` → wait for worker to run it + 4. Fire `run_retention_policies_task.delay()` → wait for worker to run it + 5. Snapshot after-state, print diff + 6. Cleanup the seeded rows (unless --keep-data) + +Requires the celery worker process to be running and connected to the +same DB this script targets. If `.get()` hangs, the worker isn't picking +up tasks (check `celery_worker.log` and supervisord status). + +DESTRUCTIVE: writes/deletes tagged data only (`__test_celery__` prefix). +Run only against a dev / staging DB. + +Usage: + cd backend + PYTHONPATH=$(pwd) python scripts/test_celery_jobs_smoke.py [--yes] [--keep-data] +""" +from __future__ import annotations + +import argparse +import datetime +import sys +import time +import uuid + +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.background.celery.celery_app import run_analytics_rollup_task +from danswer.background.celery.celery_app import run_retention_policies_task +from danswer.configs.constants import MessageType +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.models import ChatMessage +from danswer.db.models import ChatSession +from danswer.db.models import ChatSessionSharedStatus +from danswer.db.models import Persona + + +CELERY_PREFIX = "__test_celery__" + + +# --------------------------------------------------------------------------- +# Tiny harness +# --------------------------------------------------------------------------- + + +def section(name: str) -> None: + print(f"\n=== {name} ===") + + +def ok(msg: str) -> None: + print(f" ✓ {msg}") + + +def info(msg: str) -> None: + print(f" {msg}") + + +_FAILED = 0 + + +def fail(msg: str) -> None: + global _FAILED + _FAILED += 1 + print(f" ✗ {msg}") + + +# --------------------------------------------------------------------------- +# Seed +# --------------------------------------------------------------------------- + + +def seed_dummy_data() -> tuple[int, int]: + """Return (n_old_chats, n_fresh_chats) seeded.""" + engine = get_sqlalchemy_engine() + n_old, n_fresh = 0, 0 + with Session(engine) as db: + persona = db.execute( + select(Persona).order_by(Persona.id).limit(1) + ).scalar_one_or_none() + if persona is None: + sys.exit( + "No persona found. Bootstrap the DB first " + "(alembic upgrade head + start the API server once)." + ) + + now = datetime.datetime.now(tz=datetime.timezone.utc) + + # 5 OLD chats (35-90d) — chat retention with default 30d should delete. + for i in range(5): + when = now - datetime.timedelta(days=35 + i * 10) + cs = ChatSession( + user_id=None, # slackbot-style, no Danswer user + persona_id=persona.id, + description=f"{CELERY_PREFIX}old-{i}-{uuid.uuid4().hex[:6]}", + deleted=False, + one_shot=False, + shared_status=ChatSessionSharedStatus.PRIVATE, + danswerbot_flow=True, + time_created=when, + time_updated=when, + ) + db.add(cs) + db.flush() + db.add( + ChatMessage( + chat_session_id=cs.id, + message=f"old assistant reply {i}", + message_type=MessageType.ASSISTANT, + token_count=5, + time_sent=when, + ) + ) + n_old += 1 + + # 5 FRESH chats (last 6 days) — should land in the rollup window. + for i in range(5): + when = now - datetime.timedelta(days=i, hours=2) + cs = ChatSession( + user_id=None, + persona_id=persona.id, + description=f"{CELERY_PREFIX}fresh-{i}-{uuid.uuid4().hex[:6]}", + deleted=False, + one_shot=False, + shared_status=ChatSessionSharedStatus.PRIVATE, + danswerbot_flow=True, + time_created=when, + time_updated=when, + ) + db.add(cs) + db.flush() + db.add( + ChatMessage( + chat_session_id=cs.id, + message=f"fresh assistant reply {i}", + message_type=MessageType.ASSISTANT, + token_count=5, + time_sent=when, + ) + ) + n_fresh += 1 + db.commit() + return n_old, n_fresh + + +def cleanup_seeded() -> None: + engine = get_sqlalchemy_engine() + with Session(engine) as db: + # chat_message__search_doc → chat_message → chat_session, FK-safe + for sql in [ + """DELETE FROM chat_feedback WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p)""", + """DELETE FROM chat_message__search_doc WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p)""", + """DELETE FROM tool_call WHERE message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p)""", + """DELETE FROM chat_message WHERE chat_session_id IN ( + SELECT id FROM chat_session WHERE description LIKE :p)""", + "DELETE FROM chat_session WHERE description LIKE :p", + ]: + db.execute(text(sql), {"p": f"{CELERY_PREFIX}%"}) + db.commit() + + +# --------------------------------------------------------------------------- +# Snapshots +# --------------------------------------------------------------------------- + + +def snapshot() -> dict: + engine = get_sqlalchemy_engine() + with Session(engine) as db: + return { + "old_chats": int( + db.execute( + text( + "SELECT count(*) FROM chat_session " + "WHERE description LIKE :p " + "AND time_created < now() - interval '31 days'" + ), + {"p": f"{CELERY_PREFIX}%"}, + ).scalar() + or 0 + ), + "fresh_chats": int( + db.execute( + text( + "SELECT count(*) FROM chat_session " + "WHERE description LIKE :p " + "AND time_created >= now() - interval '7 days'" + ), + {"p": f"{CELERY_PREFIX}%"}, + ).scalar() + or 0 + ), + "rollup_max_rolled_up_at": db.execute( + text("SELECT max(rolled_up_at) FROM analytics_daily_rollup") + ).scalar(), + "rollup_row_count": int( + db.execute(text("SELECT count(*) FROM analytics_daily_rollup")).scalar() + or 0 + ), + "kombu_message_count": int( + db.execute(text("SELECT count(*) FROM kombu_message")).scalar() or 0 + ), + } + + +def print_snapshot(label: str, snap: dict) -> None: + print(f" [{label}]") + info(f"old chats (>31d, our prefix): {snap['old_chats']}") + info(f"fresh chats (≤7d, our prefix): {snap['fresh_chats']}") + info(f"analytics_daily_rollup rows: {snap['rollup_row_count']}") + info(f"max(rolled_up_at): {snap['rollup_max_rolled_up_at']}") + info(f"kombu_message rows total: {snap['kombu_message_count']}") + + +# --------------------------------------------------------------------------- +# Fire and wait +# --------------------------------------------------------------------------- + + +def fire_and_wait(task, label: str, timeout: int = 180) -> None: + """Submit via .delay() (broker path) and wait for the worker to ack + completion. Prints task id + final state.""" + section(f"Firing {label} via Celery .delay()") + started = time.monotonic() + result = task.delay() + info(f"task id: {result.id}") + info(f"submitted at t=0; waiting up to {timeout}s for worker…") + try: + ret = result.get(timeout=timeout) + except Exception as e: + elapsed = time.monotonic() - started + fail( + f"{label} did not complete within {timeout}s ({elapsed:.1f}s elapsed): {e}" + ) + info( + "If the script hung here, the celery worker likely isn't " + "running or isn't connected to this DB. Check " + "`celery_worker.log` and supervisord status." + ) + sys.exit(1) + elapsed = time.monotonic() - started + ok(f"{label} returned in {elapsed:.2f}s, state={result.state}, return={ret}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", action="store_true", help="Skip the destructive-op confirm." + ) + parser.add_argument( + "--keep-data", + action="store_true", + help="Don't clean up seeded rows at the end (debugging).", + ) + parser.add_argument( + "--rollup-timeout", + type=int, + default=120, + help="Seconds to wait for the rollup task (default 120).", + ) + parser.add_argument( + "--retention-timeout", + type=int, + default=300, + help="Seconds to wait for the retention task (default 300; the " + "first run against bloated kombu_message can take a while).", + ) + args = parser.parse_args() + + engine = get_sqlalchemy_engine() + safe_url = ( + f"{engine.url.drivername}://{engine.url.username}@" + f"{engine.url.host}:{engine.url.port}/{engine.url.database}" + ) + print(f"Target DB: {safe_url}") + print( + "Requires: celery worker process running and connected to this DB. " + "If `.get()` hangs, the worker isn't there." + ) + if not args.yes: + ans = input( + "This will create + delete tagged dummy data in the DB AND " + "trigger the real retention sweep (which deletes from " + "kombu_message etc.). Type 'yes' to continue: " + ) + if ans.strip().lower() != "yes": + print("Aborted.") + return 1 + + # Pre-clean any stragglers + cleanup_seeded() + + section("Phase 1 — seed dummy data") + n_old, n_fresh = seed_dummy_data() + ok(f"seeded {n_old} old chats (35-90d) and {n_fresh} fresh chats (last 6d)") + + section("Phase 2 — snapshot BEFORE") + before = snapshot() + print_snapshot("BEFORE", before) + + fire_and_wait( + run_analytics_rollup_task, + "run_analytics_rollup_task", + timeout=args.rollup_timeout, + ) + fire_and_wait( + run_retention_policies_task, + "run_retention_policies_task", + timeout=args.retention_timeout, + ) + + section("Phase 5 — snapshot AFTER") + after = snapshot() + print_snapshot("AFTER", after) + + section("Phase 6 — observed side effects") + + # Rollup: rolled_up_at should advance and row count should be >= before + if before["rollup_max_rolled_up_at"] is None: + if after["rollup_max_rolled_up_at"] is not None: + ok( + "rollup wrote rows for the first time " + f"(max rolled_up_at = {after['rollup_max_rolled_up_at']})" + ) + else: + fail("rollup task ran but rollup table is still empty") + else: + if ( + after["rollup_max_rolled_up_at"] + and after["rollup_max_rolled_up_at"] > before["rollup_max_rolled_up_at"] + ): + delta = after["rollup_max_rolled_up_at"] - before["rollup_max_rolled_up_at"] + ok(f"rollup advanced max(rolled_up_at) by {delta}") + else: + fail( + "rollup task ran but max(rolled_up_at) didn't advance " + f"(before={before['rollup_max_rolled_up_at']}, " + f"after={after['rollup_max_rolled_up_at']})" + ) + + # Retention: our 5 old chats should be gone, 5 fresh should remain + if after["old_chats"] == 0: + ok(f"retention deleted all {n_old} seeded old chats") + else: + fail( + f"retention left {after['old_chats']} of our old chats behind " + "(expected 0)" + ) + if after["fresh_chats"] == n_fresh: + ok(f"retention untouched all {n_fresh} fresh chats (correct)") + else: + fail( + f"retention deleted {n_fresh - after['fresh_chats']} fresh " + "chats it shouldn't have" + ) + + # kombu_message — informational only (count depends on broker activity) + delta = after["kombu_message_count"] - before["kombu_message_count"] + info( + f"kombu_message row count delta: {delta:+d} " + "(broker writes new rows constantly; just informational)" + ) + + if not args.keep_data: + cleanup_seeded() + print("\n cleaned up tagged dummy data") + else: + print("\n --keep-data passed; seeded rows left in place") + + if _FAILED: + print(f"\n❌ {_FAILED} assertion(s) failed.") + return 1 + print("\n✅ Both Celery tasks fired and applied side effects.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/test_deletion_lock_e2e.py b/backend/scripts/test_deletion_lock_e2e.py new file mode 100644 index 00000000000..932d3aebf6e --- /dev/null +++ b/backend/scripts/test_deletion_lock_e2e.py @@ -0,0 +1,580 @@ +"""End-to-end integration tests for the connector-deletion concurrency +fix: per-cc-pair advisory lock + API-side dedup guard. + +The unit tests in ``tests/unit/danswer/db/test_deletion_lock_keys.py`` +cover the pure key-derivation math. This script proves the full +behaviour against live Postgres: + + Phase A — lock primitive: two SQLAlchemy sessions; the second + cannot acquire while the first holds, then can after + release. (The cross-session semantics that + ``pg_try_advisory_lock`` provides; the unit test can't + cover this.) + Phase B — worker-side guard: a held lock causes + ``cleanup_connector_credential_pair_task``'s body to + early-return 0 without ever touching the cc-pair, in + milliseconds — versus the pre-fix behaviour where parallel + tasks would each spend 5 minutes retrying + ``SELECT … FOR UPDATE NOWAIT`` over the same documents. + Phase C — lock release on success: the task releases the lock after + a clean run, so the next caller can immediately acquire. + Phase D — lock release on exception: the task releases the lock even + when the body raises (rollback-before-unlock pattern). + Phase E — concurrent racing: N threads racing the same cc-pair + complete in bounded time and don't serialize on row-locks + for 5 minutes each. + Phase F — API dedup logic: the ``task_queue_jobs``-based check used + by ``/admin/deletion-attempt`` correctly identifies live + tasks (blocks duplicate dispatch), terminal tasks (allows + new dispatch), and timed-out tasks (allows new dispatch). + +DESTRUCTIVE: writes/deletes tagged data only (``__test_deletion__`` +prefix). Run only against a dev / staging DB. + +Pre-flight: refuses to run if a celery worker is processing tasks +for the test prefix; warns (but allows) if a worker is up at all. + +Usage:: + + cd backend + PYTHONPATH=$(pwd) python scripts/test_deletion_lock_e2e.py [--yes] +""" +from __future__ import annotations + +import argparse +import datetime +import sys +import threading +import time +import uuid +from collections.abc import Callable +from typing import Any + +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.background.celery.celery_app import ( + cleanup_connector_credential_pair_task, +) +from danswer.background.task_utils import name_cc_cleanup_task +from danswer.configs.constants import DocumentSource +from danswer.connectors.models import InputType +from danswer.db.connector_credential_pair import release_deletion_lock +from danswer.db.connector_credential_pair import try_acquire_deletion_lock +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.models import Connector +from danswer.db.models import ConnectorCredentialPair +from danswer.db.models import Credential +from danswer.db.models import TaskQueueState +from danswer.db.models import TaskStatus +from danswer.db.tasks import check_task_is_live_and_not_timed_out +from danswer.db.tasks import get_latest_task + + +DELETION_PREFIX = "__test_deletion__" + + +# ``cleanup_connector_credential_pair_task`` is wrapped twice (celery +# `@task` then `build_celery_task_wrapper`). Reaching ``.run.__wrapped__`` +# gives us the inner function — same body the celery worker invokes, +# but without the ``task_queue_jobs`` plumbing the wrapper requires +# (``mark_task_start`` insists on a pre-existing PENDING row from +# ``apply_async``, which we'd otherwise have to forge). +_inner_task: Callable[..., int] = cleanup_connector_credential_pair_task.run.__wrapped__ + + +# --------------------------------------------------------------------------- +# Tiny harness +# --------------------------------------------------------------------------- + + +_FAILED = 0 + + +def section(name: str) -> None: + print(f"\n=== {name} ===") + + +def passed(msg: str) -> None: + print(f" ok {msg}") + + +def failed(msg: str, detail: str | None = None) -> None: + global _FAILED + _FAILED += 1 + print(f" XX {msg}") + if detail: + for line in detail.splitlines(): + print(f" {line}") + + +def assert_eq(actual: Any, expected: Any, label: str) -> None: + if actual == expected: + passed(f"{label} — {actual}") + else: + failed(label, f"expected={expected}, actual={actual}") + + +def assert_true(cond: bool, label: str, detail: str = "") -> None: + if cond: + passed(label) + else: + failed(label, detail or None) + + +# --------------------------------------------------------------------------- +# Seeders / cleanup +# --------------------------------------------------------------------------- + + +def make_disabled_cc_pair( + db: Session, +) -> tuple[Connector, Credential, ConnectorCredentialPair]: + """Seed a connector + credential + cc-pair tagged with the test + prefix. Connector is ``disabled=True`` so the deletion endpoint's + `check_deletion_attempt_is_allowed` lets the request through.""" + connector = Connector( + name=f"{DELETION_PREFIX}{uuid.uuid4().hex[:8]}", + # Use a source that real environments are unlikely to use, so + # we never collide with production rows. + source=DocumentSource.ZULIP, + input_type=InputType.POLL, + connector_specific_config={"_test_deletion": True}, + refresh_freq=600, + disabled=True, + ) + credential = Credential(admin_public=True, credential_json={}) + db.add_all([connector, credential]) + db.flush() + ccp = ConnectorCredentialPair( + connector_id=connector.id, + credential_id=credential.id, + name=f"{DELETION_PREFIX}ccp-{uuid.uuid4().hex[:8]}", + is_public=True, + total_docs_indexed=0, + ) + db.add(ccp) + db.flush() + return connector, credential, ccp + + +def cleanup_test_data() -> None: + """Drop everything tagged with DELETION_PREFIX, FK-safe order.""" + engine = get_sqlalchemy_engine() + with Session(engine) as db: + db.execute( + text( + """ + DELETE FROM index_attempt + WHERE connector_id IN ( + SELECT id FROM connector WHERE name LIKE :p + ) + """ + ), + {"p": f"{DELETION_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM connector_credential_pair WHERE name LIKE :p"), + {"p": f"{DELETION_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM connector WHERE name LIKE :p"), + {"p": f"{DELETION_PREFIX}%"}, + ) + # task_queue_jobs rows we forged in Phase F: + db.execute( + text("DELETE FROM task_queue_jobs WHERE task_name LIKE :p"), + {"p": f"{DELETION_PREFIX}%"}, + ) + db.commit() + + +# --------------------------------------------------------------------------- +# Phase A — lock primitive (the cross-session check that the unit test +# can't cover) +# --------------------------------------------------------------------------- + + +_SYNTHETIC = (9_999_991, 9_999_992) + + +def phase_a_lock_primitive() -> None: + section("Phase A — advisory lock primitive across 2 sessions") + engine = get_sqlalchemy_engine() + sess_a = Session(engine) + sess_b = Session(engine) + try: + a1 = try_acquire_deletion_lock(sess_a, *_SYNTHETIC) + assert_eq(a1, True, "A acquires lock") + + b1 = try_acquire_deletion_lock(sess_b, *_SYNTHETIC) + assert_eq(b1, False, "B blocked while A holds") + + release_deletion_lock(sess_a, *_SYNTHETIC) + sess_a.commit() + passed("A released") + + b2 = try_acquire_deletion_lock(sess_b, *_SYNTHETIC) + assert_eq(b2, True, "B acquires after A released") + + release_deletion_lock(sess_b, *_SYNTHETIC) + sess_b.commit() + finally: + sess_a.close() + sess_b.close() + + +# --------------------------------------------------------------------------- +# Phase B — worker-side guard: held lock makes the task body return 0 fast +# --------------------------------------------------------------------------- + + +def phase_b_held_lock_blocks_task() -> None: + section("Phase B — held lock causes task body to return 0 quickly") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + _, _, ccp = make_disabled_cc_pair(db) + db.commit() + connector_id = ccp.connector_id + credential_id = ccp.credential_id + + # Hold the deletion advisory lock in a separate session, simulating + # another worker that's already running a deletion for this cc-pair. + holder = Session(engine) + try: + held = try_acquire_deletion_lock(holder, connector_id, credential_id) + assert_true(held, "external session acquires lock") + + t0 = time.monotonic() + result = _inner_task(connector_id=connector_id, credential_id=credential_id) + elapsed = time.monotonic() - t0 + + assert_eq(result, 0, "task body returns 0 (skipped)") + assert_true( + elapsed < 5.0, + "task returned in well under the pre-fix 5-minute timeout", + f"elapsed: {elapsed:.3f}s", + ) + + # Sanity: the cc-pair was NOT deleted (we held the lock). + with Session(engine) as db: + still_there = db.execute( + select(ConnectorCredentialPair).where( + ConnectorCredentialPair.connector_id == connector_id, + ConnectorCredentialPair.credential_id == credential_id, + ) + ).scalar_one_or_none() + assert_true( + still_there is not None, + "cc-pair still present after lock-blocked task", + ) + finally: + release_deletion_lock(holder, connector_id, credential_id) + holder.commit() + holder.close() + + +# --------------------------------------------------------------------------- +# Phase C — lock release after a clean run (cc-pair with 0 docs) +# --------------------------------------------------------------------------- + + +def phase_c_lock_released_on_success() -> None: + section("Phase C — lock released after task succeeds (clean delete)") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + _, _, ccp = make_disabled_cc_pair(db) + db.commit() + connector_id = ccp.connector_id + credential_id = ccp.credential_id + + # Run the task end-to-end. With 0 documents and 0 doc-sets, the + # delete loop is a single iteration and `cleanup_synced_entities` + # exits on the first pass. No Vespa traffic. Just exercises the + # lock-acquire → body → finally-release flow. + result = _inner_task(connector_id=connector_id, credential_id=credential_id) + assert_true( + isinstance(result, int) and result >= 0, + f"task ran cleanly, deleted {result} docs", + ) + + # cc-pair row should be gone (the body deletes it). + with Session(engine) as db: + still_there = db.execute( + select(ConnectorCredentialPair).where( + ConnectorCredentialPair.connector_id == connector_id, + ConnectorCredentialPair.credential_id == credential_id, + ) + ).scalar_one_or_none() + assert_true(still_there is None, "cc-pair row deleted after success") + + # Lock must now be free — a fresh session can acquire immediately. + sess = Session(engine) + try: + got = try_acquire_deletion_lock(sess, connector_id, credential_id) + assert_true(got, "lock free after success path") + finally: + release_deletion_lock(sess, connector_id, credential_id) + sess.commit() + sess.close() + + +# --------------------------------------------------------------------------- +# Phase D — lock release after exception in task body +# --------------------------------------------------------------------------- + + +def phase_d_lock_released_on_exception() -> None: + section("Phase D — lock released after task body raises") + + # Use a cc-pair id that does not exist. The task body raises + # ValueError("...does not exist") *after* acquiring the lock; the + # finally block must still release it. + connector_id = 9_999_001 + credential_id = 9_999_001 + + raised = False + try: + _inner_task(connector_id=connector_id, credential_id=credential_id) + except ValueError as e: + raised = "does not exist" in str(e) + assert_true(raised, "task raised ValueError for missing cc-pair") + + # Lock must be free despite the exception. + engine = get_sqlalchemy_engine() + sess = Session(engine) + try: + got = try_acquire_deletion_lock(sess, connector_id, credential_id) + assert_true(got, "lock free after exception path (rollback-before-unlock)") + finally: + release_deletion_lock(sess, connector_id, credential_id) + sess.commit() + sess.close() + + +# --------------------------------------------------------------------------- +# Phase E — concurrent threads racing on the same cc-pair +# --------------------------------------------------------------------------- + + +def phase_e_concurrent_threads_race() -> None: + section("Phase E — N threads race; lock serializes; bounded total time") + + n_threads = 6 + # Use a non-existent cc-pair so the inner body returns fast on the + # winner (raises ValueError) and we don't need to seed docs. + connector_id = 9_999_002 + credential_id = 9_999_002 + + results: list[Any] = [None] * n_threads + exceptions: list[Exception | None] = [None] * n_threads + + def runner(i: int) -> None: + try: + results[i] = _inner_task( + connector_id=connector_id, credential_id=credential_id + ) + except Exception as e: + exceptions[i] = e + + threads = [threading.Thread(target=runner, args=(i,)) for i in range(n_threads)] + t0 = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + elapsed = time.monotonic() - t0 + + # Pre-fix worst case: each thread hits row-lock retry for 5 min. + # We seeded a non-existent cc-pair so we don't even reach the row- + # lock code, but the broader claim — "concurrent invocations don't + # take minutes" — must hold. 30s is generous; expected ~1s. + assert_true( + elapsed < 30.0, + f"all {n_threads} threads completed in {elapsed:.2f}s (< 30s)", + ) + + # Each thread either got the lock and raised ValueError ("does not + # exist"), or didn't get the lock and returned 0. No other outcome + # is acceptable. + bad = [] + n_got_lock = 0 + n_skipped = 0 + for i in range(n_threads): + if exceptions[i] is not None: + if isinstance(exceptions[i], ValueError) and "does not exist" in str( + exceptions[i] + ): + n_got_lock += 1 + else: + bad.append(f"thread {i} raised unexpected: {exceptions[i]!r}") + elif results[i] == 0: + n_skipped += 1 + else: + bad.append(f"thread {i} returned unexpected: {results[i]!r}") + assert_true(not bad, "all threads had expected outcomes", "\n".join(bad)) + assert_true( + n_got_lock + n_skipped == n_threads, + f"{n_got_lock} acquired (raised ValueError), {n_skipped} skipped (returned 0)", + ) + + +# --------------------------------------------------------------------------- +# Phase F — API dedup logic via task_queue_jobs state +# --------------------------------------------------------------------------- + + +def _insert_task_row( + db: Session, + *, + name: str, + status: TaskStatus, + register_offset_seconds: int = 0, + start_offset_seconds: int | None = 0, +) -> None: + """Insert a synthetic task_queue_jobs row with controlled timing. + + ``register_offset_seconds`` and ``start_offset_seconds`` are + subtracted from ``now()``; positive numbers push the timestamps + *into the past* so we can simulate aged / timed-out tasks. + """ + now = datetime.datetime.now(tz=datetime.timezone.utc) + register_time = now - datetime.timedelta(seconds=register_offset_seconds) + start_time = ( + None + if start_offset_seconds is None + else now - datetime.timedelta(seconds=start_offset_seconds) + ) + row = TaskQueueState( + task_id=str(uuid.uuid4()), + task_name=name, + status=status, + register_time=register_time, + start_time=start_time, + ) + db.add(row) + db.commit() + + +def phase_f_api_dedup_logic() -> None: + section("Phase F — API dedup: task_queue_jobs state controls 409") + + # Use a synthetic cc-pair whose task name is also tag-prefixed so + # cleanup picks it up. + connector_id = 9_999_003 + credential_id = 9_999_003 + base_name = name_cc_cleanup_task( + connector_id=connector_id, credential_id=credential_id + ) + # Tag the task name so cleanup_test_data() can drop it. + name = f"{DELETION_PREFIX}{base_name}" + + engine = get_sqlalchemy_engine() + + # Case 1: no row → no live task → API would NOT 409. + with Session(engine) as db: + latest = get_latest_task(task_name=name, db_session=db) + assert_true(latest is None, "no row → get_latest_task returns None (allow)") + + # Case 2: STARTED row with recent timestamps → live → API would 409. + with Session(engine) as db: + _insert_task_row( + db, name=name, status=TaskStatus.STARTED, register_offset_seconds=5 + ) + latest = get_latest_task(task_name=name, db_session=db) + assert_true(latest is not None, "STARTED row inserted") + if latest is not None: + live = check_task_is_live_and_not_timed_out(latest, db) + assert_eq(live, True, "STARTED + recent → live (would 409)") + + # Case 3: SUCCESS row → terminal → API would NOT 409. + with Session(engine) as db: + _insert_task_row( + db, name=name, status=TaskStatus.SUCCESS, register_offset_seconds=1 + ) + latest = get_latest_task(task_name=name, db_session=db) + if latest is not None: + live = check_task_is_live_and_not_timed_out(latest, db) + assert_eq(live, False, "latest is SUCCESS → not live (allow)") + + # Case 4: STARTED row with ancient timestamps → timed out → allow. + # JOB_TIMEOUT defaults to a few hours; a register_time 999_999s in + # the past is unambiguously stale. + with Session(engine) as db: + _insert_task_row( + db, + name=name, + status=TaskStatus.STARTED, + register_offset_seconds=999_999, + start_offset_seconds=999_999, + ) + latest = get_latest_task(task_name=name, db_session=db) + if latest is not None: + live = check_task_is_live_and_not_timed_out(latest, db) + assert_eq(live, False, "STARTED but ancient → timed out (allow)") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", action="store_true", help="Skip the destructive-op confirmation." + ) + parser.add_argument( + "--keep-data", + action="store_true", + help="Don't clean tagged data at the end (debugging).", + ) + args = parser.parse_args() + + engine = get_sqlalchemy_engine() + safe_url = ( + f"{engine.url.drivername}://{engine.url.username}@" + f"{engine.url.host}:{engine.url.port}/{engine.url.database}" + ) + print(f"Target DB: {safe_url}") + if not args.yes: + ans = input( + "This will create + delete tagged rows under " + f"prefix '{DELETION_PREFIX}'. Continue? [y/N] " + ) + if ans.strip().lower() not in ("y", "yes"): + print("Aborted.") + return 1 + + cleanup_test_data() + try: + phase_a_lock_primitive() + cleanup_test_data() + phase_b_held_lock_blocks_task() + cleanup_test_data() + phase_c_lock_released_on_success() + cleanup_test_data() + phase_d_lock_released_on_exception() + cleanup_test_data() + phase_e_concurrent_threads_race() + cleanup_test_data() + phase_f_api_dedup_logic() + finally: + if not args.keep_data: + cleanup_test_data() + print("\n ok cleaned up tagged rows") + else: + print(f"\n -- kept tagged rows (--keep-data); prefix={DELETION_PREFIX}") + + if _FAILED: + print(f"\nFAIL: {_FAILED} assertion(s) failed") + return 1 + print("\nALL PHASES PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/test_features_e2e.py b/backend/scripts/test_features_e2e.py new file mode 100644 index 00000000000..6cb8e23e5df --- /dev/null +++ b/backend/scripts/test_features_e2e.py @@ -0,0 +1,734 @@ +"""End-to-end smoke tests for non-analytics features built this session. + +Companion to `test_analytics_e2e.py`. That script covers the analytics + +chat retention pipelines; this one covers everything else: + + Phase 1 Per-attempt indexing priority — verifies + `get_not_started_index_attempts` ordering (priority DESC, + time_created ASC) and `update_index_attempt_priority` + semantics (only NOT_STARTED is mutable). + Phase 2 index_attempt retention — opt-in via + RETENTION_DAYS_INDEX_ATTEMPT, with keep-last-N. Critically + also verifies the P0 status-casing fix: the SQL filters use + lowercase 'success' / 'failed' to match how + `Enum(IndexingStatus, native_enum=False)` actually stores + them. An uppercase regression here silently no-ops the policy. + Phase 3 permission_sync_run retention — only TERMINAL rows + ('success' / 'failed') are deleted; 'in_progress' rows are + preserved regardless of age, matching the safety contract in + db/retention.py. + Phase 4 Resolved-button feedback DB write — synthetic call to + `create_chat_message_feedback(predefined_feedback='resolved', + ...)` against a seeded chat message. Verifies the row shape + the Slackbot resolved-button handler relies on. + +Each phase prints PASS/FAIL with context. Exits 0 on full success, +non-zero on the first failed assertion. Re-run safe — idempotent. + +DESTRUCTIVE: writes/deletes tagged data only (`__test_features__` prefix). +Run only against a dev / staging DB. + +Usage: + cd backend + PYTHONPATH=$(pwd) python scripts/test_features_e2e.py [--yes] [--keep-data] +""" +from __future__ import annotations + +import argparse +import datetime +import os +import subprocess +import sys +import uuid +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +from danswer.auth.schemas import UserRole +from danswer.configs.constants import DocumentSource +from danswer.configs.constants import MessageType +from danswer.connectors.models import InputType +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.feedback import create_chat_message_feedback +from danswer.db.index_attempt import get_not_started_index_attempts +from danswer.db.index_attempt import mark_attempt_in_progress__no_commit +from danswer.db.index_attempt import update_index_attempt_priority +from danswer.db.models import ChatMessage +from danswer.db.models import ChatMessageFeedback +from danswer.db.models import ChatSession +from danswer.db.models import ChatSessionSharedStatus +from danswer.db.models import Connector +from danswer.db.models import ConnectorCredentialPair +from danswer.db.models import Credential +from danswer.db.models import EmbeddingModel +from danswer.db.models import IndexAttempt +from danswer.db.models import IndexingStatus +from danswer.db.models import Persona +from danswer.db.models import User + + +FEATURE_PREFIX = "__test_features__" + + +# --------------------------------------------------------------------------- +# Tiny harness (mirrors test_analytics_e2e.py for consistency) +# --------------------------------------------------------------------------- + + +_FAILED = 0 + + +def section(name: str) -> None: + print(f"\n=== {name} ===") + + +def passed(msg: str) -> None: + print(f" ✓ {msg}") + + +def failed(msg: str, detail: str | None = None) -> None: + global _FAILED + _FAILED += 1 + print(f" ✗ {msg}") + if detail: + for line in detail.splitlines(): + print(f" {line}") + + +def assert_eq(actual, expected, label: str) -> None: + if actual == expected: + passed(f"{label} — {actual}") + else: + failed(label, f"expected={expected}, actual={actual}") + + +def assert_list_eq(actual: list, expected: list, label: str) -> None: + if list(actual) == list(expected): + passed(f"{label} — {actual}") + else: + failed(label, f"expected={expected}, actual={list(actual)}") + + +def assert_true(cond: bool, label: str, detail: str = "") -> None: + if cond: + passed(label) + else: + failed(label, detail or None) + + +# --------------------------------------------------------------------------- +# Shared fixture helpers (all tagged with FEATURE_PREFIX) +# --------------------------------------------------------------------------- + + +def lookup_persona(db: Session) -> Persona: + p = db.execute(select(Persona).order_by(Persona.id).limit(1)).scalar_one_or_none() + if p is None: + sys.exit( + "No persona found. Bootstrap your DB first (alembic upgrade head + " + "start the API server once)." + ) + return p + + +def lookup_embedding_model(db: Session) -> EmbeddingModel: + m = db.execute( + select(EmbeddingModel).order_by(EmbeddingModel.id).limit(1) + ).scalar_one_or_none() + if m is None: + sys.exit("No embedding_model found.") + return m + + +def make_user(db: Session) -> User: + u = User( + id=uuid.uuid4(), + email=f"{FEATURE_PREFIX}user-{uuid.uuid4().hex[:6]}@example.test", + hashed_password="x" * 60, + is_active=True, + is_superuser=False, + is_verified=True, + role=UserRole.BASIC, + ) + db.add(u) + db.flush() + return u + + +def make_connector_and_pair( + db: Session, source: DocumentSource = DocumentSource.GITHUB +) -> tuple[Connector, Credential, ConnectorCredentialPair]: + connector = Connector( + name=f"{FEATURE_PREFIX}connector-{source.value}-{uuid.uuid4().hex[:6]}", + source=source, + input_type=InputType.POLL, + connector_specific_config={"_test_features": True}, + refresh_freq=600, + disabled=False, + ) + credential = Credential(admin_public=True, credential_json={}) + db.add_all([connector, credential]) + db.flush() + ccp = ConnectorCredentialPair( + connector_id=connector.id, + credential_id=credential.id, + name=f"{FEATURE_PREFIX}ccp-{uuid.uuid4().hex[:6]}", + is_public=True, + total_docs_indexed=0, + ) + db.add(ccp) + db.flush() + return connector, credential, ccp + + +def make_attempt( + db: Session, + *, + connector_id: int, + credential_id: int, + embedding_model_id: int, + status: IndexingStatus, + priority: int = 0, + days_old: int = 0, +) -> IndexAttempt: + when = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta( + days=days_old, seconds=uuid.uuid4().int % 100 + ) + a = IndexAttempt( + connector_id=connector_id, + credential_id=credential_id, + embedding_model_id=embedding_model_id, + from_beginning=False, + status=status, + new_docs_indexed=0, + total_docs_indexed=0, + docs_removed_from_index=0, + indexing_priority=priority, + time_created=when, + time_updated=when, + time_started=when if status != IndexingStatus.NOT_STARTED else None, + ) + db.add(a) + db.flush() + return a + + +# --------------------------------------------------------------------------- +# Phase 1 — Per-attempt indexing priority +# --------------------------------------------------------------------------- + + +def phase_priority_ordering() -> None: + section("Phase 1 — per-attempt priority ordering + update semantics") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + lookup_persona(db) + em = lookup_embedding_model(db) + make_user(db) + _conn, _cred, ccp = make_connector_and_pair(db) + + # Insert 5 NOT_STARTED with priorities, in this creation order: + # a (pri=0, oldest) + # b (pri=5) + # c (pri=10) + # d (pri=0) + # e (pri=3, newest) + # Expected `get_not_started_index_attempts` order: + # c (10) > b (5) > e (3) > a (0, oldest of the 0s) > d (0) + priorities = [0, 5, 10, 0, 3] + attempts: list[IndexAttempt] = [] + for i, pri in enumerate(priorities): + a = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=pri, + days_old=0, + ) + # Force monotonic time_created so the FIFO tiebreak is + # deterministic regardless of microsecond clock drift. + a.time_created = datetime.datetime.now( + tz=datetime.timezone.utc + ) - datetime.timedelta(seconds=(len(priorities) - i) * 10) + attempts.append(a) + db.commit() + + a, b, c, d, e = attempts + all_not_started = get_not_started_index_attempts(db) + # Filter to only ones we created (DB may have other NOT_STARTED rows + # from other tests / production). + ours = [x for x in all_not_started if x.id in {a.id, b.id, c.id, d.id, e.id}] + ordered_priorities = [x.indexing_priority for x in ours] + assert_list_eq( + ordered_priorities, + [10, 5, 3, 0, 0], + "get_not_started_index_attempts: priority DESC ordering", + ) + + # Within the priority=0 tier, the OLDER one (a, time_created + # earliest) should come before d (time_created later). + zero_tier = [x.id for x in ours if x.indexing_priority == 0] + assert_list_eq( + zero_tier, + [a.id, d.id], + "get_not_started_index_attempts: time_created ASC tiebreak", + ) + + # update_index_attempt_priority on a NOT_STARTED row succeeds + bumped = update_index_attempt_priority(a.id, 50, db) + assert_true( + bumped is not None and bumped.indexing_priority == 50, + "update_index_attempt_priority: NOT_STARTED → priority bumped", + f"got: {bumped.indexing_priority if bumped else None}", + ) + + # Bumping above the 100 ceiling is clamped + capped = update_index_attempt_priority(a.id, 1000, db) + assert_true( + capped is not None and capped.indexing_priority == 100, + "update_index_attempt_priority: clamps to ceiling=100", + f"got: {capped.indexing_priority if capped else None}", + ) + + # Mark one as IN_PROGRESS — update should now refuse (returns None) + mark_attempt_in_progress__no_commit(c) + db.commit() + rejected = update_index_attempt_priority(c.id, 99, db) + assert_eq( + rejected, + None, + "update_index_attempt_priority: IN_PROGRESS → refused (returns None)", + ) + + # Verify priority unchanged on c + db.refresh(c) + assert_eq( + c.indexing_priority, + 10, + "update_index_attempt_priority: IN_PROGRESS row priority untouched", + ) + + +# --------------------------------------------------------------------------- +# Phase 2 — index_attempt retention (also verifies P0 lowercase fix) +# --------------------------------------------------------------------------- + + +def phase_index_attempt_retention() -> None: + section("Phase 2 — index_attempt retention (lowercase fix + keep_last_N)") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + lookup_persona(db) + em = lookup_embedding_model(db) + make_user(db) + _conn, _cred, ccp = make_connector_and_pair(db) + + # Seed 25 SUCCESS attempts for one cc-pair, all 70 days old. + # With RETENTION_DAYS_INDEX_ATTEMPT=60 and KEEP_LAST_N=20, the + # retention sweep should delete the 5 oldest (rn > 20 partition rule). + for i in range(25): + make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.SUCCESS, + days_old=70 + i, # day-spread keeps row_number deterministic + ) + db.commit() + before = ( + db.query(IndexAttempt) + .filter(IndexAttempt.connector_id == ccp.connector_id) + .filter(IndexAttempt.credential_id == ccp.credential_id) + .count() + ) + assert_eq(before, 25, "index_attempt rows seeded (25 SUCCESS, 70d+ old)") + + # Run retention with index_attempt enabled via env-var injection. + # cleanup_stale_db.py reads RETENTION_DAYS_* at module import time, so + # we have to spawn a subprocess with the env baked in. + print( + " $ RETENTION_DAYS_INDEX_ATTEMPT=60 RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS=20 \\" + ) + print(" cleanup_stale_db.py --policy=index_attempt") + env = { + **os.environ, + "RETENTION_DAYS_INDEX_ATTEMPT": "60", + "RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS": "20", + } + result = subprocess.run( + [sys.executable, "scripts/cleanup_stale_db.py", "--policy=index_attempt"], + cwd=str(THIS_DIR.parent), + env=env, + capture_output=True, + text=True, + ) + if result.returncode != 0: + for line in (result.stdout or "").splitlines()[-10:]: + print(f" {line}") + for line in (result.stderr or "").splitlines()[-10:]: + print(f" {line}") + failed("cleanup_stale_db.py --policy=index_attempt failed") + return + # Print the policy summary table from cleanup_stale_db.py for context. + for line in (result.stdout or "").splitlines(): + s = line.strip() + if s.startswith("index_attempt") or s.startswith("policy"): + print(f" {line}") + + with Session(engine) as db: + after = ( + db.query(IndexAttempt) + .filter(IndexAttempt.connector_id == ccp.connector_id) + .filter(IndexAttempt.credential_id == ccp.credential_id) + .count() + ) + # P0 fix verification: if SQL still used uppercase 'SUCCESS', this + # would no-op and `after == 25`. Lowercase 'success' is what's + # actually stored, so 5 should be deleted. + assert_eq( + after, + 20, + "P0 lowercase status fix: 5 oldest deleted, 20 newest kept", + ) + + +# --------------------------------------------------------------------------- +# Phase 3 — permission_sync_run retention preserves in_progress +# --------------------------------------------------------------------------- + + +def phase_permission_sync_retention() -> None: + section("Phase 3 — permission_sync_run retention preserves in_progress") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + _conn, _cred, ccp = make_connector_and_pair(db) + # Insert via raw SQL because PermissionSyncRun.update_type / .status + # are declared without native_enum=False; SA bulk insert tries to + # cast to a non-existent PG enum type. (Same workaround as the + # main seeder.) + old = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta( + days=90 + ) + rows = [ + ("github", "user_level", ccp.id, "success", old), + ("github", "user_level", ccp.id, "success", old), + ("github", "user_level", ccp.id, "failed", old), + ("github", "user_level", ccp.id, "failed", old), + ("github", "user_level", ccp.id, "failed", old), + ("github", "user_level", ccp.id, "in_progress", old), + ("github", "user_level", ccp.id, "in_progress", old), + ("github", "user_level", ccp.id, "in_progress", old), + ] + for src, ut, ccp_id, st, when in rows: + db.execute( + text( + """ + INSERT INTO permission_sync_run + (source_type, update_type, cc_pair_id, status, + error_msg, updated_at) + VALUES + (:src, :ut, :ccp_id, :st, NULL, :when) + """ + ), + {"src": src, "ut": ut, "ccp_id": ccp_id, "st": st, "when": when}, + ) + db.commit() + + before_total = db.execute( + text("SELECT count(*) FROM permission_sync_run WHERE cc_pair_id = :id"), + {"id": ccp.id}, + ).scalar() + before_in_progress = db.execute( + text( + "SELECT count(*) FROM permission_sync_run " + "WHERE cc_pair_id = :id AND status = 'in_progress'" + ), + {"id": ccp.id}, + ).scalar() + assert_eq(int(before_total or 0), 8, "seeded permission_sync_run rows") + assert_eq(int(before_in_progress or 0), 3, "seeded in_progress rows") + + print(" $ cleanup_stale_db.py --policy=permission_sync_run") + result = subprocess.run( + [ + sys.executable, + "scripts/cleanup_stale_db.py", + "--policy=permission_sync_run", + ], + cwd=str(THIS_DIR.parent), + capture_output=True, + text=True, + ) + if result.returncode != 0: + for line in (result.stderr or "").splitlines()[-10:]: + print(f" {line}") + failed("cleanup_stale_db.py --policy=permission_sync_run failed") + return + + with Session(engine) as db: + after_total = db.execute( + text("SELECT count(*) FROM permission_sync_run WHERE cc_pair_id = :id"), + {"id": ccp.id}, + ).scalar() + after_in_progress = db.execute( + text( + "SELECT count(*) FROM permission_sync_run " + "WHERE cc_pair_id = :id AND status = 'in_progress'" + ), + {"id": ccp.id}, + ).scalar() + after_terminal = db.execute( + text( + "SELECT count(*) FROM permission_sync_run " + "WHERE cc_pair_id = :id AND status IN ('success', 'failed')" + ), + {"id": ccp.id}, + ).scalar() + + assert_eq(int(after_terminal or 0), 0, "terminal rows deleted (success + failed)") + assert_eq( + int(after_in_progress or 0), + 3, + "in_progress rows preserved despite 90d age", + ) + assert_eq(int(after_total or 0), 3, "only in_progress remain") + + +# --------------------------------------------------------------------------- +# Phase 4 — Resolved-button feedback DB write path +# --------------------------------------------------------------------------- + + +def phase_resolved_feedback_write() -> None: + section("Phase 4 — resolved-feedback DB write path") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + persona = lookup_persona(db) + # Slackbot sessions have user_id=NULL by design (the slackbot + # handler has no Danswer user, just a Slack user). The + # `create_chat_message_feedback` permission check rejects + # ownership mismatches, so the test session must mirror that. + when = datetime.datetime.now(tz=datetime.timezone.utc) + cs = ChatSession( + user_id=None, + persona_id=persona.id, + description=f"{FEATURE_PREFIX}resolved-feedback-{uuid.uuid4().hex[:6]}", + deleted=False, + one_shot=False, + shared_status=ChatSessionSharedStatus.PRIVATE, + danswerbot_flow=True, + time_created=when, + time_updated=when, + ) + db.add(cs) + db.flush() + msg = ChatMessage( + chat_session_id=cs.id, + message="Test reply", + message_type=MessageType.ASSISTANT, + token_count=5, + time_sent=when, + ) + db.add(msg) + db.flush() + msg_id = msg.id + db.commit() + + # Same call shape as `handle_followup_resolved_button` (post-fix): + # chat_message_id, predefined_feedback='resolved', no is_positive. + create_chat_message_feedback( + is_positive=None, + feedback_text="", + chat_message_id=msg_id, + user_id=None, + db_session=db, + predefined_feedback="resolved", + ) + + rows = ( + db.query(ChatMessageFeedback) + .filter(ChatMessageFeedback.chat_message_id == msg_id) + .all() + ) + + assert_eq(len(rows), 1, "exactly one feedback row written") + if not rows: + return + fb = rows[0] + assert_eq(fb.predefined_feedback, "resolved", "predefined_feedback='resolved'") + assert_eq(fb.is_positive, None, "is_positive is NULL for resolved") + assert_eq(fb.required_followup, None, "required_followup is NULL") + assert_eq(fb.chat_message_id, msg_id, "chat_message_id matches") + + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + + +def cleanup_test_data() -> None: + """Drop everything tagged with FEATURE_PREFIX, FK-safe order.""" + engine = get_sqlalchemy_engine() + with Session(engine) as db: + # chat_feedback for our messages (auto SET NULL would zombie them) + db.execute( + text( + """ + DELETE FROM chat_feedback + WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + # chat_message__search_doc / tool_call / chat_message + db.execute( + text( + """ + DELETE FROM chat_message__search_doc + WHERE chat_message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.execute( + text( + """ + DELETE FROM tool_call WHERE message_id IN ( + SELECT cm.id FROM chat_message cm + JOIN chat_session cs ON cs.id = cm.chat_session_id + WHERE cs.description LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.execute( + text( + """ + DELETE FROM chat_message + WHERE chat_session_id IN ( + SELECT id FROM chat_session WHERE description LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM chat_session WHERE description LIKE :p"), + {"p": f"{FEATURE_PREFIX}%"}, + ) + # permission_sync_run + index_attempt linked to seeded ccps + db.execute( + text( + """ + DELETE FROM permission_sync_run + WHERE cc_pair_id IN ( + SELECT id FROM connector_credential_pair WHERE name LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.execute( + text( + """ + DELETE FROM index_attempt + WHERE connector_id IN ( + SELECT id FROM connector WHERE name LIKE :p + ) + """ + ), + {"p": f"{FEATURE_PREFIX}%"}, + ) + # cc_pair → connector + db.execute( + text("DELETE FROM connector_credential_pair WHERE name LIKE :p"), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM connector WHERE name LIKE :p"), + {"p": f"{FEATURE_PREFIX}%"}, + ) + # users last + db.execute( + text("""DELETE FROM "user" WHERE email LIKE :p"""), + {"p": f"{FEATURE_PREFIX}%"}, + ) + db.commit() + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +THIS_DIR = Path(__file__).resolve().parent + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", action="store_true", help="Skip the destructive-op confirmation." + ) + parser.add_argument( + "--keep-data", + action="store_true", + help="Don't clean tagged data at the end (debugging).", + ) + args = parser.parse_args() + + engine = get_sqlalchemy_engine() + safe_url = ( + f"{engine.url.drivername}://{engine.url.username}@" + f"{engine.url.host}:{engine.url.port}/{engine.url.database}" + ) + print(f"Target DB: {safe_url}") + if not args.yes: + ans = input( + "This will create + delete tagged data in the target DB. " + "Type 'yes' to continue: " + ) + if ans.strip().lower() != "yes": + print("Aborted.") + return 1 + + # Best-effort pre-clean — if a prior run left tagged junk behind, clear it. + cleanup_test_data() + + try: + phase_priority_ordering() + phase_index_attempt_retention() + phase_permission_sync_retention() + phase_resolved_feedback_write() + finally: + if not args.keep_data: + cleanup_test_data() + print("\n cleaned up tagged test data") + + print() + if _FAILED: + print(f"❌ {_FAILED} assertion(s) failed.") + return 1 + print("✅ All phases passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/test_scheduler_e2e.py b/backend/scripts/test_scheduler_e2e.py new file mode 100644 index 00000000000..a0f1990133c --- /dev/null +++ b/backend/scripts/test_scheduler_e2e.py @@ -0,0 +1,1054 @@ +"""End-to-end integration tests for ``kickoff_indexing_jobs``. + +Companion to ``test_features_e2e.py`` and ``test_analytics_e2e.py``. This +script seeds real ``connector`` / ``credential`` / ``connector_credential_pair`` +/ ``index_attempt`` rows in the configured Postgres, then drives the real +``kickoff_indexing_jobs`` function with a recording fake Dask client. The +fake records every ``submit`` call without spawning a worker, so we can +assert on the scheduler's dispatch decisions against live SQLAlchemy ORM, +the live priority sort in ``get_not_started_index_attempts``, and the +live cap / cc-pair guard logic. + +The unit tests in ``tests/unit/danswer/background/test_indexing_scheduler.py`` +exercise the pure helpers in isolation. This script proves the same +guarantees hold when the helpers are wired through the ORM, ``Session``, +``IndexAttempt`` filters, and the ``existing_jobs`` ↔ DB-IN_PROGRESS +accounting that fixed the cap-leak bug. + +Phases: + Phase A — priority order under cap=1: highest-priority slack attempt + wins; lower-priority same-source attempts stay NOT_STARTED. + Phase B — cap-leak regression: a slack attempt sitting in + ``existing_jobs`` (Dask-queued, DB row still NOT_STARTED) + must hold its source-cap slot. A second slack candidate in + the same tick must NOT be dispatched. Pre-fix, it leaked. + Phase C — per-cc-pair guard with a real IN_PROGRESS row: a manual + Re-Index colliding with an auto-scheduled run must be + deferred (no FAILED row, just NOT_STARTED waiting). + Phase D — different sources don't share the cap: cap=1 + slack + + github + confluence all NOT_STARTED → all three dispatched + in one tick. + Phase E — same-tick same-cc-pair double-submit prevented: two + NOT_STARTED rows for the same cc-pair → only one dispatches. + Phase F — soak: run the same scheduler tick N times against a + randomized seed of attempts, asserting cap + cc-pair + invariants every iteration. This is the integration twin + of the unit-level fuzz test, but driving the real DB. + +DESTRUCTIVE: writes/deletes tagged data only (``__test_scheduler__`` +prefix). Run only against a dev / staging DB. + +Pre-flight: refuses to run if the production indexer is up — it would +race on our seeded NOT_STARTED rows. Stop it with:: + + pkill -f 'danswer/background/update.py' + +…or run with ``--force-with-indexer-running`` to override (your test +attempts may then be picked up by the real scheduler before this +script asserts on them — expect noise). + +Usage:: + + cd backend + PYTHONPATH=$(pwd) python scripts/test_scheduler_e2e.py [--yes] [--keep-data] \\ + [--soak-iterations=20] [--force-with-indexer-running] +""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import random +import subprocess +import sys +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from sqlalchemy import select +from sqlalchemy import text +from sqlalchemy.orm import Session + +import danswer.background.update as scheduler +from danswer.configs.constants import DocumentSource +from danswer.connectors.models import InputType +from danswer.db.engine import get_sqlalchemy_engine +from danswer.db.models import Connector +from danswer.db.models import ConnectorCredentialPair +from danswer.db.models import Credential +from danswer.db.models import EmbeddingModel +from danswer.db.models import IndexAttempt +from danswer.db.models import IndexingStatus + + +SCHEDULER_PREFIX = "__test_scheduler__" + + +# Sources we use for test connectors — chosen from DocumentSource members +# that are *unlikely* to be used in any real environment, so production +# NOT_STARTED rows can't leak into the cap accounting alongside ours. +# `kickoff_indexing_jobs` legitimately considers all NOT_STARTED rows in +# the DB, not just tagged ones; if the test used SLACK/GITHUB and the +# environment had production NOT_STARTED rows on those sources, the cap +# would already be claimed by them and the test wouldn't isolate +# scheduler behaviour from environment state. +_SAFE_SOURCES_PRIMARY = [ + DocumentSource.GMAIL, + DocumentSource.ZULIP, + DocumentSource.JIRA, + DocumentSource.ZENDESK, + DocumentSource.NOTION, + DocumentSource.LINEAR, +] + + +def pick_safe_sources(db: Session, n: int) -> list[DocumentSource]: + """Return ``n`` distinct DocumentSource values that have NO existing + rows in either ``connector`` or NOT_STARTED ``index_attempt`` (other + than tagged test rows). If insufficient unused sources are + available, abort with a clear message — the test framework can't + isolate scheduler behaviour from environment otherwise.""" + used: set[str] = set() + rows = db.execute( + text("SELECT DISTINCT source FROM connector WHERE name NOT LIKE :p"), + {"p": f"{SCHEDULER_PREFIX}%"}, + ).fetchall() + used.update(r[0].upper() for r in rows) + rows = db.execute( + text( + "SELECT DISTINCT co.source FROM index_attempt ia " + "JOIN connector co ON co.id = ia.connector_id " + "WHERE ia.status = 'NOT_STARTED' AND co.name NOT LIKE :p" + ), + {"p": f"{SCHEDULER_PREFIX}%"}, + ).fetchall() + used.update(r[0].upper() for r in rows) + + safe = [s for s in _SAFE_SOURCES_PRIMARY if s.name.upper() not in used] + if len(safe) < n: + sys.exit( + f"Need {n} unused DocumentSource values for the test; " + f"only {len(safe)} are free in this DB. Used by environment: " + f"{sorted(used)}. Free: {[s.name for s in safe]}. Either " + "delete the conflicting connectors or extend " + "_SAFE_SOURCES_PRIMARY." + ) + return safe[:n] + + +# --------------------------------------------------------------------------- +# Tiny harness (mirrors the other e2e scripts) +# --------------------------------------------------------------------------- + + +_FAILED = 0 + + +def section(name: str) -> None: + print(f"\n=== {name} ===") + + +def passed(msg: str) -> None: + print(f" ok {msg}") + + +def failed(msg: str, detail: str | None = None) -> None: + global _FAILED + _FAILED += 1 + print(f" XX {msg}") + if detail: + for line in detail.splitlines(): + print(f" {line}") + + +def assert_eq(actual: Any, expected: Any, label: str) -> None: + if actual == expected: + passed(f"{label} — {actual}") + else: + failed(label, f"expected={expected}, actual={actual}") + + +def assert_set_eq(actual: set, expected: set, label: str) -> None: + if actual == expected: + passed(f"{label} — {sorted(actual)}") + else: + failed( + label, + f"expected={sorted(expected)}, actual={sorted(actual)}, " + f"missing={sorted(expected - actual)}, extra={sorted(actual - expected)}", + ) + + +def assert_true(cond: bool, label: str, detail: str = "") -> None: + if cond: + passed(label) + else: + failed(label, detail or None) + + +# --------------------------------------------------------------------------- +# Recording fake Dask client +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _RecordedJob: + """Stand-in for ``dask.distributed.Future`` / ``SimpleJob``. The + scheduler stores it in ``existing_jobs`` and queries ``done()`` on + the next cleanup pass. We pretend the job is still running.""" + + job_id: int + + def done(self) -> bool: + return False + + def cancel(self) -> None: # pragma: no cover — not exercised here + pass + + def release(self) -> None: # pragma: no cover + pass + + @property + def status(self) -> str: + return "pending" + + def exception(self) -> str: # pragma: no cover + return "" + + +@dataclasses.dataclass +class _RecordedSubmit: + func: Callable[..., Any] + args: tuple[Any, ...] + kwargs: dict[str, Any] + + +class RecordingClient: + """Replacement for ``dask.distributed.Client`` that records every + ``submit`` call without spawning a worker. Crucially this is *not* + a subclass of ``dask.distributed.Client``, so the scheduler's + ``isinstance(client, Client)`` check is False and it won't pass the + ``priority`` kwarg. Priority kwarg behaviour is covered by the + unit-test ``test_priority_strict_order_under_cap_one``; here we + care about the dispatch / defer decisions.""" + + def __init__(self) -> None: + self.submitted: list[_RecordedSubmit] = [] + self._counter = 0 + + def submit( + self, func: Callable[..., Any], *args: Any, **kwargs: Any + ) -> _RecordedJob: + self._counter += 1 + self.submitted.append(_RecordedSubmit(func=func, args=args, kwargs=kwargs)) + return _RecordedJob(job_id=self._counter) + + def submitted_attempt_ids(self) -> set[int]: + # `run_indexing_entrypoint` is called as + # `submit(run_indexing_entrypoint, attempt.id, is_ee_version)`. + return {int(rec.args[0]) for rec in self.submitted if rec.args} + + +# --------------------------------------------------------------------------- +# Seed helpers (everything tagged with SCHEDULER_PREFIX) +# --------------------------------------------------------------------------- + + +def lookup_embedding_model(db: Session) -> EmbeddingModel: + em = db.execute( + select(EmbeddingModel).order_by(EmbeddingModel.id).limit(1) + ).scalar_one_or_none() + if em is None: + sys.exit( + "No embedding_model found. Bootstrap your DB first " + "(alembic upgrade head + start the API server once)." + ) + return em + + +def make_connector_and_pair( + db: Session, source: DocumentSource +) -> tuple[Connector, Credential, ConnectorCredentialPair]: + """Create a connector + credential + cc-pair tagged with the test + prefix. The connector is ``disabled=True`` so the production + scheduler's `_should_create_new_indexing` won't auto-spawn fresh + attempts mid-test (we manually seed the attempts we need).""" + connector = Connector( + name=f"{SCHEDULER_PREFIX}{source.value}-{uuid.uuid4().hex[:6]}", + source=source, + input_type=InputType.POLL, + connector_specific_config={"_test_scheduler": True}, + refresh_freq=600, + disabled=True, # belt-and-suspenders against the prod scheduler + ) + credential = Credential(admin_public=True, credential_json={}) + db.add_all([connector, credential]) + db.flush() + ccp = ConnectorCredentialPair( + connector_id=connector.id, + credential_id=credential.id, + name=f"{SCHEDULER_PREFIX}ccp-{uuid.uuid4().hex[:6]}", + is_public=True, + total_docs_indexed=0, + ) + db.add(ccp) + db.flush() + return connector, credential, ccp + + +def make_attempt( + db: Session, + *, + connector_id: int, + credential_id: int, + embedding_model_id: int, + status: IndexingStatus, + priority: int = 0, + seconds_old: int = 0, +) -> IndexAttempt: + when = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta( + seconds=seconds_old + ) + attempt = IndexAttempt( + connector_id=connector_id, + credential_id=credential_id, + embedding_model_id=embedding_model_id, + from_beginning=False, + status=status, + new_docs_indexed=0, + total_docs_indexed=0, + docs_removed_from_index=0, + indexing_priority=priority, + time_created=when, + time_updated=when, + time_started=when if status != IndexingStatus.NOT_STARTED else None, + ) + db.add(attempt) + db.flush() + return attempt + + +# --------------------------------------------------------------------------- +# Tick driver +# --------------------------------------------------------------------------- + + +def run_one_tick( + *, + cap: int, + existing_jobs: dict[int, _RecordedJob] | None = None, +) -> tuple[RecordingClient, dict[int, _RecordedJob]]: + """Drive ``kickoff_indexing_jobs`` exactly once with a fresh + recording client and the requested ``existing_jobs`` map.""" + primary = RecordingClient() + secondary = RecordingClient() + prev_cap = scheduler.PER_SOURCE_CAP + scheduler.PER_SOURCE_CAP = cap + try: + next_jobs = scheduler.kickoff_indexing_jobs( + existing_jobs or {}, + primary, # type: ignore[arg-type] + secondary, # type: ignore[arg-type] + ) + finally: + scheduler.PER_SOURCE_CAP = prev_cap + return primary, next_jobs + + +def fetch_attempt(db: Session, attempt_id: int) -> IndexAttempt: + attempt = db.execute( + select(IndexAttempt).where(IndexAttempt.id == attempt_id) + ).scalar_one() + return attempt + + +# --------------------------------------------------------------------------- +# Phase A — priority order under cap=1 +# --------------------------------------------------------------------------- + + +def phase_a_priority_under_cap() -> None: + section("Phase A — priority order under cap=1 (mirrors user scenario)") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + # Three same-source cc-pairs, all NOT_STARTED. The middle one is + # priority-bumped to 20; the others sit at 0. This is the exact + # shape the user reported (3 Slack connectors, one bumped) but + # on a source the environment isn't using. + attempt_ids: list[int] = [] + priority_for: dict[int, int] = {} + for i, prio in enumerate([0, 20, 0]): + _, _, ccp = make_connector_and_pair(db, test_source) + a = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=prio, + seconds_old=(3 - i) * 30, # FIFO tiebreak: deterministic + ) + attempt_ids.append(a.id) + priority_for[a.id] = prio + db.commit() + + bumped_id = next(aid for aid, p in priority_for.items() if p == 20) + primary, _ = run_one_tick(cap=1) + submitted = primary.submitted_attempt_ids() & set(attempt_ids) + assert_set_eq( + submitted, + {bumped_id}, + "cap=1: only the priority-20 attempt is dispatched", + ) + + # Verify the other two are still NOT_STARTED in DB (deferred, not + # marked failed). + with Session(engine) as db: + for aid in attempt_ids: + if aid == bumped_id: + continue + a = fetch_attempt(db, aid) + assert_eq( + a.status, + IndexingStatus.NOT_STARTED, + f"deferred attempt {aid} stays NOT_STARTED (no FAILED row)", + ) + + +# --------------------------------------------------------------------------- +# Phase B — cap-leak regression (THE bug fix) +# --------------------------------------------------------------------------- + + +def phase_b_cap_leak_regression() -> None: + section("Phase B — cap-leak: dispatched attempt holds its source slot") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + _, _, ccp_1 = make_connector_and_pair(db, test_source) + _, _, ccp_2 = make_connector_and_pair(db, test_source) + # Both NOT_STARTED. ``in_flight`` is the one we'll pretend was + # already dispatched in a prior tick (it sits in existing_jobs + # with the DB row still NOT_STARTED — the exact window where + # the pre-fix scheduler leaked the cap). + in_flight = make_attempt( + db, + connector_id=ccp_1.connector_id, + credential_id=ccp_1.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=0, + seconds_old=120, + ) + candidate = make_attempt( + db, + connector_id=ccp_2.connector_id, + credential_id=ccp_2.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=0, + seconds_old=60, + ) + db.commit() + in_flight_id = in_flight.id + candidate_id = candidate.id + + # Simulate "in_flight was dispatched in the previous tick but the + # worker hasn't flipped the row to IN_PROGRESS yet". + existing_jobs: dict[int, _RecordedJob] = {in_flight_id: _RecordedJob(job_id=999)} + primary, _ = run_one_tick(cap=1, existing_jobs=existing_jobs) + submitted = primary.submitted_attempt_ids() & {in_flight_id, candidate_id} + + # Pre-fix expectation (BUG): submitted == {candidate_id} (leaked). + # Post-fix expectation: submitted == set() (cap held by in_flight). + assert_set_eq( + submitted, + set(), + "cap-leak fix: dispatched-but-not-IN_PROGRESS attempt blocks " + "second same-source candidate", + ) + + with Session(engine) as db: + a = fetch_attempt(db, candidate_id) + assert_eq( + a.status, + IndexingStatus.NOT_STARTED, + f"candidate {candidate_id} stays NOT_STARTED (no FAILED)", + ) + + +# --------------------------------------------------------------------------- +# Phase C — per-cc-pair guard with a real IN_PROGRESS row +# --------------------------------------------------------------------------- + + +def phase_c_cc_pair_guard_with_in_progress() -> None: + section("Phase C — per-cc-pair guard: Re-Index collides with auto-run") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + _, _, ccp = make_connector_and_pair(db, test_source) + running = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.IN_PROGRESS, + seconds_old=60, + ) + reindex = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + ) + db.commit() + running_id = running.id + reindex_id = reindex.id + + primary, _ = run_one_tick(cap=4) # cap large enough that source isn't the limit + submitted = primary.submitted_attempt_ids() & {running_id, reindex_id} + assert_set_eq( + submitted, + set(), + "manual Re-Index attempt deferred while IN_PROGRESS exists for cc-pair", + ) + + with Session(engine) as db: + a = fetch_attempt(db, reindex_id) + assert_eq( + a.status, + IndexingStatus.NOT_STARTED, + "Re-Index stays NOT_STARTED (per-cc-pair guard, no FAILED row)", + ) + + +# --------------------------------------------------------------------------- +# Phase D — different sources don't share the cap +# --------------------------------------------------------------------------- + + +def phase_d_different_sources_independent_caps() -> None: + section("Phase D — different sources have independent caps") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + test_sources = pick_safe_sources(db, 3) + attempt_ids: list[int] = [] + for src in test_sources: + _, _, ccp = make_connector_and_pair(db, src) + a = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + ) + attempt_ids.append(a.id) + db.commit() + + primary, _ = run_one_tick(cap=1) + submitted = primary.submitted_attempt_ids() & set(attempt_ids) + assert_set_eq( + submitted, + set(attempt_ids), + "cap=1 + 3 different sources → all 3 dispatched", + ) + + +# --------------------------------------------------------------------------- +# Phase E — same-tick same-cc-pair double-submit prevented +# --------------------------------------------------------------------------- + + +def phase_e_same_cc_pair_double_submit_prevented() -> None: + section("Phase E — two NOT_STARTED for same cc-pair in one tick") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + _, _, ccp = make_connector_and_pair(db, test_source) + a = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + seconds_old=60, + ) + b = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + seconds_old=30, + ) + db.commit() + ids = {a.id, b.id} + + primary, _ = run_one_tick(cap=4) + submitted = primary.submitted_attempt_ids() & ids + assert_eq( + len(submitted), + 1, + "exactly one of the same-cc-pair NOT_STARTED rows dispatched", + ) + + +# --------------------------------------------------------------------------- +# Phase G — completed existing_jobs entries don't consume cap slots +# --------------------------------------------------------------------------- + + +def phase_g_completed_in_flight_does_not_block() -> None: + """The cap-leak fix counts dispatched-but-pre-completion attempts + toward ``running_per_source``. The flip side: an attempt that has + *finished* (DB row is SUCCESS or FAILED) must NOT count, even if + it's still sitting in ``existing_jobs`` (cleanup happens on the + next scheduler tick). Without this, a one-shot finished job would + keep its source's cap occupied until cleanup, which would + artificially serialize the queue. + + The query in ``_build_running_view`` filters with + ``IndexAttempt.status.notin_([SUCCESS, FAILED])``; this phase + proves that filter actually excludes a completed row from the + accounting against a live DB. + """ + section("Phase G — completed in_flight (SUCCESS) doesn't consume cap") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + # `finished_attempt` is in `existing_jobs` (the scheduler hasn't + # cleaned it yet) but its DB row is already SUCCESS — the + # accounting must skip it. + _, _, ccp_done = make_connector_and_pair(db, test_source) + finished_attempt = make_attempt( + db, + connector_id=ccp_done.connector_id, + credential_id=ccp_done.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.SUCCESS, + priority=0, + seconds_old=120, + ) + # Fresh NOT_STARTED of the SAME source; cap=1 must let it + # through because finished_attempt does NOT hold a slot. + _, _, ccp_new = make_connector_and_pair(db, test_source) + candidate = make_attempt( + db, + connector_id=ccp_new.connector_id, + credential_id=ccp_new.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=0, + seconds_old=60, + ) + db.commit() + finished_id = finished_attempt.id + candidate_id = candidate.id + + existing_jobs: dict[int, _RecordedJob] = { + finished_id: _RecordedJob(job_id=finished_id) + } + primary, _ = run_one_tick(cap=1, existing_jobs=existing_jobs) + submitted = primary.submitted_attempt_ids() & {finished_id, candidate_id} + assert_set_eq( + submitted, + {candidate_id}, + "completed in_flight skipped by cap accounting; new candidate dispatches", + ) + + +# --------------------------------------------------------------------------- +# Phase H — FAILED existing_jobs entries also don't consume cap slots +# --------------------------------------------------------------------------- + + +def phase_h_failed_in_flight_does_not_block() -> None: + """Same invariant as Phase G but for the FAILED status — both + terminal states must be excluded from the dispatched-pre-completion + accounting.""" + section("Phase H — FAILED in_flight doesn't consume cap") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + _, _, ccp_failed = make_connector_and_pair(db, test_source) + failed_attempt = make_attempt( + db, + connector_id=ccp_failed.connector_id, + credential_id=ccp_failed.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.FAILED, + priority=0, + seconds_old=120, + ) + _, _, ccp_new = make_connector_and_pair(db, test_source) + candidate = make_attempt( + db, + connector_id=ccp_new.connector_id, + credential_id=ccp_new.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=0, + seconds_old=60, + ) + db.commit() + failed_id = failed_attempt.id + candidate_id = candidate.id + + existing_jobs: dict[int, _RecordedJob] = {failed_id: _RecordedJob(job_id=failed_id)} + primary, _ = run_one_tick(cap=1, existing_jobs=existing_jobs) + submitted = primary.submitted_attempt_ids() & {failed_id, candidate_id} + assert_set_eq( + submitted, + {candidate_id}, + "FAILED in_flight skipped; new candidate dispatches", + ) + + +# --------------------------------------------------------------------------- +# Phase I — IN_PROGRESS dispatched attempt accounted via DB query, not +# double-counted via existing_jobs path +# --------------------------------------------------------------------------- + + +def phase_i_in_progress_dispatched_not_double_counted() -> None: + """When a dispatched attempt has flipped to IN_PROGRESS in the DB, + the primary IN_PROGRESS query already counts it. The dispatched- + pre-completion query then runs over `unaccounted_dispatched_ids`, + skipping anything already in the IN_PROGRESS set. If the dedup + logic broke, the same attempt would be counted twice and a same- + source new candidate would be deferred even though only ONE + cc-pair is actually running. + + With cap=2, an IN_PROGRESS slack + a NOT_STARTED slack candidate + must both fit (count = 1 + 1 = 2 ≤ cap). If double-counting were + happening, count would read 2 + 1 = 3 → defer the candidate. + """ + section("Phase I — IN_PROGRESS dispatched not double-counted (cap=2)") + + engine = get_sqlalchemy_engine() + with Session(engine) as db: + em = lookup_embedding_model(db) + (test_source,) = pick_safe_sources(db, 1) + _, _, ccp_running = make_connector_and_pair(db, test_source) + running_attempt = make_attempt( + db, + connector_id=ccp_running.connector_id, + credential_id=ccp_running.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.IN_PROGRESS, + priority=0, + seconds_old=60, + ) + _, _, ccp_new = make_connector_and_pair(db, test_source) + candidate = make_attempt( + db, + connector_id=ccp_new.connector_id, + credential_id=ccp_new.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=0, + seconds_old=30, + ) + db.commit() + running_id = running_attempt.id + candidate_id = candidate.id + + # `running_attempt` is in BOTH the IN_PROGRESS DB set AND + # existing_jobs — the de-dup must keep the cap count at 1. + existing_jobs: dict[int, _RecordedJob] = { + running_id: _RecordedJob(job_id=running_id) + } + primary, _ = run_one_tick(cap=2, existing_jobs=existing_jobs) + submitted = primary.submitted_attempt_ids() & {running_id, candidate_id} + assert_set_eq( + submitted, + {candidate_id}, + "no double-counting: candidate dispatches under cap=2", + ) + + +# --------------------------------------------------------------------------- +# Phase F — soak: N randomized ticks, invariants hold every time +# --------------------------------------------------------------------------- + + +def phase_f_soak_random_ticks(iterations: int) -> None: + section(f"Phase F — soak: {iterations} randomized ticks") + + rng = random.Random(0xC0DE) + engine = get_sqlalchemy_engine() + violations = 0 + with Session(engine) as db: + sources = pick_safe_sources(db, 4) + + for it in range(iterations): + # Clean prior iteration's tagged rows so each tick observes a + # fresh queue. Without this, deferred NOT_STARTED rows from + # iteration N contaminate iteration N+1's cap accounting. + cleanup_test_data() + with Session(engine) as db: + em = lookup_embedding_model(db) + n_attempts = rng.randint(3, 8) + seeded_ids: list[int] = [] + seeded_source_for: dict[int, str] = {} + seeded_cc_pair_for: dict[int, tuple[int | None, int | None, int]] = {} + cap = rng.choice([1, 1, 1, 2]) # weighted toward 1 + + # Build a small pool of cc-pairs so collisions can happen. + ccp_pool: list[tuple[Connector, Credential, ConnectorCredentialPair]] = [] + for _ in range(rng.randint(2, 5)): + src = rng.choice(sources) + ccp_pool.append(make_connector_and_pair(db, src)) + + for _ in range(n_attempts): + conn, cred, ccp = rng.choice(ccp_pool) + a = make_attempt( + db, + connector_id=ccp.connector_id, + credential_id=ccp.credential_id, + embedding_model_id=em.id, + status=IndexingStatus.NOT_STARTED, + priority=rng.choice([0, 0, 10, 20]), + seconds_old=rng.randint(1, 600), + ) + seeded_ids.append(a.id) + seeded_source_for[a.id] = conn.source.value + seeded_cc_pair_for[a.id] = ( + a.connector_id, + a.credential_id, + a.embedding_model_id, + ) + db.commit() + + # Simulate prior-tick existing_jobs: pick up to 2 attempts that + # form a VALID prior state (respect cap + cc-pair invariants). + # Without this, the soak would feed kickoff a precondition that + # already violates the invariants, and the post-tick check would + # flag a "violation" that isn't actually a scheduler bug. + in_flight_ids: set[int] = set() + in_flight_per_source: dict[str, int] = {} + in_flight_cc_pairs: set[tuple[int | None, int | None, int]] = set() + shuffled = list(seeded_ids) + rng.shuffle(shuffled) + for aid in shuffled: + if rng.random() > 0.4: # 40% chance to include + continue + src = seeded_source_for[aid] + cc = seeded_cc_pair_for[aid] + if cap > 0 and in_flight_per_source.get(src, 0) >= cap: + continue + if cc in in_flight_cc_pairs: + continue + in_flight_ids.add(aid) + in_flight_per_source[src] = in_flight_per_source.get(src, 0) + 1 + in_flight_cc_pairs.add(cc) + if len(in_flight_ids) >= 2: + break + existing_jobs = {aid: _RecordedJob(job_id=aid) for aid in in_flight_ids} + + primary, _ = run_one_tick(cap=cap, existing_jobs=existing_jobs) + submitted = primary.submitted_attempt_ids() & set(seeded_ids) + + # Invariant 1: per-source dispatched count + in-flight count ≤ cap. + per_source: dict[str, int] = {} + for aid in submitted | in_flight_ids: + src = seeded_source_for.get(aid) + if src is None: + continue + per_source[src] = per_source.get(src, 0) + 1 + if cap > 0: + for src, count in per_source.items(): + if count > cap: + violations += 1 + failed( + f"iter {it}: source {src} count={count} > cap={cap}", + f"submitted={sorted(submitted)} in_flight={sorted(in_flight_ids)}", + ) + + # Invariant 2: no two of (submitted ∪ in_flight) share a cc-pair. + seen_cc: dict[tuple[int | None, int | None, int], int] = {} + for aid in submitted | in_flight_ids: + key = seeded_cc_pair_for.get(aid) + if key is None: + continue + if key in seen_cc: + violations += 1 + failed( + f"iter {it}: cc-pair {key} held by both {seen_cc[key]} and {aid}" + ) + seen_cc[key] = aid + + if violations == 0: + passed(f"{iterations} iterations — invariants held every tick") + + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + + +def cleanup_test_data() -> None: + """Drop everything tagged with SCHEDULER_PREFIX, FK-safe order.""" + engine = get_sqlalchemy_engine() + with Session(engine) as db: + db.execute( + text( + """ + DELETE FROM index_attempt + WHERE connector_id IN ( + SELECT id FROM connector WHERE name LIKE :p + ) + """ + ), + {"p": f"{SCHEDULER_PREFIX}%"}, + ) + db.execute( + text( + """ + DELETE FROM permission_sync_run + WHERE cc_pair_id IN ( + SELECT id FROM connector_credential_pair WHERE name LIKE :p + ) + """ + ), + {"p": f"{SCHEDULER_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM connector_credential_pair WHERE name LIKE :p"), + {"p": f"{SCHEDULER_PREFIX}%"}, + ) + db.execute( + text("DELETE FROM connector WHERE name LIKE :p"), + {"p": f"{SCHEDULER_PREFIX}%"}, + ) + # credentials we created have no name tag — they have no FK to + # anything tagged after the cc-pair delete, so we leave them. + # The cc-pair / connector deletes are FK-bound so a residual + # credential row is harmless. + db.commit() + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +THIS_DIR = Path(__file__).resolve().parent + + +def precheck_indexer_not_running(force: bool) -> None: + result = subprocess.run( + ["pgrep", "-f", "danswer/background/update.py"], + capture_output=True, + text=True, + ) + pids = [p for p in result.stdout.strip().splitlines() if p] + if pids and not force: + sys.exit( + f"\nThe production indexer is running (pids: {pids}). It would race\n" + "on this script's seeded NOT_STARTED rows and clobber the assertions.\n" + "Stop it first:\n" + " pkill -f 'danswer/background/update.py'\n" + "Or re-run with --force-with-indexer-running to override (your\n" + "test attempts may then be picked up by the real scheduler before\n" + "this script gets to assert on them — expect noise).\n" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", action="store_true", help="Skip the destructive-op confirmation." + ) + parser.add_argument( + "--keep-data", + action="store_true", + help="Don't clean tagged data at the end (debugging).", + ) + parser.add_argument( + "--soak-iterations", + type=int, + default=20, + help="Number of randomized ticks for the soak phase. Default 20.", + ) + parser.add_argument( + "--force-with-indexer-running", + action="store_true", + help="Run even if the production indexer process is detected.", + ) + args = parser.parse_args() + + precheck_indexer_not_running(force=args.force_with_indexer_running) + + engine = get_sqlalchemy_engine() + safe_url = ( + f"{engine.url.drivername}://{engine.url.username}@" + f"{engine.url.host}:{engine.url.port}/{engine.url.database}" + ) + print(f"Target DB: {safe_url}") + if not args.yes: + ans = input( + "This will create + delete tagged rows under " + f"prefix '{SCHEDULER_PREFIX}'. Continue? [y/N] " + ) + if ans.strip().lower() not in ("y", "yes"): + print("Aborted.") + return 1 + + # Always clean before we start so a previous failed run can't pollute + # the per-source counts (e.g. a left-behind IN_PROGRESS slack row). + cleanup_test_data() + + # Each phase calls cleanup_test_data() before running so it starts + # against an empty tagged-row state. Without this, deferred + # NOT_STARTED rows from earlier phases would compete with later + # phases' attempts for cap slots and skew assertions. + phases = [ + phase_a_priority_under_cap, + phase_b_cap_leak_regression, + phase_c_cc_pair_guard_with_in_progress, + phase_d_different_sources_independent_caps, + phase_e_same_cc_pair_double_submit_prevented, + phase_g_completed_in_flight_does_not_block, + phase_h_failed_in_flight_does_not_block, + phase_i_in_progress_dispatched_not_double_counted, + lambda: phase_f_soak_random_ticks(args.soak_iterations), + ] + try: + for phase_fn in phases: + cleanup_test_data() + phase_fn() + finally: + if not args.keep_data: + cleanup_test_data() + print("\n ok cleaned up tagged rows") + else: + print(f"\n -- kept tagged rows (--keep-data); prefix={SCHEDULER_PREFIX}") + + if _FAILED: + print(f"\nFAIL: {_FAILED} assertion(s) failed") + return 1 + print("\nALL PHASES PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/tests/unit/danswer/background/__init__.py b/backend/tests/unit/danswer/background/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/tests/unit/danswer/background/test_indexing_scheduler.py b/backend/tests/unit/danswer/background/test_indexing_scheduler.py new file mode 100644 index 00000000000..93ff426ff8b --- /dev/null +++ b/backend/tests/unit/danswer/background/test_indexing_scheduler.py @@ -0,0 +1,707 @@ +"""Unit tests for the indexing scheduler's per-source cap and per-cc-pair +collision guard, including the cap-leak fix where attempts already +dispatched to Dask but still NOT_STARTED in the DB must be counted +against the cap. + +The scheduler decision logic is exercised through two pure helpers +exposed by ``danswer.background.update``: + + * ``_build_running_view`` — folds DB-IN_PROGRESS rows + already- + dispatched-but-pre-completion rows into the per-source counter + and per-cc-pair key set. + * ``_evaluate_dispatch_for_attempt`` — pure decision: dispatch or + defer (per-cc-pair collision / per-source cap). + +We verify: + 1. Priority sort is honored when the scheduler iterates candidates. + 2. Per-source cap is enforced. + 3. Per-cc-pair guard blocks a same-cc-pair Re-Index collision. + 4. THE BUG FIX: a dispatched-but-not-yet-IN_PROGRESS attempt counts + against the cap; this prevents the cap leak across scheduler ticks. + 5. Same-tick fairness: two same-source NOT_STARTED rows in one tick + don't both get dispatched if cap=1. + 6. Same-tick fairness for same-cc-pair NOT_STARTED rows. + +Plus a randomized fuzz test that runs the scheduler many ticks against +random NOT_STARTED queues, simulating Dask delays, worker crashes, and +priority bumps. The invariant: across all ticks, the number of attempts +running concurrently for any source is never above ``PER_SOURCE_CAP``, +and a same-cc-pair pair never runs two attempts concurrently. +""" +from __future__ import annotations + +import random +import unittest +from dataclasses import dataclass +from dataclasses import field +from enum import Enum + +from danswer.background.update import _build_running_view +from danswer.background.update import _DEFER_CC_PAIR +from danswer.background.update import _DEFER_SOURCE_CAP +from danswer.background.update import _DISPATCH +from danswer.background.update import _evaluate_dispatch_for_attempt + + +class _Source(str, Enum): + SLACK = "slack" + GITHUB = "github" + CONFLUENCE = "confluence" + SALESFORCE = "salesforce" + + +@dataclass +class _FakeConnector: + name: str + source: _Source + + +@dataclass +class _FakeAttempt: + """Minimal stand-in for ``IndexAttempt`` used by the helpers under + test. The real ORM object exposes the same attributes the helpers + touch (``id``, ``connector``, ``connector_id``, ``credential_id``, + ``embedding_model_id``, ``indexing_priority``).""" + + id: int + connector_id: int | None + credential_id: int | None + embedding_model_id: int + indexing_priority: int = 0 + connector: _FakeConnector | None = None + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _attempt( + *, + id: int, + source: _Source = _Source.SLACK, + connector_id: int | None = None, + credential_id: int = 1, + embedding_model_id: int = 2, + indexing_priority: int = 0, + name: str | None = None, +) -> _FakeAttempt: + """Construct a fake attempt with sensible defaults. + + ``connector_id`` defaults to ``id`` so each attempt sits on its own + cc-pair unless the caller overrides it. + """ + cid = id if connector_id is None else connector_id + return _FakeAttempt( + id=id, + connector_id=cid, + credential_id=credential_id, + embedding_model_id=embedding_model_id, + indexing_priority=indexing_priority, + connector=_FakeConnector( + name=name or f"conn-{cid}-{source.value}", source=source + ), + ) + + +def _simulate_tick( + candidates: list[_FakeAttempt], + in_progress: list[_FakeAttempt], + dispatched_pre_completion: list[_FakeAttempt], + per_source_cap: int, +) -> tuple[list[int], list[tuple[int, str]]]: + """Run one scheduler tick using only the public helpers. + + Returns a tuple ``(dispatched_ids, deferred)`` where ``deferred`` is + a list of ``(attempt_id, reason)`` tuples and ``reason`` is one of + the helper module's ``_DEFER_*`` sentinels. + """ + running_per_source, in_progress_cc_pair_keys = _build_running_view( + in_progress, dispatched_pre_completion, per_source_cap + ) + dispatched: list[int] = [] + deferred: list[tuple[int, str]] = [] + for attempt in candidates: + decision = _evaluate_dispatch_for_attempt( + attempt, + running_per_source, + in_progress_cc_pair_keys, + per_source_cap, + ) + if decision == _DISPATCH: + dispatched.append(attempt.id) + else: + deferred.append((attempt.id, decision)) + return dispatched, deferred + + +# --------------------------------------------------------------------------- +# Targeted scenarios +# --------------------------------------------------------------------------- + + +class TestSchedulerDecisions(unittest.TestCase): + def test_priority_order_is_respected_under_cap(self) -> None: + """Cap=1 + 3 same-source candidates → highest-priority wins. + + Mirrors the user's reported scenario: 3 Slack connectors, one + priority-bumped to 20. The bumped attempt must be dispatched + first; the others defer. + """ + # Pre-sorted as the real `get_not_started_index_attempts` does: + # priority DESC, time_created ASC. + candidates = [ + _attempt(id=928, source=_Source.SLACK, indexing_priority=20), + _attempt(id=925, source=_Source.SLACK, indexing_priority=0), + _attempt(id=930, source=_Source.SLACK, indexing_priority=0), + ] + dispatched, deferred = _simulate_tick( + candidates, in_progress=[], dispatched_pre_completion=[], per_source_cap=1 + ) + self.assertEqual(dispatched, [928]) + self.assertEqual( + sorted(deferred), [(925, _DEFER_SOURCE_CAP), (930, _DEFER_SOURCE_CAP)] + ) + + def test_cap_zero_disables_per_source_limit(self) -> None: + candidates = [ + _attempt(id=1, source=_Source.SLACK), + _attempt(id=2, source=_Source.SLACK), + _attempt(id=3, source=_Source.SLACK), + ] + dispatched, deferred = _simulate_tick( + candidates, in_progress=[], dispatched_pre_completion=[], per_source_cap=0 + ) + self.assertEqual(dispatched, [1, 2, 3]) + self.assertEqual(deferred, []) + + def test_per_source_cap_2_allows_two_same_source(self) -> None: + candidates = [ + _attempt(id=1, source=_Source.SLACK), + _attempt(id=2, source=_Source.SLACK), + _attempt(id=3, source=_Source.SLACK), + ] + dispatched, deferred = _simulate_tick( + candidates, in_progress=[], dispatched_pre_completion=[], per_source_cap=2 + ) + self.assertEqual(dispatched, [1, 2]) + self.assertEqual(deferred, [(3, _DEFER_SOURCE_CAP)]) + + def test_different_sources_do_not_share_cap(self) -> None: + candidates = [ + _attempt(id=1, source=_Source.SLACK), + _attempt(id=2, source=_Source.GITHUB), + _attempt(id=3, source=_Source.CONFLUENCE), + _attempt(id=4, source=_Source.SALESFORCE), + ] + dispatched, _ = _simulate_tick( + candidates, in_progress=[], dispatched_pre_completion=[], per_source_cap=1 + ) + self.assertEqual(dispatched, [1, 2, 3, 4]) + + def test_per_cc_pair_guard_blocks_reindex_collision(self) -> None: + """Same cc-pair has an IN_PROGRESS attempt → re-index attempt + defers (no FAILED row, just NOT_STARTED waiting).""" + running = _attempt( + id=900, source=_Source.SLACK, connector_id=7, credential_id=5 + ) + reindex = _attempt( + id=940, source=_Source.SLACK, connector_id=7, credential_id=5 + ) + dispatched, deferred = _simulate_tick( + [reindex], + in_progress=[running], + dispatched_pre_completion=[], + per_source_cap=2, + ) + self.assertEqual(dispatched, []) + self.assertEqual(deferred, [(940, _DEFER_CC_PAIR)]) + + def test_per_cc_pair_guard_blocks_same_tick_collision(self) -> None: + """Two NOT_STARTED rows for the SAME cc-pair in one tick: only + the first dispatches; the second defers via the in-tick set.""" + a = _attempt(id=1, source=_Source.SLACK, connector_id=7, credential_id=5) + b = _attempt(id=2, source=_Source.SLACK, connector_id=7, credential_id=5) + dispatched, deferred = _simulate_tick( + [a, b], in_progress=[], dispatched_pre_completion=[], per_source_cap=2 + ) + self.assertEqual(dispatched, [1]) + self.assertEqual(deferred, [(2, _DEFER_CC_PAIR)]) + + # ---------------- THE BUG-FIX REGRESSION ------------------------------ + + def test_dispatched_pre_completion_counts_toward_source_cap(self) -> None: + """Bug fix: when an attempt is sitting in Dask's queue (in + ``existing_jobs``) but its DB row is still NOT_STARTED, it must + still hold its source-cap slot. + + Pre-fix: the DB-IN_PROGRESS query returned 0 slack rows; the + cap read 0 < 1; the next slack candidate leaked through and + Dask got two slack tasks. + """ + dispatched_but_unborn = _attempt(id=928, source=_Source.SLACK) + candidate = _attempt(id=925, source=_Source.SLACK) + dispatched, deferred = _simulate_tick( + [candidate], + in_progress=[], + dispatched_pre_completion=[dispatched_but_unborn], + per_source_cap=1, + ) + self.assertEqual(dispatched, []) + self.assertEqual(deferred, [(925, _DEFER_SOURCE_CAP)]) + + def test_dispatched_pre_completion_counts_for_cc_pair_guard(self) -> None: + """Same fix: a dispatched-but-not-IN_PROGRESS attempt also + protects its cc-pair against same-tick re-submission.""" + dispatched_attempt = _attempt( + id=900, source=_Source.SLACK, connector_id=7, credential_id=5 + ) + candidate = _attempt( + id=940, source=_Source.SLACK, connector_id=7, credential_id=5 + ) + dispatched, deferred = _simulate_tick( + [candidate], + in_progress=[], + dispatched_pre_completion=[dispatched_attempt], + per_source_cap=4, + ) + self.assertEqual(dispatched, []) + self.assertEqual(deferred, [(940, _DEFER_CC_PAIR)]) + + def test_no_double_count_when_attempt_in_both_lists(self) -> None: + """If a dispatched attempt has just flipped to IN_PROGRESS, it + may appear in both ``in_progress`` and the dispatched list. The + accounting must dedupe on attempt id.""" + a = _attempt(id=10, source=_Source.SLACK) + running_per_source, _ = _build_running_view([a], [a], per_source_cap=2) + self.assertEqual(running_per_source.get("slack", 0), 1) + + def test_connector_null_attempt_is_skipped_in_view(self) -> None: + """An attempt whose connector was deleted under us must not + crash accounting and must not consume a cap slot.""" + broken = _FakeAttempt( + id=99, + connector_id=None, + credential_id=None, + embedding_model_id=2, + connector=None, + ) + running_per_source, keys = _build_running_view([broken], [], per_source_cap=1) + self.assertEqual(running_per_source, {}) + self.assertEqual(keys, set()) + + def test_completed_attempt_not_in_dispatched_list(self) -> None: + """The caller is responsible for filtering SUCCESS/FAILED rows + out of the dispatched list (the production query does so). + Sanity check: when filtered out, they don't consume the cap.""" + # Caller filters before passing in; here we verify the helper + # treats whatever it receives as still-pre-completion. + candidates = [_attempt(id=1, source=_Source.SLACK)] + dispatched, _ = _simulate_tick( + candidates, + in_progress=[], + dispatched_pre_completion=[], # caller filtered out completed + per_source_cap=1, + ) + self.assertEqual(dispatched, [1]) + + +# --------------------------------------------------------------------------- +# Multi-tick stress / fuzz +# --------------------------------------------------------------------------- + + +@dataclass +class _SimState: + """Simulator for the cross-tick interaction between scheduler, + Dask queue, and DB IN_PROGRESS state. + + We model: + * NOT_STARTED queue (sorted by priority DESC, time_created ASC). + * Dispatched-pre-completion: attempts the scheduler has handed to + Dask but which haven't flipped to IN_PROGRESS yet (Dask queueing + delay or worker spinning up). + * IN_PROGRESS: attempts a worker has picked up. + * Done: attempts that finished. + + Each tick: + 1. Build the running view from `in_progress + dispatched`. + 2. For each NOT_STARTED candidate (in priority/FIFO order), the + scheduler decides dispatch / defer-cc-pair / defer-source-cap. + 3. Newly dispatched attempts move to `dispatched_pre_completion`. + 4. The simulator advances Dask state: each dispatched attempt + flips to IN_PROGRESS with `flip_prob`, simulating Dask actually + giving it to a worker. + 5. Each IN_PROGRESS attempt finishes with `finish_prob`. + 6. With probability `crash_prob`, an IN_PROGRESS attempt is + abruptly killed (worker crash) — it returns to NOT_STARTED, no + FAILED row (matching the production revert behaviour). + + Across all ticks, two invariants must hold: + A. For each source, at most `per_source_cap` attempts are + IN_PROGRESS simultaneously. + B. No two IN_PROGRESS attempts share the same cc-pair tuple. + """ + + candidates: list[_FakeAttempt] # NOT_STARTED queue + dispatched: list[_FakeAttempt] = field(default_factory=list) + in_progress: list[_FakeAttempt] = field(default_factory=list) + done: list[int] = field(default_factory=list) + per_source_cap: int = 1 + flip_prob: float = 0.7 + finish_prob: float = 0.4 + crash_prob: float = 0.05 + rng: random.Random = field(default_factory=random.Random) + + def _sort_candidates(self) -> None: + self.candidates.sort( + key=lambda a: (-a.indexing_priority, a.id) # FIFO via id as tiebreak + ) + + def step(self) -> None: + self._sort_candidates() + running_per_source, in_progress_cc_pair_keys = _build_running_view( + self.in_progress, self.dispatched, self.per_source_cap + ) + next_candidates: list[_FakeAttempt] = [] + for attempt in self.candidates: + decision = _evaluate_dispatch_for_attempt( + attempt, + running_per_source, + in_progress_cc_pair_keys, + self.per_source_cap, + ) + if decision == _DISPATCH: + self.dispatched.append(attempt) + else: + next_candidates.append(attempt) + self.candidates = next_candidates + + # Dask: dispatched → IN_PROGRESS. + still_dispatched: list[_FakeAttempt] = [] + for d in self.dispatched: + if self.rng.random() < self.flip_prob: + self.in_progress.append(d) + else: + still_dispatched.append(d) + self.dispatched = still_dispatched + + # Workers: IN_PROGRESS → done OR crash. + still_running: list[_FakeAttempt] = [] + for ip in self.in_progress: + r = self.rng.random() + if r < self.finish_prob: + self.done.append(ip.id) + elif r < self.finish_prob + self.crash_prob: + # Crash: revert to NOT_STARTED queue (matches the + # worker-side advisory-lock revert path). + self.candidates.append(ip) + else: + still_running.append(ip) + self.in_progress = still_running + + def assert_invariants(self) -> None: + per_source: dict[str, int] = {} + cc_pair_keys: set[tuple[int | None, int | None, int]] = set() + for ip in self.in_progress: + assert ip.connector is not None + key = (ip.connector_id, ip.credential_id, ip.embedding_model_id) + assert ( + key not in cc_pair_keys + ), f"two IN_PROGRESS attempts share cc-pair {key}" + cc_pair_keys.add(key) + per_source[ip.connector.source.value] = ( + per_source.get(ip.connector.source.value, 0) + 1 + ) + if self.per_source_cap > 0: + for src, count in per_source.items(): + assert count <= self.per_source_cap, ( + f"per-source cap leaked for {src}: " + f"{count} IN_PROGRESS > cap={self.per_source_cap}" + ) + + +class TestSchedulerStress(unittest.TestCase): + """Fuzz tests. The randomized seed is cycled so failures are + reproducible: each iteration prints its seed if the assertion + trips, and the test runs N iterations.""" + + NUM_ITERATIONS = 200 + + def _build_random_queue( + self, rng: random.Random, n: int, sources: list[_Source] + ) -> list[_FakeAttempt]: + # cc-pairs are (connector_id, credential_id, embedding_model_id). + # We pick connector_id from a small pool so collisions actually + # happen (the user's bug had multiple Slack cc-pairs sharing the + # same credential). + connector_pool = list(range(1, 8)) + attempts: list[_FakeAttempt] = [] + for i in range(n): + src = rng.choice(sources) + attempts.append( + _attempt( + id=i + 1, + source=src, + connector_id=rng.choice(connector_pool), + credential_id=rng.choice([1, 2, 3]), + indexing_priority=rng.choice([0, 0, 0, 10, 20, 30]), + ) + ) + return attempts + + def test_fuzz_invariants_hold_across_random_ticks(self) -> None: + """Run NUM_ITERATIONS independent simulations and assert that + invariants A (per-source cap) and B (cc-pair uniqueness) hold + every tick. Each simulation gets a fresh seed so failures are + reproducible.""" + sources = list(_Source) + for seed in range(self.NUM_ITERATIONS): + rng = random.Random(seed) + cap = rng.choice([1, 1, 1, 2, 3]) # cap=1 weighted (real default) + n_attempts = rng.randint(5, 30) + ticks = rng.randint(20, 60) + + sim = _SimState( + candidates=self._build_random_queue(rng, n_attempts, sources), + per_source_cap=cap, + flip_prob=rng.uniform(0.4, 0.9), + finish_prob=rng.uniform(0.2, 0.6), + crash_prob=rng.uniform(0.0, 0.1), + rng=rng, + ) + + for tick in range(ticks): + try: + sim.step() + sim.assert_invariants() + except AssertionError as e: + self.fail( + f"invariant violated: seed={seed} tick={tick} " + f"cap={cap} n={n_attempts}: {e}" + ) + + def test_priority_strict_order_under_cap_one(self) -> None: + """Targeted: with cap=1 and one source bucket, attempts MUST + finish in priority-DESC, FIFO order. No lower-priority can + sneak ahead via the cap-leak path.""" + for seed in range(self.NUM_ITERATIONS): + rng = random.Random(seed) + # Build a fixed-source queue so cap=1 fully serializes it. + n = rng.randint(4, 12) + attempts: list[_FakeAttempt] = [] + for i in range(n): + attempts.append( + _attempt( + id=i + 1, + source=_Source.SLACK, + connector_id=i + 1, # distinct cc-pairs + indexing_priority=rng.choice([0, 5, 10, 20, 30]), + ) + ) + sim = _SimState( + candidates=list(attempts), + per_source_cap=1, + flip_prob=rng.uniform(0.5, 0.9), + finish_prob=rng.uniform(0.3, 0.7), + crash_prob=0.0, # no crashes — we want clean priority order + rng=rng, + ) + # Run until everything completes. + max_ticks = 500 + for _ in range(max_ticks): + sim.step() + if not sim.candidates and not sim.dispatched and not sim.in_progress: + break + else: + self.fail(f"sim did not drain in {max_ticks} ticks (seed={seed})") + + expected_order = sorted( + attempts, key=lambda a: (-a.indexing_priority, a.id) + ) + expected_ids = [a.id for a in expected_order] + self.assertEqual( + sim.done, + expected_ids, + f"priority order violated (seed={seed}): " + f"got {sim.done}, expected {expected_ids}", + ) + + def test_user_reported_scenario(self) -> None: + """Exact replay of the user-observed setup: + + * 7 Slack connectors (cc-pairs). + * One has its attempt priority bumped to 20 (the rest at 0). + * cap = 1. + * Two Dask workers (so the cap-leak bug is exercisable). + + Pre-fix: the leak path could let a prio=0 attempt run while + the prio=20 attempt sat NOT_STARTED. With the fix, no prio=0 + attempt may finish before the prio=20 one. + """ + slack_attempts = [ + _attempt(id=i + 1, source=_Source.SLACK, connector_id=i + 1) + for i in range(7) + ] + # Bump #4 to priority 20. + slack_attempts[3].indexing_priority = 20 + bumped_id = slack_attempts[3].id + + for seed in range(self.NUM_ITERATIONS): + rng = random.Random(seed) + sim = _SimState( + candidates=[ + _FakeAttempt( + id=a.id, + connector_id=a.connector_id, + credential_id=a.credential_id, + embedding_model_id=a.embedding_model_id, + indexing_priority=a.indexing_priority, + connector=a.connector, + ) + for a in slack_attempts + ], + per_source_cap=1, + # Skew flip_prob low (~ Dask queue delay) to maximally + # exercise the cap-leak path. + flip_prob=rng.uniform(0.2, 0.6), + finish_prob=rng.uniform(0.3, 0.6), + crash_prob=0.0, + rng=rng, + ) + max_ticks = 500 + for _ in range(max_ticks): + sim.step() + if not sim.candidates and not sim.dispatched and not sim.in_progress: + break + else: + self.fail(f"sim did not drain (seed={seed})") + + # The bumped attempt must finish FIRST. + self.assertEqual( + sim.done[0], + bumped_id, + f"priority-20 attempt did not finish first (seed={seed}): " + f"order was {sim.done}", + ) + + def test_buggy_view_without_fix_leaks_cap_regression_guard(self) -> None: + """Regression guard: simulate the pre-fix codepath (build view + from IN_PROGRESS only, ignoring dispatched-but-pre-completion) + and assert the invariants DO get violated. This proves the test + harness is sensitive enough to catch a regression — if someone + removes the dispatched-row accounting from + ``_build_running_view``, the targeted scenario test above would + start failing. + """ + + def _buggy_view( + in_progress: list[_FakeAttempt], + _dispatched_ignored: list[_FakeAttempt], + per_source_cap: int, + ) -> tuple[dict[str, int], set[tuple[int | None, int | None, int]]]: + # Pre-fix accounting: only DB IN_PROGRESS rows count. + return _build_running_view(in_progress, [], per_source_cap) + + observed_concurrent_violation = False + slack_attempts = [ + _attempt(id=i + 1, source=_Source.SLACK, connector_id=i + 1) + for i in range(7) + ] + slack_attempts[3].indexing_priority = 20 + + # Use a fixed seed where Dask flip_prob is low enough that a + # dispatched attempt sits queued for at least one tick (the + # exact window the cap-leak exploits). + rng = random.Random(42) + candidates = [ + _FakeAttempt( + id=a.id, + connector_id=a.connector_id, + credential_id=a.credential_id, + embedding_model_id=a.embedding_model_id, + indexing_priority=a.indexing_priority, + connector=a.connector, + ) + for a in slack_attempts + ] + in_progress: list[_FakeAttempt] = [] + dispatched: list[_FakeAttempt] = [] + cap = 1 + for _ in range(80): + running_per_source, in_progress_cc_pair_keys = _buggy_view( + in_progress, dispatched, cap + ) + next_candidates: list[_FakeAttempt] = [] + for attempt in sorted( + candidates, key=lambda a: (-a.indexing_priority, a.id) + ): + decision = _evaluate_dispatch_for_attempt( + attempt, + running_per_source, + in_progress_cc_pair_keys, + cap, + ) + if decision == _DISPATCH: + dispatched.append(attempt) + else: + next_candidates.append(attempt) + candidates = next_candidates + still_dispatched: list[_FakeAttempt] = [] + for d in dispatched: + if rng.random() < 0.3: # low flip_prob = real Dask delay + in_progress.append(d) + else: + still_dispatched.append(d) + dispatched = still_dispatched + + # Check the invariant. + per_source: dict[str, int] = {} + for ip in in_progress: + assert ip.connector is not None + per_source[ip.connector.source.value] = ( + per_source.get(ip.connector.source.value, 0) + 1 + ) + for src, count in per_source.items(): + if cap > 0 and count > cap: + observed_concurrent_violation = True + + # Drain finished attempts so the simulation makes progress. + in_progress = [ip for ip in in_progress if rng.random() > 0.4] + + self.assertTrue( + observed_concurrent_violation, + "Pre-fix simulation did not produce a cap leak — fuzz " + "harness may have lost sensitivity. Tighten flip_prob/seed " + "or extend tick count.", + ) + + def test_crash_recovery_does_not_leak_cap(self) -> None: + """High crash rate (workers dying mid-run) must not leak the + per-source cap. The crash path returns the attempt to the + NOT_STARTED queue without inflating the cap counter.""" + for seed in range(self.NUM_ITERATIONS): + rng = random.Random(seed) + n = rng.randint(8, 20) + attempts = self._build_random_queue(rng, n, list(_Source)) + sim = _SimState( + candidates=attempts, + per_source_cap=rng.choice([1, 2]), + flip_prob=rng.uniform(0.4, 0.8), + finish_prob=rng.uniform(0.2, 0.5), + crash_prob=rng.uniform(0.1, 0.3), # crashy + rng=rng, + ) + for tick in range(80): + try: + sim.step() + sim.assert_invariants() + except AssertionError as e: + self.fail( + f"invariant violated under crashy workers: " + f"seed={seed} tick={tick}: {e}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/unit/danswer/db/__init__.py b/backend/tests/unit/danswer/db/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/tests/unit/danswer/db/test_deletion_lock_keys.py b/backend/tests/unit/danswer/db/test_deletion_lock_keys.py new file mode 100644 index 00000000000..8c5155ee75e --- /dev/null +++ b/backend/tests/unit/danswer/db/test_deletion_lock_keys.py @@ -0,0 +1,95 @@ +"""Unit tests for the per-cc-pair deletion advisory-lock key derivation +(``danswer.db.connector_credential_pair._deletion_lock_key``). + +Lock acquisition itself requires a real Postgres ``pg_try_advisory_lock`` +and is exercised by the scheduler/deletion e2e — here we only verify the +pure key math: stable, deterministic, namespaced away from the indexing +lock, and within Postgres ``bigint`` range. +""" +from __future__ import annotations + +import unittest + +from danswer.db.connector_credential_pair import _deletion_lock_key +from danswer.db.connector_credential_pair import _DELETION_LOCK_KEY_OFFSET +from danswer.db.index_attempt import _cc_pair_lock_key +from danswer.db.index_attempt import _INDEXING_LOCK_KEY_OFFSET + + +_PG_BIGINT_MIN = -(1 << 63) +_PG_BIGINT_MAX = (1 << 63) - 1 + + +class TestDeletionLockKey(unittest.TestCase): + def test_namespace_distinct_from_indexing_lock(self) -> None: + """Deletion namespace prefix must NOT equal indexing prefix. + + If they collided, an active indexing run would block a deletion + sweep (and vice versa) for *unrelated* cc-pairs whose hashes + happen to match. The whole point of separate prefixes is that + the two locks live in disjoint regions of the 64-bit key space. + """ + self.assertNotEqual(_DELETION_LOCK_KEY_OFFSET, _INDEXING_LOCK_KEY_OFFSET) + + def test_same_cc_pair_distinct_keys_across_namespaces(self) -> None: + """For the same (connector_id, credential_id), the indexing lock + and the deletion lock must produce different keys — otherwise + an indexing run on cc-pair X would block deletion of cc-pair X + without us ever holding the deletion lock.""" + for cid, credid in [(1, 1), (7, 5), (111, 5), (12345, 67890)]: + idx_key = _cc_pair_lock_key(cid, credid) + del_key = _deletion_lock_key(cid, credid) + self.assertNotEqual( + idx_key, + del_key, + f"keys collided for cc-pair=({cid},{credid}): " + f"indexing={idx_key} deletion={del_key}", + ) + + def test_deterministic(self) -> None: + """Same inputs → same key, every call. No randomness, no + process-state dependency.""" + for cid, credid in [(0, 0), (1, 2), (999, 1), (12345, 67890)]: + self.assertEqual( + _deletion_lock_key(cid, credid), + _deletion_lock_key(cid, credid), + ) + + def test_distinct_cc_pairs_distinct_keys(self) -> None: + """Different cc-pairs should produce different keys (collisions + possible at the 32-bit hash level — birthday bound at ~65k + cc-pairs is below 1% — but a small smoke set must not collide).""" + seen: dict[int, tuple[int, int]] = {} + for cid in range(1, 25): + for credid in range(1, 25): + k = _deletion_lock_key(cid, credid) + if k in seen: + self.fail( + f"hash collision in small key space: " + f"({cid},{credid}) and {seen[k]} both → {k}" + ) + seen[k] = (cid, credid) + + def test_within_postgres_bigint_range(self) -> None: + """Postgres advisory-lock keys are signed 64-bit (bigint). + Anything outside that range raises a runtime error from the + driver. Sweep a large input space to catch any sign-extension + or wraparound bug in the key math.""" + for cid in range(1, 5000, 137): + for credid in range(1, 5000, 211): + k = _deletion_lock_key(cid, credid) + self.assertGreaterEqual(k, _PG_BIGINT_MIN) + self.assertLessEqual(k, _PG_BIGINT_MAX) + + def test_namespace_offset_in_high_bits(self) -> None: + """The high 32 bits of the key (before sign-extension) should + reflect the deletion namespace (b\"DELE\").""" + # Pick an input whose hash low bits are 0 so the high bits are + # the offset bits unmodified. + # The hash is `(cid * 0x9E3779B1 ^ credid) & 0xFFFFFFFF`. With + # cid=0, credid=0 the hash is 0 → key == offset. + self.assertEqual(_deletion_lock_key(0, 0), _DELETION_LOCK_KEY_OFFSET) + + +if __name__ == "__main__": + unittest.main() diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml index 99e8bb14573..cfe74aa4f7c 100644 --- a/deployment/docker_compose/docker-compose.dev.yml +++ b/deployment/docker_compose/docker-compose.dev.yml @@ -350,7 +350,6 @@ services: command: > /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev" - volumes: db_volume: diff --git a/deployment/kubernetes/analytics-bootstrap-job.yaml b/deployment/kubernetes/analytics-bootstrap-job.yaml new file mode 100644 index 00000000000..e3ff12d2d2e --- /dev/null +++ b/deployment/kubernetes/analytics-bootstrap-job.yaml @@ -0,0 +1,142 @@ +# One-time Kubernetes Job for the Darwin analytics rollup bootstrap. +# +# What it does (in order): +# 1. `alembic upgrade heads` — applies any pending migrations, +# including this PR's `c8a4e2f9d1b3_analytics_daily_rollup`. Idempotent: +# already-applied revisions are skipped. Safe to re-run. +# 2. `scripts/backfill_analytics_rollup.py` — walks every historical +# date that still has chat data, computes the daily metrics, and +# writes them into `analytics_daily_rollup` (also seeds the +# `analytics_rollup_state` checkpoint in `key_value_store`). +# Idempotent via INSERT…ON CONFLICT(date) DO UPDATE. +# +# Why a Job and not a Deployment / Pod: +# - Deployment auto-restarts on container exit — wrong for one-time +# work; the migration would loop. +# - Bare Pod doesn't track success / failure cleanly. +# - Job has run-to-completion semantics + retry-on-failure + +# TTL-after-finish for auto-cleanup. Standard K8s pattern. +# +# When to apply: +# - ONCE, after the new backend image (with this PR's code) is rolled +# out to the api-server and background-deployment, and BEFORE the +# next 08:00 UTC retention sweep on a fresh DB. If retention runs +# first, it deletes chat data older than 30 days and the backfill +# will then write zero counts for those days. +# - Re-applying is safe (both steps are idempotent), but normally +# unnecessary — the daily Celery beat task takes over. +# +# How to apply: +# 1. Update IMAGE_TAG to the tag containing the merged PR code +# (currently the api-server runs vha-119; replace as needed). +# 2. kubectl apply -f deployment/kubernetes/analytics-bootstrap-job.yaml +# 3. Watch logs: +# kubectl logs -n darwin -f job/darwin-analytics-bootstrap +# 4. Verify completion: +# kubectl get -n darwin job/darwin-analytics-bootstrap +# # COMPLETIONS should read 1/1 +# 5. Verify the rollup table: +# kubectl exec -n darwin -- psql ... \ +# -c "SELECT count(*), max(rolled_up_at) FROM analytics_daily_rollup;" +# +# How to clean up: nothing required — `ttlSecondsAfterFinished: 3600` +# auto-deletes the Job and its Pod 1 hour after success. Manual delete: +# kubectl delete -n darwin job/darwin-analytics-bootstrap +# +# Behaviour on failure: `backoffLimit: 3` retries up to 3 times +# (with exponential backoff). After that the Job is marked Failed and +# you can inspect logs of the failed Pod via `kubectl logs ...`. +apiVersion: batch/v1 +kind: Job +metadata: + name: darwin-analytics-bootstrap + namespace: darwin + labels: + app: darwin-analytics-bootstrap + purpose: one-time-migration +spec: + # Auto-cleanup the Job + its completed Pod 1 hour after success. + # Tune higher if you want more time to inspect logs. + ttlSecondsAfterFinished: 3600 + # Retry the whole pipeline up to 3 times on failure (each step is + # idempotent so retries are safe). 4xx/5xx pods are inspectable until + # ttlSecondsAfterFinished kicks in. + backoffLimit: 3 + # Hard kill if the Job runs longer than 30 minutes — backfill on a + # large chat history can take a few minutes; 30m is a generous ceiling. + activeDeadlineSeconds: 1800 + template: + metadata: + labels: + app: darwin-analytics-bootstrap + spec: + # OnFailure → if the container exits non-zero, kubelet restarts + # it within the same Pod (faster than scheduling a new Pod). + # Combined with backoffLimit above for the cross-Pod retry. + restartPolicy: OnFailure + containers: + - name: bootstrap + # IMPORTANT: bump this to the image tag that includes this PR's + # backend code (the analytics_rollup module + new migration). + # The api-server deployment is currently on vha-119; you'll + # likely roll a vha-120 (or similar) once the PR merges. + image: sfbrdevhelmweacr.azurecr.io/danswer/danswer-backend:vha-121 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + # `pipefail` isn't supported by the image's /bin/sh + # (BusyBox ash / dash). The script has no pipes anyway, + # so plain `-eu` is sufficient: any non-zero exit aborts. + set -eu + echo "=== Step 1/2: alembic upgrade heads ===" + alembic upgrade heads + echo + echo "=== Step 2/2: backfill analytics_daily_rollup ===" + # PYTHONPATH=. is needed because the script imports + # `danswer.*` and the image's WORKDIR is the backend/ dir. + PYTHONPATH=. python scripts/backfill_analytics_rollup.py + echo + echo "=== Bootstrap complete ===" + # Same Postgres creds as the api-server / background pods. + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: postgres_user + name: danswer-secrets + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: postgres_password + name: danswer-secrets + # Same shared config as the api-server. The backfill reads + # POSTGRES_HOST, encryption keys, etc. from here. + envFrom: + - configMapRef: + name: env-configmap + # PVCs match the api-server. Strictly speaking the backfill + # doesn't write to either, but mirroring the api-server config + # avoids surprises if anything in the import chain reads from + # /home/storage or /home/file_connector_storage. + volumeMounts: + - mountPath: /home/storage + name: dynamic-storage + - mountPath: /home/file_connector_storage + name: file-connector-storage + # Modest resource ask — backfill is mostly DB I/O. + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" + volumes: + - name: dynamic-storage + persistentVolumeClaim: + claimName: dynamic-pvc + - name: file-connector-storage + persistentVolumeClaim: + claimName: file-connector-pvc diff --git a/deployment/kubernetes/env-configmap.yaml b/deployment/kubernetes/env-configmap.yaml index 411266a7874..ebfcc9deb81 100644 --- a/deployment/kubernetes/env-configmap.yaml +++ b/deployment/kubernetes/env-configmap.yaml @@ -54,6 +54,11 @@ data: MIN_THREADS_ML_MODELS: "" # Indexing Configs NUM_INDEXING_WORKERS: "" + # Per-DocumentSource concurrency cap when NUM_INDEXING_WORKERS > 1. + # Default 1 = at most one indexing attempt per source type at a time + # (prevents a single PAT/credential from getting rate-limited). + # 0 = uncapped. Enforced scheduler-side in update.py. + INDEXING_PER_SOURCE_CAP: "" ENABLED_CONNECTOR_TYPES: "" DISABLE_INDEX_UPDATE_ON_SWAP: "" DASK_JOB_CLIENT_ENABLED: "" @@ -64,6 +69,22 @@ data: WEB_CONNECTOR_VALIDATE_URLS: "" GONG_CONNECTOR_START_TIME: "" NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP: "" + # DB Retention (daily Celery beat at 08:00 UTC; backend/danswer/db/retention.py). + # All defaults are sensible — override only when you need a tighter window. + RETENTION_DAYS_KOMBU: "" # default 7 (Celery broker queue) + RETENTION_DAYS_TASK_QUEUE: "" # default 30 (terminal task_queue_jobs only) + RETENTION_DAYS_INDEX_ATTEMPT: "" # default 0 = disabled (opt-in to keep history) + RETENTION_KEEP_LAST_N_INDEX_ATTEMPTS: "" # default 20 per (cc-pair, embedding model) + RETENTION_DAYS_CHAT: "" # default 30 (chat_session + chat_message + LO blobs) + RETENTION_DAYS_PERMISSION_SYNC: "" # default 30 (terminal permission_sync_run only) + RETENTION_DAYS_USAGE_REPORTS: "" # default 90 (usage_reports + file_store + LO blobs) + RETENTION_BATCH_SIZE: "" # default 5000 rows per DELETE + RETENTION_MAX_BATCHES: "" # default 200 batches per policy per run + # Analytics rollup (daily Celery beat at 07:30 UTC, 30 min before retention; + # backend/danswer/db/analytics_rollup.py). The lookback is the late-feedback + # grace period — MUST be < RETENTION_DAYS_CHAT to avoid recomputing days + # whose source rows have already been deleted. + ANALYTICS_LATE_FEEDBACK_BUFFER_DAYS: "" # default 2 # DanswerBot SlackBot Configs DANSWER_BOT_SLACK_APP_TOKEN: "" DANSWER_BOT_SLACK_BOT_TOKEN: "" diff --git a/web/package-lock.json b/web/package-lock.json index 0e19baab4e9..b61c964e258 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -28,7 +28,7 @@ "js-cookie": "^3.0.5", "lodash": "^4.17.21", "mdast-util-find-and-replace": "^3.0.1", - "next": "^14.2.3", + "next": "^14.2.35", "npm": "^10.8.0", "postcss": "^8.4.31", "prismjs": "^1.29.0", @@ -51,7 +51,7 @@ "devDependencies": { "@tailwindcss/typography": "^0.5.10", "eslint": "^8.48.0", - "eslint-config-next": "^14.1.0", + "eslint-config-next": "^14.2.35", "prettier": "2.8.8" } }, @@ -70,7 +70,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" @@ -80,12 +79,14 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.24.2", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.2.tgz", - "integrity": "sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "license": "MIT", "dependencies": { - "@babel/highlight": "^7.24.2", - "picocolors": "^1.0.0" + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" @@ -95,7 +96,6 @@ "version": "7.24.4", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.4.tgz", "integrity": "sha512-vg8Gih2MLK+kOkHJp4gBEIkyaIi00jgWot2D9QOmmfLC8jINSOzmCLta6Bvz/JSBCqnegV0L80jhxkol5GWNfQ==", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -134,7 +134,6 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "peer": true, "bin": { "semver": "bin/semver.js" } @@ -168,7 +167,6 @@ "version": "7.23.6", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.23.6.tgz", "integrity": "sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==", - "peer": true, "dependencies": { "@babel/compat-data": "^7.23.5", "@babel/helper-validator-option": "^7.23.5", @@ -184,7 +182,6 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "peer": true, "dependencies": { "yallist": "^3.0.2" } @@ -193,7 +190,6 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "peer": true, "bin": { "semver": "bin/semver.js" } @@ -244,7 +240,6 @@ "version": "7.24.5", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.5.tgz", "integrity": "sha512-9GxeY8c2d2mdQUP1Dye0ks3VDyIMS98kt/llQ2nUId8IsWqTF0l1LkSX0/uP7l7MCDrzXS009Hyhe2gzTiGW8A==", - "peer": true, "dependencies": { "@babel/helper-environment-visitor": "^7.22.20", "@babel/helper-module-imports": "^7.24.3", @@ -271,7 +266,6 @@ "version": "7.24.5", "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.5.tgz", "integrity": "sha512-uH3Hmf5q5n7n8mz7arjUlDOCbttY/DW4DYhE6FUsjKJ/oYC1kQQUvwEQWxRwUpX9qQKRXeqLwWxrqilMrf32sQ==", - "peer": true, "dependencies": { "@babel/types": "^7.24.5" }, @@ -291,17 +285,19 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.1.tgz", - "integrity": "sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.5.tgz", - "integrity": "sha512-3q93SSKX2TWCG30M2G2kwaKeTYgEUp5Snjuj8qm729SObL6nbtUldAi37qbxkD5gg3xnBio+f9nqpSepGZMvxA==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", "engines": { "node": ">=6.9.0" } @@ -310,107 +306,31 @@ "version": "7.23.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz", "integrity": "sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==", - "peer": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helpers": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.5.tgz", - "integrity": "sha512-CiQmBMMpMQHwM5m01YnrM6imUG1ebgYJ+fAIW4FZe6m4qHTPaRHti+R8cggAwkdz4oXhtO4/K9JWlh+8hIfR2Q==", - "peer": true, - "dependencies": { - "@babel/template": "^7.24.0", - "@babel/traverse": "^7.24.5", - "@babel/types": "^7.24.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.5.tgz", - "integrity": "sha512-8lLmua6AVh/8SLJRRVD6V8p73Hir9w5mJrhE+IPpILG31KKlI9iz5zmBYKcWPS59qSfgP9RaSBQSHHE81WKuEw==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", + "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", + "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.24.5", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "node_modules/@babel/parser": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", + "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "license": "MIT", "dependencies": { - "has-flag": "^3.0.0" + "@babel/types": "^7.29.0" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/parser": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.5.tgz", - "integrity": "sha512-EOv5IK8arwh3LI47dz1b0tKUb/1uhHAnHJOrjgtQMIpu1uXd9mlFrJg9IUgGUgZ41Ch0K8REPTYpO7B76b4vJg==", "bin": { "parser": "bin/babel-parser.js" }, @@ -433,24 +353,23 @@ } }, "node_modules/@babel/runtime": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.5.tgz", - "integrity": "sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==", - "dependencies": { - "regenerator-runtime": "^0.14.0" - }, + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/template": { - "version": "7.24.0", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", - "integrity": "sha512-Bkf2q8lMB0AFpX0NFEqSbx1OkTHf0f+0j82mkw+ZpzBnkk7e9Ql0891vlfgi+kHwOk8tQjiQHpqh4LaSa0fKEA==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.23.5", - "@babel/parser": "^7.24.0", - "@babel/types": "^7.24.0" + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" }, "engines": { "node": ">=6.9.0" @@ -485,13 +404,13 @@ } }, "node_modules/@babel/types": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.5.tgz", - "integrity": "sha512-6mQNsaLeXTw0nxYUYu+NSa4Hx4BlF1x1x8/PMFbiR+GBSr+2DkECc69b8hgy2frEodNcvPffeH8YfWd3LI6jhQ==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.24.1", - "@babel/helper-validator-identifier": "^7.24.5", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" @@ -512,6 +431,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.1.0.tgz", "integrity": "sha512-J3cQBClB4TVxwGo3KEjssGEXNJqGVWx17aRTZ1ob0FliR5IjYgTxl5YJbKTzA6IzrtelotH19v6y7uoIRUZPSg==", + "peer": true, "dependencies": { "@dnd-kit/accessibility": "^3.1.0", "@dnd-kit/utilities": "^3.2.2", @@ -583,25 +503,30 @@ "integrity": "sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg==" }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, + "license": "MIT", "dependencies": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, + "funding": { + "url": "https://opencollective.com/eslint" + }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", - "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, + "license": "MIT", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } @@ -762,6 +687,7 @@ "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -775,9 +701,10 @@ } }, "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -786,11 +713,12 @@ } }, "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", "dependencies": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^6.2.2" }, "engines": { "node": ">=12" @@ -843,26 +771,98 @@ } }, "node_modules/@next/env": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz", - "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA==" + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.35.tgz", + "integrity": "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==", + "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.3.tgz", - "integrity": "sha512-L3oDricIIjgj1AVnRdRor21gI7mShlSwU/1ZGHmqM3LzHhXXhdkrfeNY5zif25Bi5Dd7fiJHsbhoZCHfXYvlAw==", + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.35.tgz", + "integrity": "sha512-Jw9A3ICz2183qSsqwi7fgq4SBPiNfmOLmTPXKvlnzstUwyvBrtySiY+8RXJweNAs9KThb1+bYhZh9XWcNOr2zQ==", "dev": true, + "license": "MIT", "dependencies": { "glob": "10.3.10" } }, + "node_modules/@next/eslint-plugin-next/node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/glob": { + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@next/swc-darwin-arm64": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz", - "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.33.tgz", + "integrity": "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "darwin" @@ -872,12 +872,13 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz", - "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.33.tgz", + "integrity": "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "darwin" @@ -887,12 +888,13 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz", - "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.33.tgz", + "integrity": "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -902,12 +904,13 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz", - "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.33.tgz", + "integrity": "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -917,12 +920,13 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz", - "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.33.tgz", + "integrity": "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -932,12 +936,13 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz", - "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.33.tgz", + "integrity": "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -947,12 +952,13 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz", - "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.33.tgz", + "integrity": "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -962,12 +968,13 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz", - "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", + "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", "cpu": [ "ia32" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -977,12 +984,13 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz", - "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.33.tgz", + "integrity": "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -1039,6 +1047,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", "optional": true, "engines": { "node": ">=14" @@ -1560,21 +1569,31 @@ "@babel/runtime": "^7.13.10" } }, + "node_modules/@rtsao/scc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", + "integrity": "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==", + "dev": true, + "license": "MIT" + }, "node_modules/@rushstack/eslint-patch": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.3.tgz", - "integrity": "sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==", - "dev": true + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.16.1.tgz", + "integrity": "sha512-TvZbIpeKqGQQ7X0zSCvPH9riMSFQFSggnfBjFZ1mEoILW+UuXCKwOoPcgjMwiUtRqFZ8jWhPJc4um14vC6I4ag==", + "dev": true, + "license": "MIT" }, "node_modules/@swc/counter": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==" + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", + "license": "Apache-2.0" }, "node_modules/@swc/helpers": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz", "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==", + "license": "Apache-2.0", "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" @@ -1779,6 +1798,7 @@ "version": "18.0.32", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.32.tgz", "integrity": "sha512-gYGXdtPQ9Cj0w2Fwqg5/ak6BcK3Z15YgjSqtyDizWUfx7mQ8drs0NBUzRRsAdoFVTO8kJ8L2TL8Skm7OFPnLUw==", + "peer": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -1789,6 +1809,7 @@ "version": "18.0.11", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.11.tgz", "integrity": "sha512-O38bPbI2CWtgw/OoQoY+BRelw7uysmXbWvw3nLWO21H1HSh+GOlqPuXshJfjmpNlKiiSDG9cc1JZAaMmVdcTlw==", + "peer": true, "dependencies": { "@types/react": "*" } @@ -1808,136 +1829,292 @@ "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==" }, - "node_modules/@typescript-eslint/parser": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.2.0.tgz", - "integrity": "sha512-5FKsVcHTk6TafQKQbuIVkXq58Fnbkd2wDL4LB7AURN7RUOu1utVP+G8+6u3ZhEroW3DF6hyo3ZEXxgKgp4KeCg==", + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.0.tgz", + "integrity": "sha512-HyAZtpdkgZwpq8Sz3FSUvCR4c+ScbuWa9AksK2Jweub7w4M3yTz4O11AqVJzLYjy/B9ZWPyc81I+mOdJU/bDQw==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "7.2.0", - "@typescript-eslint/types": "7.2.0", - "@typescript-eslint/typescript-estree": "7.2.0", - "@typescript-eslint/visitor-keys": "7.2.0", - "debug": "^4.3.4" + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/type-utils": "8.59.0", + "@typescript-eslint/utils": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "@typescript-eslint/parser": "^8.59.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.2.0.tgz", - "integrity": "sha512-Qh976RbQM/fYtjx9hs4XkayYujB/aPwglw2choHmf3zBjB4qOywWSdt9+KLRdHubGcoSwBnXUH2sR3hkyaERRg==", + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.0.tgz", + "integrity": "sha512-TI1XGwKbDpo9tRW8UDIXCOeLk55qe9ZFGs8MTKU6/M08HWTw52DD/IYhfQtOEhEdPhLMT26Ka/x7p70nd3dzDg==", "dev": true, + "license": "MIT", + "peer": true, "dependencies": { - "@typescript-eslint/types": "7.2.0", - "@typescript-eslint/visitor-keys": "7.2.0" + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", + "debug": "^4.4.3" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/types": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.2.0.tgz", - "integrity": "sha512-XFtUHPI/abFhm4cbCDc5Ykc8npOKBSJePY3a3s+lwumt7XWJuzP5cZcfZ610MIPHjQjNsOLlYK8ASPaNG8UiyA==", + "node_modules/@typescript-eslint/project-service": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.0.tgz", + "integrity": "sha512-Lw5ITrR5s5TbC19YSvlr63ZfLaJoU6vtKTHyB0GQOpX0W7d5/Ir6vUahWi/8Sps/nOukZQ0IB3SmlxZnjaKVnw==", "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.59.0", + "@typescript-eslint/types": "^8.59.0", + "debug": "^4.4.3" + }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.2.0.tgz", - "integrity": "sha512-cyxS5WQQCoBwSakpMrvMXuMDEbhOo9bNHHrNcEWis6XHx6KF518tkF1wBvKIn/tpq5ZpUYK7Bdklu8qY0MsFIA==", + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.0.tgz", + "integrity": "sha512-UzR16Ut8IpA3Mc4DbgAShlPPkVm8xXMWafXxB0BocaVRHs8ZGakAxGRskF7FId3sdk9lgGD73GSFaWmWFDE4dg==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "7.2.0", - "@typescript-eslint/visitor-keys": "7.2.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "minimatch": "9.0.3", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.0.tgz", + "integrity": "sha512-91Sbl3s4Kb3SybliIY6muFBmHVv+pYXfybC4Oolp3dvk8BvIE3wOPc+403CWIT7mJNkfQRGtdqghzs2+Z91Tqg==", "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", - "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "node_modules/@typescript-eslint/type-utils": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.0.tgz", + "integrity": "sha512-3TRiZaQSltGqGeNrJzzr1+8YcEobKH9rHnqIp/1psfKFmhRQDNMGP5hBufanYTGznwShzVLs3Mz+gDN7HkWfXg==", "dev": true, + "license": "MIT", "dependencies": { - "brace-expansion": "^2.0.1" + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0", + "@typescript-eslint/utils": "8.59.0", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { - "url": "https://github.com/sponsors/isaacs" + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.2.0.tgz", - "integrity": "sha512-c6EIQRHhcpl6+tO8EMR+kjkkV+ugUNXOmeASA1rlzkd8EPIriavpWoiEz1HR/VLhbVIdhqnV6E7JZm00cBDx2A==", + "node_modules/@typescript-eslint/types": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.0.tgz", + "integrity": "sha512-nLzdsT1gdOgFxxxwrlNVUBzSNBEEHJ86bblmk4QAS6stfig7rcJzWKqCyxFy3YRRHXDWEkb2NralA1nOYkkm/A==", "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.2.0", - "eslint-visitor-keys": "^3.4.1" - }, + "license": "MIT", "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.0.tgz", + "integrity": "sha512-O9Re9P1BmBLFJyikRbQpLku/QA3/AueZNO9WePLBwQrvkixTmDe8u76B6CYUAITRl/rHawggEqUGn5QIkVRLMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.59.0", + "@typescript-eslint/tsconfig-utils": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.0.tgz", + "integrity": "sha512-I1R/K7V07XsMJ12Oaxg/O9GfrysGTmCRhvZJBv0RE0NcULMzjqVpR5kRRQjHsz3J/bElU7HwCO7zkqL+MSUz+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.0.tgz", + "integrity": "sha512-/uejZt4dSere1bx12WLlPfv8GktzcaDtuJ7s42/HEZ5zGj9oxRaD4bj7qwSunXkf+pbAhFt2zjpHYUiT5lHf0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.59.0", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" }, "node_modules/acorn": { @@ -1945,6 +2122,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", "dev": true, + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1962,10 +2140,11 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, + "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -2039,22 +2218,24 @@ } }, "node_modules/aria-query": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", - "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", + "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", "dev": true, - "dependencies": { - "dequal": "^2.0.3" + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" } }, "node_modules/array-buffer-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz", - "integrity": "sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", + "integrity": "sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", - "is-array-buffer": "^3.0.4" + "call-bound": "^1.0.3", + "is-array-buffer": "^3.0.5" }, "engines": { "node": ">= 0.4" @@ -2064,17 +2245,20 @@ } }, "node_modules/array-includes": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.8.tgz", - "integrity": "sha512-itaWrbYbqpGXkGhZPGUulwnhVf5Hpy1xiCFsGqyIGglbBxmG5vSjxQen3/WGOjPpNEv1RtBLKxbmVXm8HpJStQ==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.9.tgz", + "integrity": "sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", "define-properties": "^1.2.1", - "es-abstract": "^1.23.2", - "es-object-atoms": "^1.0.0", - "get-intrinsic": "^1.2.4", - "is-string": "^1.0.7" + "es-abstract": "^1.24.0", + "es-object-atoms": "^1.1.1", + "get-intrinsic": "^1.3.0", + "is-string": "^1.1.1", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -2083,20 +2267,12 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/array.prototype.findlast": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz", "integrity": "sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", @@ -2113,17 +2289,19 @@ } }, "node_modules/array.prototype.findlastindex": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.5.tgz", - "integrity": "sha512-zfETvRFA8o7EiNn++N5f/kaCw221hrpGsDmcpndVupkPzEc1Wuf3VgC0qby1BbHs7f5DVYjgtEU2LLh5bqeGfQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.6.tgz", + "integrity": "sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", "define-properties": "^1.2.1", - "es-abstract": "^1.23.2", + "es-abstract": "^1.23.9", "es-errors": "^1.3.0", - "es-object-atoms": "^1.0.0", - "es-shim-unscopables": "^1.0.2" + "es-object-atoms": "^1.1.1", + "es-shim-unscopables": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -2133,15 +2311,16 @@ } }, "node_modules/array.prototype.flat": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", - "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.3.tgz", + "integrity": "sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "es-shim-unscopables": "^1.0.0" + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.5", + "es-shim-unscopables": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -2151,15 +2330,16 @@ } }, "node_modules/array.prototype.flatmap": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz", - "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.3.tgz", + "integrity": "sha512-Y7Wt51eKJSyi80hFrJCePGGNo5ktJCslFuboqJsbf57CCPcm5zztluPlc4/aD8sWsKvlwatezpV4U1efk8kpjg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "es-shim-unscopables": "^1.0.0" + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.5", + "es-shim-unscopables": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -2168,45 +2348,37 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array.prototype.toreversed": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/array.prototype.toreversed/-/array.prototype.toreversed-1.1.2.tgz", - "integrity": "sha512-wwDCoT4Ck4Cz7sLtgUmzR5UV3YF5mFHUlbChCzZBQZ+0m2cl/DH3tKgvphv1nKgFsJ48oCSg6p91q2Vm0I/ZMA==", - "dev": true, - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "es-shim-unscopables": "^1.0.0" - } - }, "node_modules/array.prototype.tosorted": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.3.tgz", - "integrity": "sha512-/DdH4TiTmOKzyQbp/eadcCVexiCb36xJg7HshYOYJnNZFDj33GEv0P7GxsynpShhq4OLYJzbGcBDkLsDt7MnNg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.4.tgz", + "integrity": "sha512-p6Fx8B7b7ZhL/gmUsAy0D15WhvDccw3mnGNbZpi3pmeJdxtWsj2jEaI4Y6oo3XiHfzuSgPwKc04MYt6KgvC/wA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", + "call-bind": "^1.0.7", "define-properties": "^1.2.1", - "es-abstract": "^1.22.3", - "es-errors": "^1.1.0", + "es-abstract": "^1.23.3", + "es-errors": "^1.3.0", "es-shim-unscopables": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" } }, "node_modules/arraybuffer.prototype.slice": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz", - "integrity": "sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.4.tgz", + "integrity": "sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==", "dev": true, + "license": "MIT", "dependencies": { "array-buffer-byte-length": "^1.0.1", - "call-bind": "^1.0.5", + "call-bind": "^1.0.8", "define-properties": "^1.2.1", - "es-abstract": "^1.22.3", - "es-errors": "^1.2.1", - "get-intrinsic": "^1.2.3", - "is-array-buffer": "^3.0.4", - "is-shared-array-buffer": "^1.0.2" + "es-abstract": "^1.23.5", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "is-array-buffer": "^3.0.4" }, "engines": { "node": ">= 0.4" @@ -2219,7 +2391,18 @@ "version": "0.0.8", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz", "integrity": "sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==", - "dev": true + "dev": true, + "license": "MIT" + }, + "node_modules/async-function": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", + "integrity": "sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } }, "node_modules/attr-accept": { "version": "2.2.2", @@ -2270,6 +2453,7 @@ "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", "dev": true, + "license": "MIT", "dependencies": { "possible-typed-array-names": "^1.0.0" }, @@ -2281,21 +2465,23 @@ } }, "node_modules/axe-core": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz", - "integrity": "sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.11.3.tgz", + "integrity": "sha512-zBQouZixDTbo3jMGqHKyePxYxr1e5W8UdTmBQ7sNtaA9M2bE32daxxPLS/jojhKOHxQ7LWwPjfiwf/fhaJWzlg==", "dev": true, + "license": "MPL-2.0", "engines": { "node": ">=4" } }, "node_modules/axobject-query": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz", - "integrity": "sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", "dev": true, - "dependencies": { - "dequal": "^2.0.3" + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" } }, "node_modules/b4a": { @@ -2333,44 +2519,99 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/bare-events": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.2.2.tgz", - "integrity": "sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==", - "optional": true + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } }, "node_modules/bare-fs": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.0.tgz", - "integrity": "sha512-TNFqa1B4N99pds2a5NYHR15o0ZpdNKbAeKTE/+G6ED/UeOavv8RY3dr/Fu99HW3zU3pXpo2kDNO8Sjsm2esfOw==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.1.tgz", + "integrity": "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw==", + "license": "Apache-2.0", "optional": true, "dependencies": { - "bare-events": "^2.0.0", - "bare-path": "^2.0.0", - "bare-stream": "^1.0.0" + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } } }, "node_modules/bare-os": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.3.0.tgz", - "integrity": "sha512-oPb8oMM1xZbhRQBngTgpcQ5gXw6kjOaRsSWsIeNyRxGed2w/ARyP7ScBYpWR1qfX2E5rS3gBw6OWcSQo+s+kUg==", - "optional": true + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.0.tgz", + "integrity": "sha512-JTjuZyNIDpw+GytMO4a6TK1VXdVKKJr6DRxEHasyuYyShV2deuiHJK/ahGZlebc+SG0/wJCB9XK8gprBGDFi/Q==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } }, "node_modules/bare-path": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.2.tgz", - "integrity": "sha512-o7KSt4prEphWUHa3QUwCxUI00R86VdjiuxmJK0iNVDHYPGo+HsDaVCnqCmPbf/MiW1ok8F4p3m8RTHlWk8K2ig==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", "optional": true, "dependencies": { - "bare-os": "^2.1.0" + "bare-os": "^3.0.1" } }, "node_modules/bare-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-1.0.0.tgz", - "integrity": "sha512-KhNUoDL40iP4gFaLSsoGE479t0jHijfYdIcxRn/XtezA2BaUD0NRf/JGRpsMq6dMNM+SrCrB0YSSo/5wBY4rOQ==", + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.0.tgz", + "integrity": "sha512-3zAJRZMDFGjdn+RVnNpF9kuELw+0Fl3lpndM4NcEOhb9zwtSo/deETfuIwMSE5BXanA0FrN1qVjffGwAg2Y7EA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.2.tgz", + "integrity": "sha512-/9a2j4ac6ckpmAHvod/ob7x439OAHst/drc2Clnq+reRYd/ovddwcF4LfoxHyNk5AuGBnPg+HqFjmE/Zpq6v0A==", + "license": "Apache-2.0", "optional": true, "dependencies": { - "streamx": "^2.16.1" + "bare-path": "^3.0.0" } }, "node_modules/base64-js": { @@ -2414,21 +2655,23 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -2452,6 +2695,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -2500,16 +2744,47 @@ } }, "node_modules/call-bind": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", "dependencies": { - "es-define-property": "^1.0.0", "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" }, "engines": { "node": ">= 0.4" @@ -2738,13 +3013,13 @@ "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "peer": true + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -2902,17 +3177,19 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==", - "dev": true + "dev": true, + "license": "BSD-2-Clause" }, "node_modules/data-view-buffer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", - "integrity": "sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.2.tgz", + "integrity": "sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-data-view": "^1.0.1" + "is-data-view": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -2922,29 +3199,31 @@ } }, "node_modules/data-view-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz", - "integrity": "sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.2.tgz", + "integrity": "sha512-tuhGbE6CfTM9+5ANGf+oQb72Ky/0+s3xKUpHvShfiz2RxMFgFPjsXuRLBVMtvMs15awe45SRb83D6wH4ew6wlQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-data-view": "^1.0.1" + "is-data-view": "^1.0.2" }, "engines": { "node": ">= 0.4" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "url": "https://github.com/sponsors/inspect-js" } }, "node_modules/data-view-byte-offset": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.0.tgz", - "integrity": "sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.1.tgz", + "integrity": "sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.2", "es-errors": "^1.3.0", "is-data-view": "^1.0.1" }, @@ -2959,17 +3238,19 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/kossnocorp" } }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -3076,9 +3357,10 @@ } }, "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", "engines": { "node": ">=8" } @@ -3105,18 +3387,6 @@ "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==" }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -3143,10 +3413,26 @@ "csstype": "^3.0.2" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" }, "node_modules/electron-to-chromium": { "version": "1.4.773", @@ -3191,57 +3477,66 @@ } }, "node_modules/es-abstract": { - "version": "1.23.3", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.3.tgz", - "integrity": "sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A==", + "version": "1.24.2", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.24.2.tgz", + "integrity": "sha512-2FpH9Q5i2RRwyEP1AylXe6nYLR5OhaJTZwmlcP0dL/+JCbgg7yyEo/sEK6HeGZRf3dFpWwThaRHVApXSkW3xeg==", "dev": true, + "license": "MIT", "dependencies": { - "array-buffer-byte-length": "^1.0.1", - "arraybuffer.prototype.slice": "^1.0.3", + "array-buffer-byte-length": "^1.0.2", + "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "data-view-buffer": "^1.0.1", - "data-view-byte-length": "^1.0.1", - "data-view-byte-offset": "^1.0.0", - "es-define-property": "^1.0.0", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "data-view-buffer": "^1.0.2", + "data-view-byte-length": "^1.0.2", + "data-view-byte-offset": "^1.0.1", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", - "es-object-atoms": "^1.0.0", - "es-set-tostringtag": "^2.0.3", - "es-to-primitive": "^1.2.1", - "function.prototype.name": "^1.1.6", - "get-intrinsic": "^1.2.4", - "get-symbol-description": "^1.0.2", - "globalthis": "^1.0.3", - "gopd": "^1.0.1", + "es-object-atoms": "^1.1.1", + "es-set-tostringtag": "^2.1.0", + "es-to-primitive": "^1.3.0", + "function.prototype.name": "^1.1.8", + "get-intrinsic": "^1.3.0", + "get-proto": "^1.0.1", + "get-symbol-description": "^1.1.0", + "globalthis": "^1.0.4", + "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", - "has-proto": "^1.0.3", - "has-symbols": "^1.0.3", + "has-proto": "^1.2.0", + "has-symbols": "^1.1.0", "hasown": "^2.0.2", - "internal-slot": "^1.0.7", - "is-array-buffer": "^3.0.4", + "internal-slot": "^1.1.0", + "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", - "is-data-view": "^1.0.1", + "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", - "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.3", - "is-string": "^1.0.7", - "is-typed-array": "^1.1.13", - "is-weakref": "^1.0.2", - "object-inspect": "^1.13.1", + "is-regex": "^1.2.1", + "is-set": "^2.0.3", + "is-shared-array-buffer": "^1.0.4", + "is-string": "^1.1.1", + "is-typed-array": "^1.1.15", + "is-weakref": "^1.1.1", + "math-intrinsics": "^1.1.0", + "object-inspect": "^1.13.4", "object-keys": "^1.1.1", - "object.assign": "^4.1.5", - "regexp.prototype.flags": "^1.5.2", - "safe-array-concat": "^1.1.2", - "safe-regex-test": "^1.0.3", - "string.prototype.trim": "^1.2.9", - "string.prototype.trimend": "^1.0.8", + "object.assign": "^4.1.7", + "own-keys": "^1.0.1", + "regexp.prototype.flags": "^1.5.4", + "safe-array-concat": "^1.1.3", + "safe-push-apply": "^1.0.0", + "safe-regex-test": "^1.1.0", + "set-proto": "^1.0.0", + "stop-iteration-iterator": "^1.1.0", + "string.prototype.trim": "^1.2.10", + "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", - "typed-array-buffer": "^1.0.2", - "typed-array-byte-length": "^1.0.1", - "typed-array-byte-offset": "^1.0.2", - "typed-array-length": "^1.0.6", - "unbox-primitive": "^1.0.2", - "which-typed-array": "^1.1.15" + "typed-array-buffer": "^1.0.3", + "typed-array-byte-length": "^1.0.3", + "typed-array-byte-offset": "^1.0.4", + "typed-array-length": "^1.0.7", + "unbox-primitive": "^1.1.0", + "which-typed-array": "^1.1.19" }, "engines": { "node": ">= 0.4" @@ -3251,13 +3546,11 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "dev": true, - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -3272,35 +3565,39 @@ } }, "node_modules/es-iterator-helpers": { - "version": "1.0.19", - "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.19.tgz", - "integrity": "sha512-zoMwbCcH5hwUkKJkT8kDIBZSz9I6mVG//+lDCinLCGov4+r7NIy0ld8o03M0cJxl2spVf6ESYVS6/gpIfq1FFw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.3.2.tgz", + "integrity": "sha512-HVLACW1TppGYjJ8H6/jqH/pqOtKRw6wMlrB23xfExmFWxFquAIWCmwoLsOyN96K4a5KbmOf5At9ZUO3GZbetAw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", "define-properties": "^1.2.1", - "es-abstract": "^1.23.3", + "es-abstract": "^1.24.2", "es-errors": "^1.3.0", - "es-set-tostringtag": "^2.0.3", + "es-set-tostringtag": "^2.1.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "globalthis": "^1.0.3", + "get-intrinsic": "^1.3.0", + "globalthis": "^1.0.4", + "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", - "has-proto": "^1.0.3", - "has-symbols": "^1.0.3", - "internal-slot": "^1.0.7", - "iterator.prototype": "^1.1.2", - "safe-array-concat": "^1.1.2" + "has-proto": "^1.2.0", + "has-symbols": "^1.1.0", + "internal-slot": "^1.1.0", + "iterator.prototype": "^1.1.5", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" } }, "node_modules/es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "dev": true, + "license": "MIT", "dependencies": { "es-errors": "^1.3.0" }, @@ -3309,37 +3606,44 @@ } }, "node_modules/es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "dev": true, + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" } }, "node_modules/es-shim-unscopables": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz", - "integrity": "sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.1.0.tgz", + "integrity": "sha512-d9T8ucsEhh8Bi1woXCf+TIKDIROLG5WCkxg8geBCbvk22kzwC5G2OnXVMO6FUsvQlgUUXQ2itephWDLqDzbeCw==", "dev": true, + "license": "MIT", "dependencies": { - "hasown": "^2.0.0" + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" } }, "node_modules/es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.3.0.tgz", + "integrity": "sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==", "dev": true, + "license": "MIT", "dependencies": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" + "is-callable": "^1.2.7", + "is-date-object": "^1.0.5", + "is-symbol": "^1.0.4" }, "engines": { "node": ">= 0.4" @@ -3373,6 +3677,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", "dev": true, + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -3424,14 +3729,16 @@ } }, "node_modules/eslint-config-next": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.2.3.tgz", - "integrity": "sha512-ZkNztm3Q7hjqvB1rRlOX8P9E/cXRL9ajRcs8jufEtwMfTVYRqnmtnaSu57QqHyBlovMuiB8LEzfLBkh5RYV6Fg==", + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.2.35.tgz", + "integrity": "sha512-BpLsv01UisH193WyT/1lpHqq5iJ/Orfz9h/NOOlAmTUq4GY349PextQ62K4XpnaM9supeiEn3TaOTeQO07gURg==", "dev": true, + "license": "MIT", "dependencies": { - "@next/eslint-plugin-next": "14.2.3", + "@next/eslint-plugin-next": "14.2.35", "@rushstack/eslint-patch": "^1.3.3", - "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || 7.0.0 - 7.2.0", + "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", + "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.28.1", @@ -3495,10 +3802,11 @@ } }, "node_modules/eslint-module-utils": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz", - "integrity": "sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==", + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.12.1.tgz", + "integrity": "sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw==", "dev": true, + "license": "MIT", "dependencies": { "debug": "^3.2.7" }, @@ -3521,34 +3829,38 @@ } }, "node_modules/eslint-plugin-import": { - "version": "2.29.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz", - "integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==", + "version": "2.32.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", + "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, + "license": "MIT", + "peer": true, "dependencies": { - "array-includes": "^3.1.7", - "array.prototype.findlastindex": "^1.2.3", - "array.prototype.flat": "^1.3.2", - "array.prototype.flatmap": "^1.3.2", + "@rtsao/scc": "^1.1.0", + "array-includes": "^3.1.9", + "array.prototype.findlastindex": "^1.2.6", + "array.prototype.flat": "^1.3.3", + "array.prototype.flatmap": "^1.3.3", "debug": "^3.2.7", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.9", - "eslint-module-utils": "^2.8.0", - "hasown": "^2.0.0", - "is-core-module": "^2.13.1", + "eslint-module-utils": "^2.12.1", + "hasown": "^2.0.2", + "is-core-module": "^2.16.1", "is-glob": "^4.0.3", "minimatch": "^3.1.2", - "object.fromentries": "^2.0.7", - "object.groupby": "^1.0.1", - "object.values": "^1.1.7", + "object.fromentries": "^2.0.8", + "object.groupby": "^1.0.3", + "object.values": "^1.2.1", "semver": "^6.3.1", + "string.prototype.trimend": "^1.0.9", "tsconfig-paths": "^3.15.0" }, "engines": { "node": ">=4" }, "peerDependencies": { - "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8" + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9" } }, "node_modules/eslint-plugin-import/node_modules/debug": { @@ -3582,72 +3894,74 @@ } }, "node_modules/eslint-plugin-jsx-a11y": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.8.0.tgz", - "integrity": "sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA==", + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.10.2.tgz", + "integrity": "sha512-scB3nz4WmG75pV8+3eRUQOHZlNSUhFNq37xnpgRkCCELU3XMvXAxLk1eqWWyE22Ki4Q01Fnsw9BA3cJHDPgn2Q==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/runtime": "^7.23.2", - "aria-query": "^5.3.0", - "array-includes": "^3.1.7", + "aria-query": "^5.3.2", + "array-includes": "^3.1.8", "array.prototype.flatmap": "^1.3.2", "ast-types-flow": "^0.0.8", - "axe-core": "=4.7.0", - "axobject-query": "^3.2.1", + "axe-core": "^4.10.0", + "axobject-query": "^4.1.0", "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", - "es-iterator-helpers": "^1.0.15", - "hasown": "^2.0.0", + "hasown": "^2.0.2", "jsx-ast-utils": "^3.3.5", "language-tags": "^1.0.9", "minimatch": "^3.1.2", - "object.entries": "^1.1.7", - "object.fromentries": "^2.0.7" + "object.fromentries": "^2.0.8", + "safe-regex-test": "^1.0.3", + "string.prototype.includes": "^2.0.1" }, "engines": { "node": ">=4.0" }, "peerDependencies": { - "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9" } }, "node_modules/eslint-plugin-react": { - "version": "7.34.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.34.1.tgz", - "integrity": "sha512-N97CxlouPT1AHt8Jn0mhhN2RrADlUAsk1/atcT2KyA/l9Q/E6ll7OIGwNumFmWfZ9skV3XXccYS19h80rHtgkw==", + "version": "7.37.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.5.tgz", + "integrity": "sha512-Qteup0SqU15kdocexFNAJMvCJEfa2xUKNV4CC1xsVMrIIqEy3SQ/rqyxCWNzfrd3/ldy6HMlD2e0JDVpDg2qIA==", "dev": true, + "license": "MIT", "dependencies": { - "array-includes": "^3.1.7", - "array.prototype.findlast": "^1.2.4", - "array.prototype.flatmap": "^1.3.2", - "array.prototype.toreversed": "^1.1.2", - "array.prototype.tosorted": "^1.1.3", + "array-includes": "^3.1.8", + "array.prototype.findlast": "^1.2.5", + "array.prototype.flatmap": "^1.3.3", + "array.prototype.tosorted": "^1.1.4", "doctrine": "^2.1.0", - "es-iterator-helpers": "^1.0.17", + "es-iterator-helpers": "^1.2.1", "estraverse": "^5.3.0", + "hasown": "^2.0.2", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", - "object.entries": "^1.1.7", - "object.fromentries": "^2.0.7", - "object.hasown": "^1.1.3", - "object.values": "^1.1.7", + "object.entries": "^1.1.9", + "object.fromentries": "^2.0.8", + "object.values": "^1.2.1", "prop-types": "^15.8.1", "resolve": "^2.0.0-next.5", "semver": "^6.3.1", - "string.prototype.matchall": "^4.0.10" + "string.prototype.matchall": "^4.0.12", + "string.prototype.repeat": "^1.0.0" }, "engines": { "node": ">=4" }, "peerDependencies": { - "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9.7" } }, "node_modules/eslint-plugin-react-hooks": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.2.tgz", - "integrity": "sha512-QzliNJq4GinDBcD8gPB5v0wh6g8q3SUi6EFF0x8N/BL9PoVs0atuGc47ozMRyOWAKdwaZ5OnbOEa3WR+dSGKuQ==", + "version": "5.0.0-canary-7118f5dd7-20230705", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-5.0.0-canary-7118f5dd7-20230705.tgz", + "integrity": "sha512-AZYbMo/NW9chdL7vk6HQzQhT+PvTAEVqWk9ziruUoW2kAOcN5qNyelv70e0F1VNQAbvutOC9oc+xfWycI9FxDw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -3660,6 +3974,7 @@ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", "dev": true, + "license": "Apache-2.0", "dependencies": { "esutils": "^2.0.2" }, @@ -3668,18 +3983,25 @@ } }, "node_modules/eslint-plugin-react/node_modules/resolve": { - "version": "2.0.0-next.5", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", - "integrity": "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==", + "version": "2.0.0-next.6", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.6.tgz", + "integrity": "sha512-3JmVl5hMGtJ3kMmB3zi3DL25KfkCEyy3Tw7Gmw7z5w8M9WlwoPFnIvwChzu1+cF3iaK3sp18hhPz8ANeimdJfA==", "dev": true, + "license": "MIT", "dependencies": { - "is-core-module": "^2.13.0", + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "node-exports-info": "^1.6.0", + "object-keys": "^1.1.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -3689,6 +4011,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, + "license": "ISC", "bin": { "semver": "bin/semver.js" } @@ -3794,6 +4117,15 @@ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -3896,9 +4228,10 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -3937,18 +4270,26 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", - "dev": true + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", + "dev": true, + "license": "ISC" }, "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "dev": true, + "license": "MIT", "dependencies": { - "is-callable": "^1.1.3" + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/foreground-child": { @@ -4035,15 +4376,18 @@ } }, "node_modules/function.prototype.name": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz", - "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.8.tgz", + "integrity": "sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "functions-have-names": "^1.2.3" + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "define-properties": "^1.2.1", + "functions-have-names": "^1.2.3", + "hasown": "^2.0.2", + "is-callable": "^1.2.7" }, "engines": { "node": ">= 0.4" @@ -4057,30 +4401,46 @@ "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true, + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "peer": true, "engines": { "node": ">=6.9.0" } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dev": true, + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -4097,15 +4457,30 @@ "node": ">=6" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-symbol-description": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.2.tgz", - "integrity": "sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", + "integrity": "sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4" + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -4132,22 +4507,22 @@ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" }, "node_modules/glob": { - "version": "10.3.10", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", - "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", - "jackspeak": "^2.3.5", - "minimatch": "^9.0.1", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", - "path-scurry": "^1.10.1" + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -4164,19 +4539,21 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/glob/node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -4216,33 +4593,14 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "dev": true, - "dependencies": { - "get-intrinsic": "^1.1.3" + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4260,10 +4618,14 @@ "dev": true }, "node_modules/has-bigints": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", - "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", + "integrity": "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==", "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -4290,10 +4652,14 @@ } }, "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.2.0.tgz", + "integrity": "sha512-KIL7eQPfHQRC8+XluaIw7BHUwwqL19bQn4hzNgdr+1wXoU0KKj6rufu47lhY7KbJR2C6T6+PfyN0Ea7wkSS+qQ==", "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.0" + }, "engines": { "node": ">= 0.4" }, @@ -4302,10 +4668,11 @@ } }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -4603,14 +4970,15 @@ "integrity": "sha512-qlD8YNDqyTKTyuITrDOffsl6Tdhv+UC4hcdAVuQsK4IMQ99nSgd1MIA/Q+jQYoh9r3hVUXhYh7urSRmXPkW04g==" }, "node_modules/internal-slot": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz", - "integrity": "sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", + "integrity": "sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==", "dev": true, + "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "hasown": "^2.0.0", - "side-channel": "^1.0.4" + "hasown": "^2.0.2", + "side-channel": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -4655,13 +5023,15 @@ } }, "node_modules/is-array-buffer": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", - "integrity": "sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", + "integrity": "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.1" + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -4676,12 +5046,17 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/is-async-function": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz", - "integrity": "sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.1.1.tgz", + "integrity": "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "async-function": "^1.0.0", + "call-bound": "^1.0.3", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -4691,12 +5066,16 @@ } }, "node_modules/is-bigint": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", - "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.1.0.tgz", + "integrity": "sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==", "dev": true, + "license": "MIT", "dependencies": { - "has-bigints": "^1.0.1" + "has-bigints": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4714,13 +5093,14 @@ } }, "node_modules/is-boolean-object": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", - "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.2.2.tgz", + "integrity": "sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -4734,6 +5114,7 @@ "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -4742,22 +5123,29 @@ } }, "node_modules/is-core-module": { - "version": "2.13.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", - "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", "dependencies": { - "hasown": "^2.0.0" + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-data-view": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.1.tgz", - "integrity": "sha512-AHkaJrsUVW6wq6JS8y3JnM/GJF/9cf+k20+iDzlSaJrinEo5+7vRiteOSwBhHRiAyQATN1AmY4hwzxJKPmYf+w==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.2.tgz", + "integrity": "sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw==", "dev": true, + "license": "MIT", "dependencies": { + "call-bound": "^1.0.2", + "get-intrinsic": "^1.2.6", "is-typed-array": "^1.1.13" }, "engines": { @@ -4768,12 +5156,14 @@ } }, "node_modules/is-date-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", - "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", + "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -4800,12 +5190,16 @@ } }, "node_modules/is-finalizationregistry": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.0.2.tgz", - "integrity": "sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.1.1.tgz", + "integrity": "sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2" + "call-bound": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4815,17 +5209,23 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -4859,6 +5259,7 @@ "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", "integrity": "sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -4882,17 +5283,20 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", "engines": { "node": ">=0.12.0" } }, "node_modules/is-number-object": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", - "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.1.1.tgz", + "integrity": "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -4922,13 +5326,16 @@ } }, "node_modules/is-regex": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", - "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -4942,6 +5349,7 @@ "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz", "integrity": "sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -4950,12 +5358,13 @@ } }, "node_modules/is-shared-array-buffer": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz", - "integrity": "sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.4.tgz", + "integrity": "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7" + "call-bound": "^1.0.3" }, "engines": { "node": ">= 0.4" @@ -4965,12 +5374,14 @@ } }, "node_modules/is-string": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", - "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", + "integrity": "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -4980,12 +5391,15 @@ } }, "node_modules/is-symbol": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", - "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.1.1.tgz", + "integrity": "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w==", "dev": true, + "license": "MIT", "dependencies": { - "has-symbols": "^1.0.2" + "call-bound": "^1.0.2", + "has-symbols": "^1.1.0", + "safe-regex-test": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -4995,12 +5409,13 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", - "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "dev": true, + "license": "MIT", "dependencies": { - "which-typed-array": "^1.1.14" + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -5014,6 +5429,7 @@ "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", "integrity": "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -5022,25 +5438,30 @@ } }, "node_modules/is-weakref": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", - "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.1.1.tgz", + "integrity": "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2" + "call-bound": "^1.0.3" }, - "funding": { + "engines": { + "node": ">= 0.4" + }, + "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-weakset": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.3.tgz", - "integrity": "sha512-LvIm3/KWzS9oRFHugab7d+M/GcBXuXX5xZkzPmN+NxihdQlZUQ4dWuSV1xR/sq6upL1TJEDrfBgRepHFdBtSNQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.4.tgz", + "integrity": "sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", - "get-intrinsic": "^1.2.4" + "call-bound": "^1.0.3", + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -5053,7 +5474,8 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", @@ -5061,28 +5483,31 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/iterator.prototype": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", - "integrity": "sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz", + "integrity": "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==", "dev": true, + "license": "MIT", "dependencies": { - "define-properties": "^1.2.1", - "get-intrinsic": "^1.2.1", - "has-symbols": "^1.0.3", - "reflect.getprototypeof": "^1.0.4", - "set-function-name": "^2.0.1" + "define-data-property": "^1.1.4", + "es-object-atoms": "^1.0.0", + "get-intrinsic": "^1.2.6", + "get-proto": "^1.0.0", + "has-symbols": "^1.1.0", + "set-function-name": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" } }, "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, - "engines": { - "node": ">=14" - }, "funding": { "url": "https://github.com/sponsors/isaacs" }, @@ -5112,10 +5537,11 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, @@ -5156,7 +5582,6 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "peer": true, "bin": { "json5": "lib/cli.js" }, @@ -5169,6 +5594,7 @@ "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", "dev": true, + "license": "MIT", "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", @@ -5189,16 +5615,18 @@ } }, "node_modules/language-subtag-registry": { - "version": "0.3.22", - "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz", - "integrity": "sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==", - "dev": true + "version": "0.3.23", + "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz", + "integrity": "sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==", + "dev": true, + "license": "CC0-1.0" }, "node_modules/language-tags": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.9.tgz", "integrity": "sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==", "dev": true, + "license": "MIT", "dependencies": { "language-subtag-registry": "^0.3.20" }, @@ -5248,14 +5676,16 @@ } }, "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" }, "node_modules/lodash-es": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", - "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" }, "node_modules/lodash.castarray": { "version": "4.4.0", @@ -5312,6 +5742,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mdast-util-find-and-replace": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz", @@ -5528,9 +5968,10 @@ } }, "node_modules/mdast-util-to-hast": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.1.0.tgz", - "integrity": "sha512-/e2l/6+OdGp/FB+ctrJ9Avz71AN/GRH3oi/3KAx/kMnoUsD6q0woXlDT8lLEeViVKE7oZxE7RXzvO3T8kF2/sA==", + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", @@ -6122,11 +6563,12 @@ ] }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -6145,10 +6587,11 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -6165,9 +6608,10 @@ } }, "node_modules/minipass": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.1.tgz", - "integrity": "sha512-UZ7eQ+h8ywIRAW1hIEl2AqdwzJucU/Kp59+8kkZeSvafXhZjul247BvIJjEVFVeON6d7lM46XX1HXCduKAS8VA==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" } @@ -6178,9 +6622,10 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/mz": { "version": "2.7.0", @@ -6193,15 +6638,16 @@ } }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -6221,11 +6667,12 @@ "dev": true }, "node_modules/next": { - "version": "14.2.3", - "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz", - "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==", + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.35.tgz", + "integrity": "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==", + "license": "MIT", "dependencies": { - "@next/env": "14.2.3", + "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", @@ -6240,15 +6687,15 @@ "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "14.2.3", - "@next/swc-darwin-x64": "14.2.3", - "@next/swc-linux-arm64-gnu": "14.2.3", - "@next/swc-linux-arm64-musl": "14.2.3", - "@next/swc-linux-x64-gnu": "14.2.3", - "@next/swc-linux-x64-musl": "14.2.3", - "@next/swc-win32-arm64-msvc": "14.2.3", - "@next/swc-win32-ia32-msvc": "14.2.3", - "@next/swc-win32-x64-msvc": "14.2.3" + "@next/swc-darwin-arm64": "14.2.33", + "@next/swc-darwin-x64": "14.2.33", + "@next/swc-linux-arm64-gnu": "14.2.33", + "@next/swc-linux-arm64-musl": "14.2.33", + "@next/swc-linux-x64-gnu": "14.2.33", + "@next/swc-linux-x64-musl": "14.2.33", + "@next/swc-win32-arm64-msvc": "14.2.33", + "@next/swc-win32-ia32-msvc": "14.2.33", + "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -6312,6 +6759,35 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" }, + "node_modules/node-exports-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.0.tgz", + "integrity": "sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "array.prototype.flatmap": "^1.3.3", + "es-errors": "^1.3.0", + "object.entries": "^1.1.9", + "semver": "^6.3.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/node-exports-info/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/node-releases": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", @@ -6334,9 +6810,9 @@ } }, "node_modules/npm": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/npm/-/npm-10.8.0.tgz", - "integrity": "sha512-wh93uRczgp7HDnPMiLXcCkv2hagdJS0zJ9KT/31d0FoXP02+qgN2AOwpaW85fxRWkinl2rELfPw+CjBXW48/jQ==", + "version": "10.9.8", + "resolved": "https://registry.npmjs.org/npm/-/npm-10.9.8.tgz", + "integrity": "sha512-fYwb6ODSmHkqrJQQaCxY3M2lPf/mpgC7ik0HSzzIwG5CGtabRp4bNqikatvCoT42b5INQSqudVH0R7yVmC9hVg==", "bundleDependencies": [ "@isaacs/string-locale-compare", "@npmcli/arborist", @@ -6407,75 +6883,83 @@ "which", "write-file-atomic" ], + "license": "Artistic-2.0", + "workspaces": [ + "docs", + "smoke-tests", + "mock-globals", + "mock-registry", + "workspaces/*" + ], "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", - "@npmcli/arborist": "^7.5.2", - "@npmcli/config": "^8.3.2", - "@npmcli/fs": "^3.1.1", - "@npmcli/map-workspaces": "^3.0.6", - "@npmcli/package-json": "^5.1.0", - "@npmcli/promise-spawn": "^7.0.2", - "@npmcli/redact": "^2.0.0", - "@npmcli/run-script": "^8.1.0", - "@sigstore/tuf": "^2.3.3", - "abbrev": "^2.0.0", + "@npmcli/arborist": "^8.0.5", + "@npmcli/config": "^9.0.0", + "@npmcli/fs": "^4.0.0", + "@npmcli/map-workspaces": "^4.0.2", + "@npmcli/package-json": "^6.2.0", + "@npmcli/promise-spawn": "^8.0.3", + "@npmcli/redact": "^3.2.2", + "@npmcli/run-script": "^9.1.0", + "@sigstore/tuf": "^3.1.1", + "abbrev": "^3.0.1", "archy": "~1.0.0", - "cacache": "^18.0.3", - "chalk": "^5.3.0", - "ci-info": "^4.0.0", + "cacache": "^19.0.1", + "chalk": "^5.6.2", + "ci-info": "^4.4.0", "cli-columns": "^4.0.0", "fastest-levenshtein": "^1.0.16", "fs-minipass": "^3.0.3", - "glob": "^10.3.15", + "glob": "^10.5.0", "graceful-fs": "^4.2.11", - "hosted-git-info": "^7.0.2", - "ini": "^4.1.2", - "init-package-json": "^6.0.3", - "is-cidr": "^5.0.5", - "json-parse-even-better-errors": "^3.0.2", - "libnpmaccess": "^8.0.6", - "libnpmdiff": "^6.1.2", - "libnpmexec": "^8.1.1", - "libnpmfund": "^5.0.10", - "libnpmhook": "^10.0.5", - "libnpmorg": "^6.0.6", - "libnpmpack": "^7.0.2", - "libnpmpublish": "^9.0.8", - "libnpmsearch": "^7.0.5", - "libnpmteam": "^6.0.5", - "libnpmversion": "^6.0.2", - "make-fetch-happen": "^13.0.1", - "minimatch": "^9.0.4", - "minipass": "^7.1.1", + "hosted-git-info": "^8.1.0", + "ini": "^5.0.0", + "init-package-json": "^7.0.2", + "is-cidr": "^5.1.1", + "json-parse-even-better-errors": "^4.0.0", + "libnpmaccess": "^9.0.0", + "libnpmdiff": "^7.0.5", + "libnpmexec": "^9.0.5", + "libnpmfund": "^6.0.5", + "libnpmhook": "^11.0.0", + "libnpmorg": "^7.0.0", + "libnpmpack": "^8.0.5", + "libnpmpublish": "^10.0.2", + "libnpmsearch": "^8.0.0", + "libnpmteam": "^7.0.0", + "libnpmversion": "^7.0.0", + "make-fetch-happen": "^14.0.3", + "minimatch": "^9.0.9", + "minipass": "^7.1.3", "minipass-pipeline": "^1.2.4", "ms": "^2.1.2", - "node-gyp": "^10.1.0", - "nopt": "^7.2.1", - "normalize-package-data": "^6.0.1", - "npm-audit-report": "^5.0.0", - "npm-install-checks": "^6.3.0", - "npm-package-arg": "^11.0.2", - "npm-pick-manifest": "^9.0.1", - "npm-profile": "^10.0.0", - "npm-registry-fetch": "^17.0.1", - "npm-user-validate": "^2.0.1", - "p-map": "^4.0.0", - "pacote": "^18.0.6", - "parse-conflict-json": "^3.0.1", - "proc-log": "^4.2.0", + "node-gyp": "^11.5.0", + "nopt": "^8.1.0", + "normalize-package-data": "^7.0.1", + "npm-audit-report": "^6.0.0", + "npm-install-checks": "^7.1.2", + "npm-package-arg": "^12.0.2", + "npm-pick-manifest": "^10.0.0", + "npm-profile": "^11.0.1", + "npm-registry-fetch": "^18.0.2", + "npm-user-validate": "^3.0.0", + "p-map": "^7.0.4", + "pacote": "^19.0.1", + "parse-conflict-json": "^4.0.0", + "proc-log": "^5.0.0", "qrcode-terminal": "^0.12.0", - "read": "^3.0.1", - "semver": "^7.6.2", + "read": "^4.1.0", + "semver": "^7.7.4", "spdx-expression-parse": "^4.0.0", - "ssri": "^10.0.6", + "ssri": "^12.0.0", "supports-color": "^9.4.0", - "tar": "^6.2.1", + "tar": "^7.5.11", "text-table": "~0.2.0", "tiny-relative-date": "^1.3.0", "treeverse": "^3.0.0", - "validate-npm-package-name": "^5.0.1", - "which": "^4.0.0", - "write-file-atomic": "^5.0.1" + "validate-npm-package-name": "^6.0.2", + "which": "^5.0.0", + "write-file-atomic": "^6.0.0" }, "bin": { "npm": "bin/npm-cli.js", @@ -6502,7 +6986,7 @@ } }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", + "version": "6.2.2", "inBundle": true, "license": "MIT", "engines": { @@ -6534,11 +7018,11 @@ } }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", + "version": "7.2.0", "inBundle": true, "license": "MIT", "dependencies": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^6.2.2" }, "engines": { "node": ">=12" @@ -6547,13 +7031,24 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/npm/node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "inBundle": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/npm/node_modules/@isaacs/string-locale-compare": { "version": "1.1.0", "inBundle": true, "license": "ISC" }, "node_modules/npm/node_modules/@npmcli/agent": { - "version": "2.2.2", + "version": "3.0.0", "inBundle": true, "license": "ISC", "dependencies": { @@ -6564,47 +7059,48 @@ "socks-proxy-agent": "^8.0.3" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/arborist": { - "version": "7.5.2", + "version": "8.0.5", "inBundle": true, "license": "ISC", "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", - "@npmcli/fs": "^3.1.1", - "@npmcli/installed-package-contents": "^2.1.0", - "@npmcli/map-workspaces": "^3.0.2", - "@npmcli/metavuln-calculator": "^7.1.1", - "@npmcli/name-from-folder": "^2.0.0", - "@npmcli/node-gyp": "^3.0.0", - "@npmcli/package-json": "^5.1.0", - "@npmcli/query": "^3.1.0", - "@npmcli/redact": "^2.0.0", - "@npmcli/run-script": "^8.1.0", - "bin-links": "^4.0.4", - "cacache": "^18.0.3", + "@npmcli/fs": "^4.0.0", + "@npmcli/installed-package-contents": "^3.0.0", + "@npmcli/map-workspaces": "^4.0.1", + "@npmcli/metavuln-calculator": "^8.0.0", + "@npmcli/name-from-folder": "^3.0.0", + "@npmcli/node-gyp": "^4.0.0", + "@npmcli/package-json": "^6.0.1", + "@npmcli/query": "^4.0.0", + "@npmcli/redact": "^3.0.0", + "@npmcli/run-script": "^9.0.1", + "bin-links": "^5.0.0", + "cacache": "^19.0.1", "common-ancestor-path": "^1.0.1", - "hosted-git-info": "^7.0.2", - "json-parse-even-better-errors": "^3.0.2", + "hosted-git-info": "^8.0.0", + "json-parse-even-better-errors": "^4.0.0", "json-stringify-nice": "^1.1.4", "lru-cache": "^10.2.2", "minimatch": "^9.0.4", - "nopt": "^7.2.1", - "npm-install-checks": "^6.2.0", - "npm-package-arg": "^11.0.2", - "npm-pick-manifest": "^9.0.1", - "npm-registry-fetch": "^17.0.1", - "pacote": "^18.0.6", - "parse-conflict-json": "^3.0.0", - "proc-log": "^4.2.0", - "proggy": "^2.0.0", + "nopt": "^8.0.0", + "npm-install-checks": "^7.1.0", + "npm-package-arg": "^12.0.0", + "npm-pick-manifest": "^10.0.0", + "npm-registry-fetch": "^18.0.1", + "pacote": "^19.0.0", + "parse-conflict-json": "^4.0.0", + "proc-log": "^5.0.0", + "proggy": "^3.0.0", "promise-all-reject-late": "^1.0.0", "promise-call-limit": "^3.0.1", - "read-package-json-fast": "^3.0.2", + "promise-retry": "^2.0.1", + "read-package-json-fast": "^4.0.0", "semver": "^7.3.7", - "ssri": "^10.0.6", + "ssri": "^12.0.0", "treeverse": "^3.0.0", "walk-up-path": "^3.0.1" }, @@ -6612,177 +7108,207 @@ "arborist": "bin/index.js" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/config": { - "version": "8.3.2", + "version": "9.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/map-workspaces": "^3.0.2", + "@npmcli/map-workspaces": "^4.0.1", + "@npmcli/package-json": "^6.0.1", "ci-info": "^4.0.0", - "ini": "^4.1.2", - "nopt": "^7.2.1", - "proc-log": "^4.2.0", - "read-package-json-fast": "^3.0.2", + "ini": "^5.0.0", + "nopt": "^8.0.0", + "proc-log": "^5.0.0", "semver": "^7.3.5", "walk-up-path": "^3.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/fs": { - "version": "3.1.1", + "version": "4.0.0", "inBundle": true, "license": "ISC", "dependencies": { "semver": "^7.3.5" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/git": { - "version": "5.0.7", + "version": "6.0.3", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/promise-spawn": "^7.0.0", + "@npmcli/promise-spawn": "^8.0.0", + "ini": "^5.0.0", "lru-cache": "^10.0.1", - "npm-pick-manifest": "^9.0.0", - "proc-log": "^4.0.0", - "promise-inflight": "^1.0.1", + "npm-pick-manifest": "^10.0.0", + "proc-log": "^5.0.0", "promise-retry": "^2.0.1", "semver": "^7.3.5", - "which": "^4.0.0" + "which": "^5.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/installed-package-contents": { - "version": "2.1.0", + "version": "3.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "npm-bundled": "^3.0.0", - "npm-normalize-package-bin": "^3.0.0" + "npm-bundled": "^4.0.0", + "npm-normalize-package-bin": "^4.0.0" }, "bin": { "installed-package-contents": "bin/index.js" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/map-workspaces": { - "version": "3.0.6", + "version": "4.0.2", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/name-from-folder": "^2.0.0", + "@npmcli/name-from-folder": "^3.0.0", + "@npmcli/package-json": "^6.0.0", "glob": "^10.2.2", - "minimatch": "^9.0.0", - "read-package-json-fast": "^3.0.0" + "minimatch": "^9.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/metavuln-calculator": { - "version": "7.1.1", + "version": "8.0.1", "inBundle": true, "license": "ISC", "dependencies": { - "cacache": "^18.0.0", - "json-parse-even-better-errors": "^3.0.0", - "pacote": "^18.0.0", - "proc-log": "^4.1.0", + "cacache": "^19.0.0", + "json-parse-even-better-errors": "^4.0.0", + "pacote": "^20.0.0", + "proc-log": "^5.0.0", "semver": "^7.3.5" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/@npmcli/metavuln-calculator/node_modules/pacote": { + "version": "20.0.1", + "inBundle": true, + "license": "ISC", + "dependencies": { + "@npmcli/git": "^6.0.0", + "@npmcli/installed-package-contents": "^3.0.0", + "@npmcli/package-json": "^6.0.0", + "@npmcli/promise-spawn": "^8.0.0", + "@npmcli/run-script": "^9.0.0", + "cacache": "^19.0.0", + "fs-minipass": "^3.0.0", + "minipass": "^7.0.2", + "npm-package-arg": "^12.0.0", + "npm-packlist": "^9.0.0", + "npm-pick-manifest": "^10.0.0", + "npm-registry-fetch": "^18.0.0", + "proc-log": "^5.0.0", + "promise-retry": "^2.0.1", + "sigstore": "^3.0.0", + "ssri": "^12.0.0", + "tar": "^7.5.10" + }, + "bin": { + "pacote": "bin/index.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/name-from-folder": { - "version": "2.0.0", + "version": "3.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/node-gyp": { - "version": "3.0.0", + "version": "4.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/package-json": { - "version": "5.1.0", + "version": "6.2.0", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/git": "^5.0.0", + "@npmcli/git": "^6.0.0", "glob": "^10.2.2", - "hosted-git-info": "^7.0.0", - "json-parse-even-better-errors": "^3.0.0", - "normalize-package-data": "^6.0.0", - "proc-log": "^4.0.0", - "semver": "^7.5.3" + "hosted-git-info": "^8.0.0", + "json-parse-even-better-errors": "^4.0.0", + "proc-log": "^5.0.0", + "semver": "^7.5.3", + "validate-npm-package-license": "^3.0.4" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/promise-spawn": { - "version": "7.0.2", + "version": "8.0.3", "inBundle": true, "license": "ISC", "dependencies": { - "which": "^4.0.0" + "which": "^5.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/query": { - "version": "3.1.0", + "version": "4.0.1", "inBundle": true, "license": "ISC", "dependencies": { - "postcss-selector-parser": "^6.0.10" + "postcss-selector-parser": "^7.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/redact": { - "version": "2.0.0", + "version": "3.2.2", "inBundle": true, "license": "ISC", "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@npmcli/run-script": { - "version": "8.1.0", + "version": "9.1.0", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/node-gyp": "^3.0.0", - "@npmcli/package-json": "^5.0.0", - "@npmcli/promise-spawn": "^7.0.0", - "node-gyp": "^10.0.0", - "proc-log": "^4.0.0", - "which": "^4.0.0" + "@npmcli/node-gyp": "^4.0.0", + "@npmcli/package-json": "^6.0.0", + "@npmcli/promise-spawn": "^8.0.0", + "node-gyp": "^11.0.0", + "proc-log": "^5.0.0", + "which": "^5.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@pkgjs/parseargs": { @@ -6795,71 +7321,71 @@ } }, "node_modules/npm/node_modules/@sigstore/bundle": { - "version": "2.3.1", + "version": "3.1.0", "inBundle": true, "license": "Apache-2.0", "dependencies": { - "@sigstore/protobuf-specs": "^0.3.1" + "@sigstore/protobuf-specs": "^0.4.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@sigstore/core": { - "version": "1.1.0", + "version": "2.0.0", "inBundle": true, "license": "Apache-2.0", "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@sigstore/protobuf-specs": { - "version": "0.3.2", + "version": "0.4.3", "inBundle": true, "license": "Apache-2.0", "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@sigstore/sign": { - "version": "2.3.1", + "version": "3.1.0", "inBundle": true, "license": "Apache-2.0", "dependencies": { - "@sigstore/bundle": "^2.3.0", - "@sigstore/core": "^1.0.0", - "@sigstore/protobuf-specs": "^0.3.1", - "make-fetch-happen": "^13.0.1", - "proc-log": "^4.2.0", + "@sigstore/bundle": "^3.1.0", + "@sigstore/core": "^2.0.0", + "@sigstore/protobuf-specs": "^0.4.0", + "make-fetch-happen": "^14.0.2", + "proc-log": "^5.0.0", "promise-retry": "^2.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@sigstore/tuf": { - "version": "2.3.3", + "version": "3.1.1", "inBundle": true, "license": "Apache-2.0", "dependencies": { - "@sigstore/protobuf-specs": "^0.3.0", - "tuf-js": "^2.2.1" + "@sigstore/protobuf-specs": "^0.4.1", + "tuf-js": "^3.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@sigstore/verify": { - "version": "1.2.0", + "version": "2.1.1", "inBundle": true, "license": "Apache-2.0", "dependencies": { - "@sigstore/bundle": "^2.3.1", - "@sigstore/core": "^1.1.0", - "@sigstore/protobuf-specs": "^0.3.1" + "@sigstore/bundle": "^3.1.0", + "@sigstore/core": "^2.0.0", + "@sigstore/protobuf-specs": "^0.4.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/@tufjs/canonical-json": { @@ -6870,49 +7396,22 @@ "node": "^16.14.0 || >=18.0.0" } }, - "node_modules/npm/node_modules/@tufjs/models": { - "version": "2.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "@tufjs/canonical-json": "2.0.0", - "minimatch": "^9.0.4" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, "node_modules/npm/node_modules/abbrev": { - "version": "2.0.0", + "version": "3.0.1", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/agent-base": { - "version": "7.1.1", + "version": "7.1.4", "inBundle": true, "license": "MIT", - "dependencies": { - "debug": "^4.3.4" - }, "engines": { "node": ">= 14" } }, - "node_modules/npm/node_modules/aggregate-error": { - "version": "3.1.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/npm/node_modules/ansi-regex": { "version": "5.0.1", "inBundle": true, @@ -6922,7 +7421,7 @@ } }, "node_modules/npm/node_modules/ansi-styles": { - "version": "6.2.1", + "version": "6.2.3", "inBundle": true, "license": "MIT", "engines": { @@ -6933,7 +7432,7 @@ } }, "node_modules/npm/node_modules/aproba": { - "version": "2.0.0", + "version": "2.1.0", "inBundle": true, "license": "ISC" }, @@ -6948,17 +7447,18 @@ "license": "MIT" }, "node_modules/npm/node_modules/bin-links": { - "version": "4.0.4", + "version": "5.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "cmd-shim": "^6.0.0", - "npm-normalize-package-bin": "^3.0.0", - "read-cmd-shim": "^4.0.0", - "write-file-atomic": "^5.0.0" + "cmd-shim": "^7.0.0", + "npm-normalize-package-bin": "^4.0.0", + "proc-log": "^5.0.0", + "read-cmd-shim": "^5.0.0", + "write-file-atomic": "^6.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/binary-extensions": { @@ -6973,7 +7473,7 @@ } }, "node_modules/npm/node_modules/brace-expansion": { - "version": "2.0.1", + "version": "2.0.2", "inBundle": true, "license": "MIT", "dependencies": { @@ -6981,11 +7481,11 @@ } }, "node_modules/npm/node_modules/cacache": { - "version": "18.0.3", + "version": "19.0.1", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/fs": "^3.1.0", + "@npmcli/fs": "^4.0.0", "fs-minipass": "^3.0.0", "glob": "^10.2.2", "lru-cache": "^10.0.1", @@ -6993,17 +7493,17 @@ "minipass-collect": "^2.0.1", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", - "p-map": "^4.0.0", - "ssri": "^10.0.0", - "tar": "^6.1.11", - "unique-filename": "^3.0.0" + "p-map": "^7.0.2", + "ssri": "^12.0.0", + "tar": "^7.4.3", + "unique-filename": "^4.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/chalk": { - "version": "5.3.0", + "version": "5.6.2", "inBundle": true, "license": "MIT", "engines": { @@ -7014,15 +7514,15 @@ } }, "node_modules/npm/node_modules/chownr": { - "version": "2.0.0", + "version": "3.0.0", "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/npm/node_modules/ci-info": { - "version": "4.0.0", + "version": "4.4.0", "funding": [ { "type": "github", @@ -7036,7 +7536,7 @@ } }, "node_modules/npm/node_modules/cidr-regex": { - "version": "4.0.5", + "version": "4.1.3", "inBundle": true, "license": "BSD-2-Clause", "dependencies": { @@ -7046,14 +7546,6 @@ "node": ">=14" } }, - "node_modules/npm/node_modules/clean-stack": { - "version": "2.2.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/npm/node_modules/cli-columns": { "version": "4.0.0", "inBundle": true, @@ -7067,11 +7559,11 @@ } }, "node_modules/npm/node_modules/cmd-shim": { - "version": "6.0.3", + "version": "7.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/color-convert": { @@ -7096,7 +7588,7 @@ "license": "ISC" }, "node_modules/npm/node_modules/cross-spawn": { - "version": "7.0.3", + "version": "7.0.6", "inBundle": true, "license": "MIT", "dependencies": { @@ -7134,11 +7626,11 @@ } }, "node_modules/npm/node_modules/debug": { - "version": "4.3.4", + "version": "4.4.3", "inBundle": true, "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -7149,13 +7641,8 @@ } } }, - "node_modules/npm/node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "inBundle": true, - "license": "MIT" - }, "node_modules/npm/node_modules/diff": { - "version": "5.2.0", + "version": "5.2.2", "inBundle": true, "license": "BSD-3-Clause", "engines": { @@ -7195,7 +7682,7 @@ "license": "MIT" }, "node_modules/npm/node_modules/exponential-backoff": { - "version": "3.1.1", + "version": "3.1.3", "inBundle": true, "license": "Apache-2.0" }, @@ -7207,12 +7694,28 @@ "node": ">= 4.9.1" } }, + "node_modules/npm/node_modules/fdir": { + "version": "6.5.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/npm/node_modules/foreground-child": { - "version": "3.1.1", + "version": "3.3.1", "inBundle": true, "license": "ISC", "dependencies": { - "cross-spawn": "^7.0.0", + "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" }, "engines": { @@ -7233,31 +7736,21 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/npm/node_modules/function-bind": { - "version": "1.1.2", - "inBundle": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/npm/node_modules/glob": { - "version": "10.3.15", + "version": "10.5.0", "inBundle": true, "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.11.0" + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -7267,30 +7760,19 @@ "inBundle": true, "license": "ISC" }, - "node_modules/npm/node_modules/hasown": { - "version": "2.0.2", - "inBundle": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/npm/node_modules/hosted-git-info": { - "version": "7.0.2", + "version": "8.1.0", "inBundle": true, "license": "ISC", "dependencies": { "lru-cache": "^10.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/http-cache-semantics": { - "version": "4.1.1", + "version": "4.2.0", "inBundle": true, "license": "BSD-2-Clause" }, @@ -7307,11 +7789,11 @@ } }, "node_modules/npm/node_modules/https-proxy-agent": { - "version": "7.0.4", + "version": "7.0.6", "inBundle": true, "license": "MIT", "dependencies": { - "agent-base": "^7.0.2", + "agent-base": "^7.1.2", "debug": "4" }, "engines": { @@ -7331,14 +7813,14 @@ } }, "node_modules/npm/node_modules/ignore-walk": { - "version": "6.0.5", + "version": "7.0.0", "inBundle": true, "license": "ISC", "dependencies": { "minimatch": "^9.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/imurmurhash": { @@ -7349,47 +7831,35 @@ "node": ">=0.8.19" } }, - "node_modules/npm/node_modules/indent-string": { - "version": "4.0.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/npm/node_modules/ini": { - "version": "4.1.2", + "version": "5.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/init-package-json": { - "version": "6.0.3", + "version": "7.0.2", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/package-json": "^5.0.0", - "npm-package-arg": "^11.0.0", - "promzard": "^1.0.0", - "read": "^3.0.1", + "@npmcli/package-json": "^6.0.0", + "npm-package-arg": "^12.0.0", + "promzard": "^2.0.0", + "read": "^4.0.0", "semver": "^7.3.5", "validate-npm-package-license": "^3.0.4", - "validate-npm-package-name": "^5.0.0" + "validate-npm-package-name": "^6.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/ip-address": { - "version": "9.0.5", + "version": "10.1.0", "inBundle": true, "license": "MIT", - "dependencies": { - "jsbn": "1.1.0", - "sprintf-js": "^1.1.3" - }, "engines": { "node": ">= 12" } @@ -7406,27 +7876,16 @@ } }, "node_modules/npm/node_modules/is-cidr": { - "version": "5.0.5", + "version": "5.1.1", "inBundle": true, "license": "BSD-2-Clause", "dependencies": { - "cidr-regex": "^4.0.4" + "cidr-regex": "^4.1.1" }, "engines": { "node": ">=14" } }, - "node_modules/npm/node_modules/is-core-module": { - "version": "2.13.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/npm/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "inBundle": true, @@ -7435,26 +7894,18 @@ "node": ">=8" } }, - "node_modules/npm/node_modules/is-lambda": { - "version": "1.0.1", - "inBundle": true, - "license": "MIT" - }, "node_modules/npm/node_modules/isexe": { "version": "2.0.0", "inBundle": true, "license": "ISC" }, "node_modules/npm/node_modules/jackspeak": { - "version": "2.3.6", + "version": "3.4.3", "inBundle": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, - "engines": { - "node": ">=14" - }, "funding": { "url": "https://github.com/sponsors/isaacs" }, @@ -7462,17 +7913,12 @@ "@pkgjs/parseargs": "^0.11.0" } }, - "node_modules/npm/node_modules/jsbn": { - "version": "1.1.0", - "inBundle": true, - "license": "MIT" - }, "node_modules/npm/node_modules/json-parse-even-better-errors": { - "version": "3.0.2", + "version": "4.0.0", "inBundle": true, "license": "MIT", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/json-stringify-nice": { @@ -7502,196 +7948,192 @@ "license": "MIT" }, "node_modules/npm/node_modules/libnpmaccess": { - "version": "8.0.6", + "version": "9.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "npm-package-arg": "^11.0.2", - "npm-registry-fetch": "^17.0.1" + "npm-package-arg": "^12.0.0", + "npm-registry-fetch": "^18.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmdiff": { - "version": "6.1.2", + "version": "7.0.5", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/arborist": "^7.5.2", - "@npmcli/installed-package-contents": "^2.1.0", + "@npmcli/arborist": "^8.0.5", + "@npmcli/installed-package-contents": "^3.0.0", "binary-extensions": "^2.3.0", "diff": "^5.1.0", "minimatch": "^9.0.4", - "npm-package-arg": "^11.0.2", - "pacote": "^18.0.6", - "tar": "^6.2.1" + "npm-package-arg": "^12.0.0", + "pacote": "^19.0.0", + "tar": "^7.5.11" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmexec": { - "version": "8.1.1", + "version": "9.0.5", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/arborist": "^7.5.2", - "@npmcli/run-script": "^8.1.0", + "@npmcli/arborist": "^8.0.5", + "@npmcli/run-script": "^9.0.1", "ci-info": "^4.0.0", - "npm-package-arg": "^11.0.2", - "pacote": "^18.0.6", - "proc-log": "^4.2.0", - "read": "^3.0.1", - "read-package-json-fast": "^3.0.2", + "npm-package-arg": "^12.0.0", + "pacote": "^19.0.0", + "proc-log": "^5.0.0", + "read": "^4.0.0", + "read-package-json-fast": "^4.0.0", "semver": "^7.3.7", "walk-up-path": "^3.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmfund": { - "version": "5.0.10", + "version": "6.0.5", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/arborist": "^7.5.2" + "@npmcli/arborist": "^8.0.5" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmhook": { - "version": "10.0.5", + "version": "11.0.0", "inBundle": true, "license": "ISC", "dependencies": { "aproba": "^2.0.0", - "npm-registry-fetch": "^17.0.1" + "npm-registry-fetch": "^18.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmorg": { - "version": "6.0.6", + "version": "7.0.0", "inBundle": true, "license": "ISC", "dependencies": { "aproba": "^2.0.0", - "npm-registry-fetch": "^17.0.1" + "npm-registry-fetch": "^18.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmpack": { - "version": "7.0.2", + "version": "8.0.5", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/arborist": "^7.5.2", - "@npmcli/run-script": "^8.1.0", - "npm-package-arg": "^11.0.2", - "pacote": "^18.0.6" + "@npmcli/arborist": "^8.0.5", + "@npmcli/run-script": "^9.0.1", + "npm-package-arg": "^12.0.0", + "pacote": "^19.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmpublish": { - "version": "9.0.8", + "version": "10.0.2", "inBundle": true, "license": "ISC", "dependencies": { "ci-info": "^4.0.0", - "normalize-package-data": "^6.0.1", - "npm-package-arg": "^11.0.2", - "npm-registry-fetch": "^17.0.1", - "proc-log": "^4.2.0", + "normalize-package-data": "^7.0.0", + "npm-package-arg": "^12.0.0", + "npm-registry-fetch": "^18.0.1", + "proc-log": "^5.0.0", "semver": "^7.3.7", - "sigstore": "^2.2.0", - "ssri": "^10.0.6" + "sigstore": "^3.0.0", + "ssri": "^12.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmsearch": { - "version": "7.0.5", + "version": "8.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "npm-registry-fetch": "^17.0.1" + "npm-registry-fetch": "^18.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmteam": { - "version": "6.0.5", + "version": "7.0.0", "inBundle": true, "license": "ISC", "dependencies": { "aproba": "^2.0.0", - "npm-registry-fetch": "^17.0.1" + "npm-registry-fetch": "^18.0.1" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/libnpmversion": { - "version": "6.0.2", + "version": "7.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/git": "^5.0.7", - "@npmcli/run-script": "^8.1.0", - "json-parse-even-better-errors": "^3.0.2", - "proc-log": "^4.2.0", + "@npmcli/git": "^6.0.1", + "@npmcli/run-script": "^9.0.1", + "json-parse-even-better-errors": "^4.0.0", + "proc-log": "^5.0.0", "semver": "^7.3.7" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/lru-cache": { - "version": "10.2.2", + "version": "10.4.3", "inBundle": true, - "license": "ISC", - "engines": { - "node": "14 || >=16.14" - } + "license": "ISC" }, "node_modules/npm/node_modules/make-fetch-happen": { - "version": "13.0.1", + "version": "14.0.3", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/agent": "^2.0.0", - "cacache": "^18.0.0", + "@npmcli/agent": "^3.0.0", + "cacache": "^19.0.1", "http-cache-semantics": "^4.1.1", - "is-lambda": "^1.0.1", "minipass": "^7.0.2", - "minipass-fetch": "^3.0.0", + "minipass-fetch": "^4.0.0", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", - "negotiator": "^0.6.3", - "proc-log": "^4.2.0", + "negotiator": "^1.0.0", + "proc-log": "^5.0.0", "promise-retry": "^2.0.1", - "ssri": "^10.0.0" + "ssri": "^12.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/minimatch": { - "version": "9.0.4", + "version": "9.0.9", "inBundle": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -7701,9 +8143,9 @@ } }, "node_modules/npm/node_modules/minipass": { - "version": "7.1.1", + "version": "7.1.3", "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" } @@ -7720,16 +8162,16 @@ } }, "node_modules/npm/node_modules/minipass-fetch": { - "version": "3.0.5", + "version": "4.0.1", "inBundle": true, "license": "MIT", "dependencies": { "minipass": "^7.0.3", "minipass-sized": "^1.0.3", - "minizlib": "^2.1.2" + "minizlib": "^3.0.1" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" }, "optionalDependencies": { "encoding": "^0.1.13" @@ -7757,25 +8199,10 @@ "node": ">=8" } }, - "node_modules/npm/node_modules/minipass-json-stream": { - "version": "1.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "jsonparse": "^1.3.1", - "minipass": "^3.0.0" - } - }, - "node_modules/npm/node_modules/minipass-json-stream/node_modules/minipass": { - "version": "3.3.6", + "node_modules/npm/node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", "inBundle": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } + "license": "ISC" }, "node_modules/npm/node_modules/minipass-pipeline": { "version": "1.2.4", @@ -7799,6 +8226,11 @@ "node": ">=8" } }, + "node_modules/npm/node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "inBundle": true, + "license": "ISC" + }, "node_modules/npm/node_modules/minipass-sized": { "version": "1.0.3", "inBundle": true, @@ -7821,38 +8253,20 @@ "node": ">=8" } }, - "node_modules/npm/node_modules/minizlib": { - "version": "2.1.2", - "inBundle": true, - "license": "MIT", - "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/npm/node_modules/minizlib/node_modules/minipass": { - "version": "3.3.6", + "node_modules/npm/node_modules/minipass-sized/node_modules/yallist": { + "version": "4.0.0", "inBundle": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } + "license": "ISC" }, - "node_modules/npm/node_modules/mkdirp": { - "version": "1.0.4", + "node_modules/npm/node_modules/minizlib": { + "version": "3.1.0", "inBundle": true, "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" + "dependencies": { + "minipass": "^7.1.2" }, "engines": { - "node": ">=10" + "node": ">= 18" } }, "node_modules/npm/node_modules/ms": { @@ -7861,15 +8275,15 @@ "license": "MIT" }, "node_modules/npm/node_modules/mute-stream": { - "version": "1.0.0", + "version": "2.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/negotiator": { - "version": "0.6.3", + "version": "1.0.0", "inBundle": true, "license": "MIT", "engines": { @@ -7877,234 +8291,227 @@ } }, "node_modules/npm/node_modules/node-gyp": { - "version": "10.1.0", + "version": "11.5.0", "inBundle": true, "license": "MIT", "dependencies": { "env-paths": "^2.2.0", "exponential-backoff": "^3.1.1", - "glob": "^10.3.10", "graceful-fs": "^4.2.6", - "make-fetch-happen": "^13.0.0", - "nopt": "^7.0.0", - "proc-log": "^3.0.0", + "make-fetch-happen": "^14.0.3", + "nopt": "^8.0.0", + "proc-log": "^5.0.0", "semver": "^7.3.5", - "tar": "^6.1.2", - "which": "^4.0.0" + "tar": "^7.4.3", + "tinyglobby": "^0.2.12", + "which": "^5.0.0" }, "bin": { "node-gyp": "bin/node-gyp.js" }, "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/npm/node_modules/node-gyp/node_modules/proc-log": { - "version": "3.0.0", - "inBundle": true, - "license": "ISC", - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/nopt": { - "version": "7.2.1", + "version": "8.1.0", "inBundle": true, "license": "ISC", "dependencies": { - "abbrev": "^2.0.0" + "abbrev": "^3.0.0" }, "bin": { "nopt": "bin/nopt.js" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/normalize-package-data": { - "version": "6.0.1", + "version": "7.0.1", "inBundle": true, "license": "BSD-2-Clause", "dependencies": { - "hosted-git-info": "^7.0.0", - "is-core-module": "^2.8.1", + "hosted-git-info": "^8.0.0", "semver": "^7.3.5", "validate-npm-package-license": "^3.0.4" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-audit-report": { - "version": "5.0.0", + "version": "6.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-bundled": { - "version": "3.0.1", + "version": "4.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "npm-normalize-package-bin": "^3.0.0" + "npm-normalize-package-bin": "^4.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-install-checks": { - "version": "6.3.0", + "version": "7.1.2", "inBundle": true, "license": "BSD-2-Clause", "dependencies": { "semver": "^7.1.1" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-normalize-package-bin": { - "version": "3.0.1", + "version": "4.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-package-arg": { - "version": "11.0.2", + "version": "12.0.2", "inBundle": true, "license": "ISC", "dependencies": { - "hosted-git-info": "^7.0.0", - "proc-log": "^4.0.0", + "hosted-git-info": "^8.0.0", + "proc-log": "^5.0.0", "semver": "^7.3.5", - "validate-npm-package-name": "^5.0.0" + "validate-npm-package-name": "^6.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-packlist": { - "version": "8.0.2", + "version": "9.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "ignore-walk": "^6.0.4" + "ignore-walk": "^7.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-pick-manifest": { - "version": "9.0.1", + "version": "10.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "npm-install-checks": "^6.0.0", - "npm-normalize-package-bin": "^3.0.0", - "npm-package-arg": "^11.0.0", + "npm-install-checks": "^7.1.0", + "npm-normalize-package-bin": "^4.0.0", + "npm-package-arg": "^12.0.0", "semver": "^7.3.5" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-profile": { - "version": "10.0.0", + "version": "11.0.1", "inBundle": true, "license": "ISC", "dependencies": { - "npm-registry-fetch": "^17.0.1", - "proc-log": "^4.0.0" + "npm-registry-fetch": "^18.0.0", + "proc-log": "^5.0.0" }, "engines": { - "node": ">=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-registry-fetch": { - "version": "17.0.1", + "version": "18.0.2", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/redact": "^2.0.0", - "make-fetch-happen": "^13.0.0", + "@npmcli/redact": "^3.0.0", + "jsonparse": "^1.3.1", + "make-fetch-happen": "^14.0.0", "minipass": "^7.0.2", - "minipass-fetch": "^3.0.0", - "minipass-json-stream": "^1.0.1", - "minizlib": "^2.1.2", - "npm-package-arg": "^11.0.0", - "proc-log": "^4.0.0" + "minipass-fetch": "^4.0.0", + "minizlib": "^3.0.1", + "npm-package-arg": "^12.0.0", + "proc-log": "^5.0.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/npm-user-validate": { - "version": "2.0.1", + "version": "3.0.0", "inBundle": true, "license": "BSD-2-Clause", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/p-map": { - "version": "4.0.0", + "version": "7.0.4", "inBundle": true, "license": "MIT", - "dependencies": { - "aggregate-error": "^3.0.0" - }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/npm/node_modules/package-json-from-dist": { + "version": "1.0.1", + "inBundle": true, + "license": "BlueOak-1.0.0" + }, "node_modules/npm/node_modules/pacote": { - "version": "18.0.6", + "version": "19.0.2", "inBundle": true, "license": "ISC", "dependencies": { - "@npmcli/git": "^5.0.0", - "@npmcli/installed-package-contents": "^2.0.1", - "@npmcli/package-json": "^5.1.0", - "@npmcli/promise-spawn": "^7.0.0", - "@npmcli/run-script": "^8.0.0", - "cacache": "^18.0.0", + "@npmcli/git": "^6.0.0", + "@npmcli/installed-package-contents": "^3.0.0", + "@npmcli/package-json": "^6.0.0", + "@npmcli/promise-spawn": "^8.0.0", + "@npmcli/run-script": "^9.0.0", + "cacache": "^19.0.0", "fs-minipass": "^3.0.0", "minipass": "^7.0.2", - "npm-package-arg": "^11.0.0", - "npm-packlist": "^8.0.0", - "npm-pick-manifest": "^9.0.0", - "npm-registry-fetch": "^17.0.0", - "proc-log": "^4.0.0", + "npm-package-arg": "^12.0.0", + "npm-packlist": "^9.0.0", + "npm-pick-manifest": "^10.0.0", + "npm-registry-fetch": "^18.0.0", + "proc-log": "^5.0.0", "promise-retry": "^2.0.1", - "sigstore": "^2.2.0", - "ssri": "^10.0.0", - "tar": "^6.1.11" + "sigstore": "^3.0.0", + "ssri": "^12.0.0", + "tar": "^7.5.10" }, "bin": { "pacote": "bin/index.js" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/parse-conflict-json": { - "version": "3.0.1", + "version": "4.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "json-parse-even-better-errors": "^3.0.0", + "json-parse-even-better-errors": "^4.0.0", "just-diff": "^6.0.0", "just-diff-apply": "^5.2.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/path-key": { @@ -8130,8 +8537,20 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/npm/node_modules/picomatch": { + "version": "4.0.3", + "inBundle": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/npm/node_modules/postcss-selector-parser": { - "version": "6.0.16", + "version": "7.1.1", "inBundle": true, "license": "MIT", "dependencies": { @@ -8143,19 +8562,19 @@ } }, "node_modules/npm/node_modules/proc-log": { - "version": "4.2.0", + "version": "5.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/proggy": { - "version": "2.0.0", + "version": "3.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/promise-all-reject-late": { @@ -8167,18 +8586,13 @@ } }, "node_modules/npm/node_modules/promise-call-limit": { - "version": "3.0.1", + "version": "3.0.2", "inBundle": true, "license": "ISC", "funding": { "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/npm/node_modules/promise-inflight": { - "version": "1.0.1", - "inBundle": true, - "license": "ISC" - }, "node_modules/npm/node_modules/promise-retry": { "version": "2.0.1", "inBundle": true, @@ -8192,14 +8606,14 @@ } }, "node_modules/npm/node_modules/promzard": { - "version": "1.0.2", + "version": "2.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "read": "^3.0.1" + "read": "^4.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/qrcode-terminal": { @@ -8210,34 +8624,34 @@ } }, "node_modules/npm/node_modules/read": { - "version": "3.0.1", + "version": "4.1.0", "inBundle": true, "license": "ISC", "dependencies": { - "mute-stream": "^1.0.0" + "mute-stream": "^2.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/read-cmd-shim": { - "version": "4.0.0", + "version": "5.0.0", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/read-package-json-fast": { - "version": "3.0.2", + "version": "4.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "json-parse-even-better-errors": "^3.0.0", - "npm-normalize-package-bin": "^3.0.0" + "json-parse-even-better-errors": "^4.0.0", + "npm-normalize-package-bin": "^4.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/retry": { @@ -8255,7 +8669,7 @@ "optional": true }, "node_modules/npm/node_modules/semver": { - "version": "7.6.2", + "version": "7.7.4", "inBundle": true, "license": "ISC", "bin": { @@ -8296,19 +8710,19 @@ } }, "node_modules/npm/node_modules/sigstore": { - "version": "2.3.0", + "version": "3.1.0", "inBundle": true, "license": "Apache-2.0", "dependencies": { - "@sigstore/bundle": "^2.3.1", - "@sigstore/core": "^1.0.0", - "@sigstore/protobuf-specs": "^0.3.1", - "@sigstore/sign": "^2.3.0", - "@sigstore/tuf": "^2.3.1", - "@sigstore/verify": "^1.2.0" + "@sigstore/bundle": "^3.1.0", + "@sigstore/core": "^2.0.0", + "@sigstore/protobuf-specs": "^0.4.0", + "@sigstore/sign": "^3.1.0", + "@sigstore/tuf": "^3.1.0", + "@sigstore/verify": "^2.1.0" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/smart-buffer": { @@ -8321,11 +8735,11 @@ } }, "node_modules/npm/node_modules/socks": { - "version": "2.8.3", + "version": "2.8.7", "inBundle": true, "license": "MIT", "dependencies": { - "ip-address": "^9.0.5", + "ip-address": "^10.0.1", "smart-buffer": "^4.2.0" }, "engines": { @@ -8334,13 +8748,13 @@ } }, "node_modules/npm/node_modules/socks-proxy-agent": { - "version": "8.0.3", + "version": "8.0.5", "inBundle": true, "license": "MIT", "dependencies": { - "agent-base": "^7.1.1", + "agent-base": "^7.1.2", "debug": "^4.3.4", - "socks": "^2.7.1" + "socks": "^2.8.3" }, "engines": { "node": ">= 14" @@ -8379,24 +8793,19 @@ } }, "node_modules/npm/node_modules/spdx-license-ids": { - "version": "3.0.17", + "version": "3.0.23", "inBundle": true, "license": "CC0-1.0" }, - "node_modules/npm/node_modules/sprintf-js": { - "version": "1.1.3", - "inBundle": true, - "license": "BSD-3-Clause" - }, "node_modules/npm/node_modules/ssri": { - "version": "10.0.6", + "version": "12.0.0", "inBundle": true, "license": "ISC", "dependencies": { "minipass": "^7.0.3" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/string-width": { @@ -8461,49 +8870,18 @@ } }, "node_modules/npm/node_modules/tar": { - "version": "6.2.1", + "version": "7.5.11", "inBundle": true, - "license": "ISC", - "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/npm/node_modules/tar/node_modules/fs-minipass": { - "version": "2.1.0", - "inBundle": true, - "license": "ISC", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/npm/node_modules/tar/node_modules/fs-minipass/node_modules/minipass": { - "version": "3.3.6", - "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "yallist": "^4.0.0" + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" }, "engines": { - "node": ">=8" - } - }, - "node_modules/npm/node_modules/tar/node_modules/minipass": { - "version": "5.0.0", - "inBundle": true, - "license": "ISC", - "engines": { - "node": ">=8" + "node": ">=18" } }, "node_modules/npm/node_modules/text-table": { @@ -8516,6 +8894,21 @@ "inBundle": true, "license": "MIT" }, + "node_modules/npm/node_modules/tinyglobby": { + "version": "0.2.15", + "inBundle": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, "node_modules/npm/node_modules/treeverse": { "version": "3.0.0", "inBundle": true, @@ -8525,38 +8918,50 @@ } }, "node_modules/npm/node_modules/tuf-js": { - "version": "2.2.1", + "version": "3.1.0", "inBundle": true, "license": "MIT", "dependencies": { - "@tufjs/models": "2.0.1", - "debug": "^4.3.4", - "make-fetch-happen": "^13.0.1" + "@tufjs/models": "3.0.1", + "debug": "^4.4.1", + "make-fetch-happen": "^14.0.3" }, "engines": { - "node": "^16.14.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/tuf-js/node_modules/@tufjs/models": { + "version": "3.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "@tufjs/canonical-json": "2.0.0", + "minimatch": "^9.0.5" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/unique-filename": { - "version": "3.0.0", + "version": "4.0.0", "inBundle": true, "license": "ISC", "dependencies": { - "unique-slug": "^4.0.0" + "unique-slug": "^5.0.0" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/unique-slug": { - "version": "4.0.0", + "version": "5.0.0", "inBundle": true, "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/util-deprecate": { @@ -8583,11 +8988,11 @@ } }, "node_modules/npm/node_modules/validate-npm-package-name": { - "version": "5.0.1", + "version": "6.0.2", "inBundle": true, "license": "ISC", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/walk-up-path": { @@ -8596,7 +9001,7 @@ "license": "ISC" }, "node_modules/npm/node_modules/which": { - "version": "4.0.0", + "version": "5.0.0", "inBundle": true, "license": "ISC", "dependencies": { @@ -8606,15 +9011,15 @@ "node-which": "bin/which.js" }, "engines": { - "node": "^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/which/node_modules/isexe": { - "version": "3.1.1", + "version": "3.1.5", "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "engines": { - "node": ">=16" + "node": ">=18" } }, "node_modules/npm/node_modules/wrap-ansi": { @@ -8665,7 +9070,7 @@ } }, "node_modules/npm/node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "6.0.1", + "version": "6.2.2", "inBundle": true, "license": "MIT", "engines": { @@ -8697,11 +9102,11 @@ } }, "node_modules/npm/node_modules/wrap-ansi/node_modules/strip-ansi": { - "version": "7.1.0", + "version": "7.2.0", "inBundle": true, "license": "MIT", "dependencies": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^6.2.2" }, "engines": { "node": ">=12" @@ -8711,7 +9116,7 @@ } }, "node_modules/npm/node_modules/write-file-atomic": { - "version": "5.0.1", + "version": "6.0.0", "inBundle": true, "license": "ISC", "dependencies": { @@ -8719,13 +9124,16 @@ "signal-exit": "^4.0.1" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/npm/node_modules/yallist": { - "version": "4.0.0", + "version": "5.0.0", "inBundle": true, - "license": "ISC" + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } }, "node_modules/object-assign": { "version": "4.1.1", @@ -8744,10 +9152,14 @@ } }, "node_modules/object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -8762,14 +9174,17 @@ } }, "node_modules/object.assign": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", - "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.7.tgz", + "integrity": "sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", "define-properties": "^1.2.1", - "has-symbols": "^1.0.3", + "es-object-atoms": "^1.0.0", + "has-symbols": "^1.1.0", "object-keys": "^1.1.1" }, "engines": { @@ -8780,14 +9195,16 @@ } }, "node_modules/object.entries": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.8.tgz", - "integrity": "sha512-cmopxi8VwRIAw/fkijJohSfpef5PdN0pMQJN6VC/ZKvn0LIknWD8KtgY6KlQdEc4tIjcQ3HxSMmnvtzIscdaYQ==", + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.9.tgz", + "integrity": "sha512-8u/hfXFRBD1O0hPUjioLhoWFHRmt6tKA4/vZPyckBr18l1KE9uHrFaFaUi8MDRTpi4uak2goyPTSNJLXX2k2Hw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", "define-properties": "^1.2.1", - "es-object-atoms": "^1.0.0" + "es-object-atoms": "^1.1.1" }, "engines": { "node": ">= 0.4" @@ -8825,30 +9242,15 @@ "node": ">= 0.4" } }, - "node_modules/object.hasown": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.4.tgz", - "integrity": "sha512-FZ9LZt9/RHzGySlBARE3VF+gE26TxR38SdmqOqliuTnl9wrKulaQs+4dee1V+Io8VfxqzAfHu6YuRgUy8OHoTg==", - "dev": true, - "dependencies": { - "define-properties": "^1.2.1", - "es-abstract": "^1.23.2", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/object.values": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.0.tgz", - "integrity": "sha512-yBYjY9QX2hnRmZHAjG/f13MzmBzxzYgQhFrke06TTyKY5zSTEqkOeukBzIdVA3j3ulu8Qa3MbVFShV7T2RmGtQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.1.tgz", + "integrity": "sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" }, @@ -8884,6 +9286,24 @@ "node": ">= 0.8.0" } }, + "node_modules/own-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz", + "integrity": "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.6", + "object-keys": "^1.1.1", + "safe-push-apply": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8914,6 +9334,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -9012,24 +9438,17 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/picocolors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", - "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -9054,10 +9473,11 @@ } }, "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -9080,6 +9500,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "nanoid": "^3.3.7", "picocolors": "^1.0.0", @@ -9242,9 +9663,10 @@ } }, "node_modules/prebuild-install/node_modules/tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -9292,9 +9714,10 @@ } }, "node_modules/prismjs": { - "version": "1.29.0", - "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", - "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==", + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", + "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "license": "MIT", "engines": { "node": ">=6" } @@ -9360,11 +9783,6 @@ } ] }, - "node_modules/queue-tick": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", - "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" - }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -9391,6 +9809,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -9415,6 +9834,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -9669,18 +10089,20 @@ } }, "node_modules/reflect.getprototypeof": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz", - "integrity": "sha512-fmfw4XgoDke3kdI6h4xcUz1dG8uaiv5q9gcEwLS4Pnth2kxT+GZ7YehS1JTMGBQmtV7Y4GFGbs2re2NqhdozUg==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", + "integrity": "sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "define-properties": "^1.2.1", - "es-abstract": "^1.23.1", + "es-abstract": "^1.23.9", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "globalthis": "^1.0.3", - "which-builtin-type": "^1.1.3" + "es-object-atoms": "^1.0.0", + "get-intrinsic": "^1.2.7", + "get-proto": "^1.0.1", + "which-builtin-type": "^1.2.1" }, "engines": { "node": ">= 0.4" @@ -9717,21 +10139,19 @@ "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz", "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==" }, - "node_modules/regenerator-runtime": { - "version": "0.14.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", - "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" - }, "node_modules/regexp.prototype.flags": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", - "integrity": "sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", + "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-errors": "^1.3.0", - "set-function-name": "^2.0.1" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "set-function-name": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -9930,14 +10350,16 @@ } }, "node_modules/safe-array-concat": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", - "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz", + "integrity": "sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", - "get-intrinsic": "^1.2.4", - "has-symbols": "^1.0.3", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", + "get-intrinsic": "^1.3.0", + "has-symbols": "^1.1.0", "isarray": "^2.0.5" }, "engines": { @@ -9966,15 +10388,33 @@ } ] }, + "node_modules/safe-push-apply": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", + "integrity": "sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "isarray": "^2.0.5" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safe-regex-test": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz", - "integrity": "sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.2", "es-errors": "^1.3.0", - "is-regex": "^1.1.4" + "is-regex": "^1.2.1" }, "engines": { "node": ">= 0.4" @@ -9992,9 +10432,10 @@ } }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -10024,6 +10465,7 @@ "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", "dev": true, + "license": "MIT", "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -10034,6 +10476,21 @@ "node": ">= 0.4" } }, + "node_modules/set-proto": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/set-proto/-/set-proto-1.0.0.tgz", + "integrity": "sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/shallowequal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", @@ -10081,15 +10538,73 @@ } }, "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" }, "engines": { "node": ">= 0.4" @@ -10160,15 +10675,6 @@ "is-arrayish": "^0.3.1" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/source-map-js": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", @@ -10186,6 +10692,20 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/stop-iteration-iterator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", + "integrity": "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "internal-slot": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -10195,15 +10715,14 @@ } }, "node_modules/streamx": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz", - "integrity": "sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==", + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "license": "MIT", "dependencies": { - "fast-fifo": "^1.1.0", - "queue-tick": "^1.0.1" - }, - "optionalDependencies": { - "bare-events": "^2.2.0" + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" } }, "node_modules/string_decoder": { @@ -10218,6 +10737,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -10235,6 +10755,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -10247,12 +10768,14 @@ "node_modules/string-width-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" }, "node_modules/string-width/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -10261,11 +10784,12 @@ } }, "node_modules/string-width/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", "dependencies": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^6.2.2" }, "engines": { "node": ">=12" @@ -10274,24 +10798,41 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/string.prototype.matchall": { - "version": "4.0.11", - "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.11.tgz", - "integrity": "sha512-NUdh0aDavY2og7IbBPenWqR9exH+E26Sv8e0/eTe1tltDGZL+GtBkDAnnyBtmekfK6/Dq3MkcGtzXFEd1LQrtg==", + "node_modules/string.prototype.includes": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.includes/-/string.prototype.includes-2.0.1.tgz", + "integrity": "sha512-o7+c9bW6zpAdJHTtujeePODAhkuicdAryFsfVKwA+wGw89wJ4GTY484WTucM9hLtDEOpOvI+aHnzqnC5lHp4Rg==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", - "es-abstract": "^1.23.2", + "es-abstract": "^1.23.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/string.prototype.matchall": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz", + "integrity": "sha512-6CC9uyBL+/48dYizRf7H7VAYCMCNTBeM78x/VTUe9bFEaxBepPJDa1Ow99LqI/1yF7kuy7Q3cQsYMrcjGUcskA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.6", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-symbols": "^1.0.3", - "internal-slot": "^1.0.7", - "regexp.prototype.flags": "^1.5.2", + "get-intrinsic": "^1.2.6", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "internal-slot": "^1.1.0", + "regexp.prototype.flags": "^1.5.3", "set-function-name": "^2.0.2", - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -10300,16 +10841,31 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/string.prototype.repeat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-1.0.0.tgz", + "integrity": "sha512-0u/TldDbKD8bFCQ/4f5+mNRrXwZ8hg2w7ZR8wa16e8z9XpePWl3eGEcUD0OXpEH/VJH/2G3gjUtR3ZOiBe2S/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, "node_modules/string.prototype.trim": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz", - "integrity": "sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw==", + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz", + "integrity": "sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.2", + "define-data-property": "^1.1.4", "define-properties": "^1.2.1", - "es-abstract": "^1.23.0", - "es-object-atoms": "^1.0.0" + "es-abstract": "^1.23.5", + "es-object-atoms": "^1.0.0", + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -10319,15 +10875,20 @@ } }, "node_modules/string.prototype.trimend": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz", - "integrity": "sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.9.tgz", + "integrity": "sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.2", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -10378,6 +10939,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, @@ -10418,6 +10980,7 @@ "version": "5.3.11", "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-5.3.11.tgz", "integrity": "sha512-uuzIIfnVkagcVHv9nE0VPlHPSCmXIUGKfJ42LNjxCCTDTL5sgnJ8Z7GZBq0EnLYGln77tPpEpExt2+qa+cZqSw==", + "peer": true, "dependencies": { "@babel/helper-module-imports": "^7.0.0", "@babel/traverse": "^7.4.5", @@ -10466,6 +11029,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz", "integrity": "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==", + "license": "MIT", "dependencies": { "client-only": "0.0.1" }, @@ -10563,6 +11127,7 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.3.tgz", "integrity": "sha512-U7sxQk/n397Bmx4JHbJx/iSOOv5G+II3f1kpLpY2QeUv5DcPdcTsYLlusZfq1NthHS1c1cZoyFmmkex1rzke0A==", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -10617,16 +11182,17 @@ } }, "node_modules/tar-fs": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", - "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "license": "MIT", "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" }, "optionalDependencies": { - "bare-fs": "^2.1.1", - "bare-path": "^2.1.0" + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" } }, "node_modules/tar-stream": { @@ -10639,6 +11205,25 @@ "streamx": "^2.15.0" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "optional": true, + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -10679,18 +11264,60 @@ "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "node_modules/tinyglobby": { + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, "engines": { - "node": ">=4" + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -10722,15 +11349,16 @@ } }, "node_modules/ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", "dev": true, + "license": "MIT", "engines": { - "node": ">=16" + "node": ">=18.12" }, "peerDependencies": { - "typescript": ">=4.2.0" + "typescript": ">=4.8.4" } }, "node_modules/ts-interface-checker": { @@ -10763,9 +11391,10 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/tunnel-agent": { "version": "0.6.0", @@ -10803,30 +11432,32 @@ } }, "node_modules/typed-array-buffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz", - "integrity": "sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-typed-array": "^1.1.13" + "is-typed-array": "^1.1.14" }, "engines": { "node": ">= 0.4" } }, "node_modules/typed-array-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz", - "integrity": "sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.3.tgz", + "integrity": "sha512-BaXgOuIxz8n8pIq3e7Atg/7s+DpiYrxn4vdot3w9KbnBhcRQq6o3xemQdIfynqSeXeDrF32x+WvfzmOjPiY9lg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-proto": "^1.0.3", - "is-typed-array": "^1.1.13" + "gopd": "^1.2.0", + "has-proto": "^1.2.0", + "is-typed-array": "^1.1.14" }, "engines": { "node": ">= 0.4" @@ -10836,17 +11467,19 @@ } }, "node_modules/typed-array-byte-offset": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz", - "integrity": "sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.4.tgz", + "integrity": "sha512-bTlAFB/FBYMcuX81gbL4OcpH5PmlFHqlCCpAl8AlEzMz5k53oNDvN8p1PNOWLEmI2x4orp3raOFB51tv9X+MFQ==", "dev": true, + "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-proto": "^1.0.3", - "is-typed-array": "^1.1.13" + "gopd": "^1.2.0", + "has-proto": "^1.2.0", + "is-typed-array": "^1.1.15", + "reflect.getprototypeof": "^1.0.9" }, "engines": { "node": ">= 0.4" @@ -10856,17 +11489,18 @@ } }, "node_modules/typed-array-length": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.6.tgz", - "integrity": "sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.7.tgz", + "integrity": "sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "for-each": "^0.3.3", "gopd": "^1.0.1", - "has-proto": "^1.0.3", "is-typed-array": "^1.1.13", - "possible-typed-array-names": "^1.0.0" + "possible-typed-array-names": "^1.0.0", + "reflect.getprototypeof": "^1.0.6" }, "engines": { "node": ">= 0.4" @@ -10879,6 +11513,7 @@ "version": "5.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.3.tgz", "integrity": "sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA==", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -10888,15 +11523,19 @@ } }, "node_modules/unbox-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", - "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", + "integrity": "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", + "call-bound": "^1.0.3", "has-bigints": "^1.0.2", - "has-symbols": "^1.0.3", - "which-boxed-primitive": "^1.0.2" + "has-symbols": "^1.1.0", + "which-boxed-primitive": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -11106,6 +11745,7 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } @@ -11195,39 +11835,45 @@ } }, "node_modules/which-boxed-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", - "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz", + "integrity": "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==", "dev": true, + "license": "MIT", "dependencies": { - "is-bigint": "^1.0.1", - "is-boolean-object": "^1.1.0", - "is-number-object": "^1.0.4", - "is-string": "^1.0.5", - "is-symbol": "^1.0.3" + "is-bigint": "^1.1.0", + "is-boolean-object": "^1.2.1", + "is-number-object": "^1.1.1", + "is-string": "^1.1.1", + "is-symbol": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/which-builtin-type": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.1.3.tgz", - "integrity": "sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.2.1.tgz", + "integrity": "sha512-6iBczoX+kDQ7a3+YJBnh3T+KZRxM/iYNPXicqk66/Qfm1b93iu+yOImkg0zHbj5LNOcNv1TEADiZ0xa34B4q6Q==", "dev": true, + "license": "MIT", "dependencies": { - "function.prototype.name": "^1.1.5", - "has-tostringtag": "^1.0.0", + "call-bound": "^1.0.2", + "function.prototype.name": "^1.1.6", + "has-tostringtag": "^1.0.2", "is-async-function": "^2.0.0", - "is-date-object": "^1.0.5", - "is-finalizationregistry": "^1.0.2", + "is-date-object": "^1.1.0", + "is-finalizationregistry": "^1.1.0", "is-generator-function": "^1.0.10", - "is-regex": "^1.1.4", + "is-regex": "^1.2.1", "is-weakref": "^1.0.2", "isarray": "^2.0.5", - "which-boxed-primitive": "^1.0.2", - "which-collection": "^1.0.1", - "which-typed-array": "^1.1.9" + "which-boxed-primitive": "^1.1.0", + "which-collection": "^1.0.2", + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -11241,6 +11887,7 @@ "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.2.tgz", "integrity": "sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==", "dev": true, + "license": "MIT", "dependencies": { "is-map": "^2.0.3", "is-set": "^2.0.3", @@ -11255,15 +11902,18 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", - "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", + "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", "dev": true, + "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" }, "engines": { @@ -11286,6 +11936,7 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -11303,6 +11954,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -11318,12 +11970,14 @@ "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" }, "node_modules/wrap-ansi-cjs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -11334,9 +11988,10 @@ } }, "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -11345,9 +12000,10 @@ } }, "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -11356,11 +12012,12 @@ } }, "node_modules/wrap-ansi/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", "dependencies": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^6.2.2" }, "engines": { "node": ">=12" @@ -11377,18 +12034,21 @@ "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "peer": true + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "node_modules/yaml": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.2.tgz", - "integrity": "sha512-B3VqDZ+JAg1nZpaEmWtTXUlBneoGx6CPM9b0TENK6aoSu5t73dItudwdgmi6tHlIZZId4dZ9skcAQ2UbcyAeVA==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, "engines": { - "node": ">= 14" + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" } }, "node_modules/yocto-queue": { diff --git a/web/package.json b/web/package.json index 1ba7286d401..a9bb458b8b7 100644 --- a/web/package.json +++ b/web/package.json @@ -29,7 +29,7 @@ "js-cookie": "^3.0.5", "lodash": "^4.17.21", "mdast-util-find-and-replace": "^3.0.1", - "next": "^14.2.3", + "next": "^14.2.35", "npm": "^10.8.0", "postcss": "^8.4.31", "prismjs": "^1.29.0", @@ -52,7 +52,7 @@ "devDependencies": { "@tailwindcss/typography": "^0.5.10", "eslint": "^8.48.0", - "eslint-config-next": "^14.1.0", + "eslint-config-next": "^14.2.35", "prettier": "2.8.8" } } diff --git a/web/src/app/admin/analytics/page.tsx b/web/src/app/admin/analytics/page.tsx new file mode 100644 index 00000000000..79a101dea49 --- /dev/null +++ b/web/src/app/admin/analytics/page.tsx @@ -0,0 +1,495 @@ +"use client"; + +import { useMemo, useState } from "react"; +import useSWR from "swr"; +import { + AreaChart, + BarList, + Card, + DateRangePicker, + DateRangePickerValue, + Grid, + Metric, + Text, + Title, +} from "@tremor/react"; +import { FiBarChart2 } from "react-icons/fi"; + +import { AdminPageTitle } from "@/components/admin/Title"; +import { LoadingAnimation } from "@/components/Loading"; +import { errorHandlingFetcher } from "@/lib/fetcher"; + +// Backend response shapes — keep in sync with +// `backend/danswer/server/analytics/api.py` Pydantic models. `date` is a +// Python `datetime.date` which Pydantic serializes as a YYYY-MM-DD string. +interface QueryAnalyticsRow { + total_queries: number; + total_likes: number; + total_dislikes: number; + // Resolved-button presses on Slackbot answers — counted as a positive + // signal alongside likes for "strict NPS". Backed by + // chat_feedback.predefined_feedback = 'resolved'. + total_resolved: number; + // "I need more help" button presses — counted as a negative signal + // alongside dislikes. Backed by chat_feedback.required_followup = TRUE. + total_needs_help: number; + date: string; +} + +interface UserAnalyticsRow { + total_active_users: number; + date: string; +} + +interface DanswerbotAnalyticsRow { + total_queries: number; + auto_resolved: number; + date: string; +} + +interface TotalDocsResponse { + total_docs_indexed: number; + unique_docs: number; +} + +interface DocsPerSourceRow { + source: string; + docs_indexed: number; +} + +interface SlackChannelsResponse { + total_configs: number; + enabled_channels: number; +} + +type Granularity = "day" | "month"; + +const DEFAULT_LOOKBACK_DAYS = 30; + +function defaultRange(): DateRangePickerValue { + const now = new Date(); + const from = new Date(now); + from.setDate(from.getDate() - DEFAULT_LOOKBACK_DAYS); + return { from, to: now }; +} + +function buildURL(path: string, range?: DateRangePickerValue): string { + // Skip the URL params if the picker is empty — the backend defaults + // to a 30-day window when start/end are absent. + if (!range) return `/api${path}`; + const params = new URLSearchParams(); + if (range.from) params.set("start", range.from.toISOString()); + if (range.to) params.set("end", range.to.toISOString()); + const qs = params.toString(); + return qs ? `/api${path}?${qs}` : `/api${path}`; +} + +function monthKey(isoDate: string): string { + // "2026-05-14" → "2026-05". Pydantic always serializes dates this way, + // so a slice is sufficient. + return isoDate.slice(0, 7); +} + +// Bucket daily rows into monthly. Numeric fields are summed unless their +// name is in `peakFields`, in which case we take the max within the +// month — needed for distinct-counts (e.g. active users) where summing +// would double-count people who logged in on multiple days. +function bucketToMonth & { date: string }>( + rows: R[], + peakFields: ReadonlySet = new Set() +): R[] { + const buckets = new Map(); + for (const row of rows) { + const key = monthKey(row.date); + const existing = buckets.get(key); + if (!existing) { + buckets.set(key, { ...row, date: key }); + continue; + } + const merged: Record = { ...existing }; + for (const [k, v] of Object.entries(row)) { + if (k === "date") continue; + if (typeof v !== "number") continue; + const prev = (existing as Record)[k]; + if (typeof prev !== "number") { + merged[k] = v; + continue; + } + merged[k] = peakFields.has(k) ? Math.max(prev, v) : prev + v; + } + buckets.set(key, merged as R); + } + return Array.from(buckets.values()).sort((a, b) => + a.date < b.date ? -1 : a.date > b.date ? 1 : 0 + ); +} + +export default function AnalyticsPage() { + const [range, setRange] = useState(defaultRange()); + const [granularity, setGranularity] = useState("day"); + + const swrOpts = { keepPreviousData: true }; + + // Time-series endpoints — driven by the date range picker. + const { + data: queryData, + isLoading: queryLoading, + error: queryErr, + } = useSWR( + buildURL("/analytics/admin/query", range), + errorHandlingFetcher, + swrOpts + ); + + const { + data: userData, + isLoading: userLoading, + error: userErr, + } = useSWR( + buildURL("/analytics/admin/user", range), + errorHandlingFetcher, + swrOpts + ); + + const { + data: botData, + isLoading: botLoading, + error: botErr, + } = useSWR( + buildURL("/analytics/admin/danswerbot", range), + errorHandlingFetcher, + swrOpts + ); + + // Snapshot endpoints — independent of date range, refresh on mount only. + const { data: totalDocs, error: totalDocsErr } = useSWR( + buildURL("/analytics/admin/total-docs"), + errorHandlingFetcher, + swrOpts + ); + const { data: docsBySource, error: docsBySourceErr } = useSWR< + DocsPerSourceRow[] + >( + buildURL("/analytics/admin/docs-per-source"), + errorHandlingFetcher, + swrOpts + ); + const { data: slackChannels, error: slackChannelsErr } = + useSWR( + buildURL("/analytics/admin/slack-channels"), + errorHandlingFetcher, + swrOpts + ); + + const isInitialLoading = + (queryLoading && !queryData) || + (userLoading && !userData) || + (botLoading && !botData); + const hasError = + queryErr || + userErr || + botErr || + totalDocsErr || + docsBySourceErr || + slackChannelsErr; + + // KPI aggregates — collapse the time-series data down to single + // numbers for the cards above the charts. + const kpis = useMemo(() => { + const totalQueries = (queryData ?? []).reduce( + (s, r) => s + r.total_queries, + 0 + ); + const totalLikes = (queryData ?? []).reduce((s, r) => s + r.total_likes, 0); + const totalDislikes = (queryData ?? []).reduce( + (s, r) => s + r.total_dislikes, + 0 + ); + const totalResolved = (queryData ?? []).reduce( + (s, r) => s + (r.total_resolved ?? 0), + 0 + ); + const totalNeedsHelp = (queryData ?? []).reduce( + (s, r) => s + (r.total_needs_help ?? 0), + 0 + ); + + const positivity = + totalLikes + totalDislikes > 0 + ? Math.round((totalLikes / (totalLikes + totalDislikes)) * 100) + : null; + + // Strict NPS — explicit signals only. Promoters = likes + resolved + // (the Slackbot's "I'm all set!" / "Mark Resolved" buttons). + // Detractors = dislikes + needs_help (the "I need more help" button). + // Excludes messages with no feedback row at all (silent users). + // Range -100..+100. NPS is well-defined only when at least one + // explicit signal exists in the range. + const strictPromoters = totalLikes + totalResolved; + const strictDetractors = totalDislikes + totalNeedsHelp; + const strictDenominator = strictPromoters + strictDetractors; + const npsStrict = + strictDenominator > 0 + ? Math.round( + ((strictPromoters - strictDetractors) / strictDenominator) * 100 + ) + : null; + + // "Peak daily" instead of sum-of-distinct because the per-day + // distinct counts can't be added across days without double-counting + // the same user. Peak gives a meaningful "biggest day" number. + const peakActiveUsers = (userData ?? []).reduce( + (peak, r) => Math.max(peak, r.total_active_users), + 0 + ); + + const totalBotQueries = (botData ?? []).reduce( + (s, r) => s + r.total_queries, + 0 + ); + const totalAutoResolved = (botData ?? []).reduce( + (s, r) => s + r.auto_resolved, + 0 + ); + const autoResolvePct = + totalBotQueries > 0 + ? Math.round((totalAutoResolved / totalBotQueries) * 100) + : null; + + return { + totalQueries, + peakActiveUsers, + autoResolvePct, + positivity, + npsStrict, + strictDenominator, + }; + }, [queryData, userData, botData]); + + // Combined query-performance series: queries (from /query) overlaid + // with active users (from /user). Date join is on ISO date string. + const queryPerformanceDaily = useMemo(() => { + const userByDate = new Map(); + (userData ?? []).forEach((r) => + userByDate.set(r.date, r.total_active_users) + ); + return (queryData ?? []).map((r) => ({ + date: r.date, + Queries: r.total_queries, + "Active Users": userByDate.get(r.date) ?? 0, + })); + }, [queryData, userData]); + + const feedbackDaily = useMemo( + () => + (queryData ?? []).map((r) => ({ + date: r.date, + Likes: r.total_likes, + Dislikes: r.total_dislikes, + })), + [queryData] + ); + + // Apply granularity. Active Users is summed-by-distinct so monthly + // requires PEAK (sum would double-count). Other fields are simple sums. + const queryPerformanceData = useMemo( + () => + granularity === "day" + ? queryPerformanceDaily + : bucketToMonth(queryPerformanceDaily, new Set(["Active Users"])), + [queryPerformanceDaily, granularity] + ); + const feedbackData = useMemo( + () => + granularity === "day" ? feedbackDaily : bucketToMonth(feedbackDaily), + [feedbackDaily, granularity] + ); + + const docsBySourceBars = useMemo( + () => + (docsBySource ?? []) + .filter((r) => r.docs_indexed > 0) + .map((r) => ({ name: r.source, value: r.docs_indexed })), + [docsBySource] + ); + + return ( +
+ } title="Analytics" /> + +
+ Date range: + + + Granularity: + +
+ + {hasError && ( + + + Error loading analytics data. Make sure you're logged in as an + admin and the backend is reachable. + + + )} + + {isInitialLoading ? ( + + ) : ( + <> + {/* Top row: range-scoped KPIs */} + + + Total Queries (range) + {kpis.totalQueries.toLocaleString()} + + + Peak Daily Active Users + {kpis.peakActiveUsers.toLocaleString()} + + + Auto-Resolution Rate (Slack) + + {kpis.autoResolvePct !== null ? `${kpis.autoResolvePct}%` : "—"} + + + + NPS — strict + + {kpis.npsStrict !== null + ? `${kpis.npsStrict > 0 ? "+" : ""}${kpis.npsStrict}` + : "—"} + + + (likes + resolved) vs (dislikes + needs-help). N= + {kpis.strictDenominator} + + + + + {/* Snapshot KPIs — current state, independent of date range */} + + + Total Docs Indexed + + {totalDocs + ? totalDocs.total_docs_indexed.toLocaleString() + : "—"} + + + {totalDocs + ? `${totalDocs.unique_docs.toLocaleString()} unique` + : ""} + + + + Slack Channels Enabled + + {slackChannels + ? slackChannels.enabled_channels.toLocaleString() + : "—"} + + + {slackChannels + ? `across ${slackChannels.total_configs} config(s)` + : ""} + + + + Positive Feedback % + + {kpis.positivity !== null ? `${kpis.positivity}%` : "—"} + + over selected date range + + + Sources Active + + {docsBySource ? docsBySourceBars.length.toLocaleString() : "—"} + + + {docsBySource ? `of ${docsBySource.length} configured` : ""} + + + + + + + Users and Query Trend + + {granularity === "day" + ? "Daily" + : "Monthly (Active Users = peak day)"}{" "} + assistant replies overlaid with active users + + + + + + Feedback Trend + + {granularity === "day" ? "Daily" : "Monthly"} thumbs up vs + thumbs down + + + + + + + Docs Indexed by Source + Snapshot — sum across all cc-pairs per source type + {docsBySourceBars.length > 0 ? ( + n.toLocaleString()} + /> + ) : ( + No documents indexed yet. + )} + + + )} +
+ ); +} diff --git a/web/src/app/admin/connector/[ccPairId]/CredentialSection.tsx b/web/src/app/admin/connector/[ccPairId]/CredentialSection.tsx new file mode 100644 index 00000000000..8af6efdb229 --- /dev/null +++ b/web/src/app/admin/connector/[ccPairId]/CredentialSection.tsx @@ -0,0 +1,180 @@ +"use client"; + +import { useState } from "react"; +import * as Yup from "yup"; +import { Button, Card, Divider, Text, Title } from "@tremor/react"; +import { Formik, Form } from "formik"; +import { Credential } from "@/lib/types"; +import { updateCredential } from "@/lib/credential"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { EditIcon } from "@/components/icons/icons"; +import { Popup } from "@/components/admin/connectors/Popup"; + +const SENSITIVE_KEY_PATTERNS = [ + "password", + "secret", + "token", + "key", + "private", +]; + +function isSensitiveKey(key: string): boolean { + const lower = key.toLowerCase(); + return SENSITIVE_KEY_PATTERNS.some((p) => lower.includes(p)); +} + +function humanizeKey(key: string): string { + return key.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); +} + +function maskValue(key: string, value: unknown): string { + if (value == null) return ""; + const str = String(value); + if (!isSensitiveKey(key)) return str; + if (str.length <= 4) return "••••"; + return `${"•".repeat(8)}${str.slice(-4)}`; +} + +interface Props { + credential: Credential>; + onUpdated: () => void; +} + +export function CredentialSection({ credential, onUpdated }: Props) { + const [isEditing, setIsEditing] = useState(false); + const [popup, setPopup] = useState<{ + message: string; + type: "success" | "error"; + } | null>(null); + + const credentialJson = credential.credential_json || {}; + const keys = Object.keys(credentialJson); + + if (keys.length === 0) { + return null; + } + + // Pre-build the validation schema and initial values from the existing + // credential JSON. Every existing key is treated as required (matches + // the create-time forms which require everything). + const initialValues: Record = {}; + const schemaShape: Record = {}; + for (const key of keys) { + const value = credentialJson[key]; + initialValues[key] = value == null ? "" : String(value); + schemaShape[key] = Yup.string().required(`${humanizeKey(key)} is required`); + } + const validationSchema = Yup.object().shape(schemaShape); + + return ( +
+ {popup && } +
+ Credential + +
+ + {!isEditing ? ( +
+ + Credential ID {credential.id}. Sensitive values are masked. + Click the pencil to edit. + + +
+ + {keys.map((key) => ( + + + + + ))} + +
+ {humanizeKey(key)} + + {maskValue(key, credentialJson[key])} +
+ + + ) : ( + + + Update the credential JSON below. The change is saved against the + existing credential row, so all connectors that share this + credential pick it up on their next run — no re-indexing needed. + + { + helpers.setSubmitting(true); + try { + const resp = await updateCredential(credential.id, { + credential_json: values, + admin_public: credential.admin_public, + }); + if (resp.ok) { + setPopup({ message: "Credential updated", type: "success" }); + setIsEditing(false); + onUpdated(); + } else { + const err = await resp.json().catch(() => ({})); + setPopup({ + message: `Error: ${err.detail || resp.statusText}`, + type: "error", + }); + } + } catch (e) { + setPopup({ message: `Error: ${e}`, type: "error" }); + } finally { + helpers.setSubmitting(false); + setTimeout(() => setPopup(null), 4000); + } + }} + > + {({ isSubmitting }) => ( +
+ {keys.map((key) => ( + + ))} +
+ + +
+ + )} +
+ + )} + + + ); +} diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx index eeca261979d..7481a6ee342 100644 --- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx +++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx @@ -19,6 +19,10 @@ import { localizeAndPrettify } from "@/lib/time"; import { getDocsProcessedPerMinute } from "@/lib/indexAttempt"; import { Modal } from "@/components/Modal"; import { CheckmarkIcon, CopyIcon } from "@/components/icons/icons"; +import { updateIndexAttemptPriority } from "@/lib/connector"; +import { mutate } from "swr"; +import { buildCCPairInfoUrl } from "./lib"; +import { usePopup } from "@/components/admin/connectors/Popup"; const NUM_IN_PAGE = 8; @@ -31,6 +35,29 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) { (indexAttempt) => indexAttempt.id === indexAttemptTracePopupId ); const [copyClicked, setCopyClicked] = useState(false); + const { popup, setPopup } = usePopup(); + const [updatingPriorityId, setUpdatingPriorityId] = useState( + null + ); + + async function bumpPriority(indexAttemptId: number, nextValue: number) { + setUpdatingPriorityId(indexAttemptId); + const errorMsg = await updateIndexAttemptPriority( + indexAttemptId, + Math.max(0, Math.min(100, Math.floor(nextValue))) + ); + setUpdatingPriorityId(null); + if (errorMsg) { + setPopup({ message: errorMsg, type: "error" }); + } else { + setPopup({ + message: `Priority updated to ${nextValue}`, + type: "success", + }); + } + setTimeout(() => setPopup(null), 3000); + mutate(buildCCPairInfoUrl(ccPair.id)); + } return ( <> @@ -74,11 +101,13 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) { )} + {popup} Time Started Status + Priority New Doc Cnt Total Doc Cnt Error Msg @@ -90,6 +119,9 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) { .map((indexAttempt) => { const docsPerMinute = getDocsProcessedPerMinute(indexAttempt)?.toFixed(2); + const priority = indexAttempt.indexing_priority ?? 0; + const isNotStarted = indexAttempt.status === "not_started"; + const isUpdating = updatingPriorityId === indexAttempt.id; return ( @@ -108,6 +140,47 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) { )} + + {isNotStarted ? ( +
+ + 0 + ? "text-xs font-semibold px-2 py-0.5 rounded bg-emerald-100 text-emerald-800" + : "text-xs px-2 py-0.5 text-subtle" + } + > + {priority} + + +
+ ) : priority > 0 ? ( + + {priority} + + ) : ( + - + )} +
diff --git a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx index 35da4e09998..7fd7c20ba1a 100644 --- a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx +++ b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx @@ -22,11 +22,15 @@ function ReIndexPopup({ setPopup: (popupSpec: PopupSpec | null) => void; hide: () => void; }) { + const [priority, setPriority] = useState(0); + async function triggerIndexing(fromBeginning: boolean) { + const clamped = Math.max(0, Math.min(100, Math.floor(priority || 0))); const errorMsg = await runConnector( connectorId, [credentialId], - fromBeginning + fromBeginning, + clamped ); if (errorMsg) { setPopup({ @@ -35,7 +39,10 @@ function ReIndexPopup({ }); } else { setPopup({ - message: "Triggered connector run", + message: + clamped > 0 + ? `Triggered connector run with priority ${clamped}` + : "Triggered connector run", type: "success", }); } @@ -45,6 +52,24 @@ function ReIndexPopup({ return (
+
+ + setPriority(Number(e.target.value))} + className="border rounded px-2 py-1 w-20 bg-background text-sm" + /> + + 0 = normal (default). Higher values jump the queue ahead of other + queued runs without affecting them. Steps of 10; conventional + ceiling: 100. + +
+ +
+ {isEditingCredential && ( + + + Update the Confluence credential below. All connectors using + this credential pick up the change on their next poll. + + + existingCredentialId={confluenceCredential.id} + formBody={ + <> + + + + } + validationSchema={Yup.object().shape({ + confluence_username: Yup.string().required( + "Please enter your username on Confluence" + ), + confluence_access_token: Yup.string().required( + "Please enter your Confluence access token" + ), + })} + initialValues={{ + confluence_username: + confluenceCredential.credential_json?.confluence_username || + "", + confluence_access_token: + confluenceCredential.credential_json + ?.confluence_access_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} ) : ( <> @@ -339,4 +400,4 @@ export default function Page() {
); -} \ No newline at end of file +} diff --git a/web/src/app/admin/connectors/github-files/page.tsx b/web/src/app/admin/connectors/github-files/page.tsx new file mode 100644 index 00000000000..eeea6809413 --- /dev/null +++ b/web/src/app/admin/connectors/github-files/page.tsx @@ -0,0 +1,332 @@ +"use client"; + +import * as Yup from "yup"; +import { useState } from "react"; +import { EditIcon, GithubIcon, TrashIcon } from "@/components/icons/icons"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import useSWR, { useSWRConfig } from "swr"; +import { errorHandlingFetcher } from "@/lib/fetcher"; +import { ErrorCallout } from "@/components/ErrorCallout"; +import { + GithubFilesConfig, + GithubCredentialJson, + Credential, + ConnectorIndexingStatus, +} from "@/lib/types"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { LoadingAnimation } from "@/components/Loading"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { usePublicCredentials } from "@/lib/hooks"; +import { Button, Card, Divider, Text, Title } from "@tremor/react"; +import { AdminPageTitle } from "@/components/admin/Title"; + +const Main = () => { + const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: connectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + errorHandlingFetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: credentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (connectorIndexingStatusesError || !connectorIndexingStatuses) { + return ( + + ); + } + + if (credentialsError || !credentialsData) { + return ( + + ); + } + + const indexingStatuses: ConnectorIndexingStatus< + GithubFilesConfig, + GithubCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "github_files" + ); + const githubCredential: Credential | undefined = + credentialsData.find( + (credential) => credential.credential_json?.github_access_token + ); + + return ( + <> + + Step 1: Provide your GitHub access token + + {githubCredential ? ( + <> +
+

Existing Access Token:

+

+ {githubCredential.credential_json.github_access_token} +

+ + +
+ {isEditingCredential && ( + + + Update the GitHub personal access token. Both the GitHub and + GitHub-Files connectors share this credential. + + + existingCredentialId={githubCredential.id} + formBody={ + + } + validationSchema={Yup.object().shape({ + github_access_token: Yup.string().required( + "Please enter the access token for Github" + ), + })} + initialValues={{ + github_access_token: + githubCredential.credential_json.github_access_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} + + ) : ( + <> + + The same access token used for the standard GitHub connector works + here. The token needs repo scope (or just + public_repo for public repos). + + + + formBody={ + + } + validationSchema={Yup.object().shape({ + github_access_token: Yup.string().required( + "Please enter the access token for Github" + ), + })} + initialValues={{ github_access_token: "" }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> + + + )} + + + Step 2: Configure the file scraper + + + {indexingStatuses.length > 0 && ( + <> + + Configured GitHub-Files connectors below. We re-fetch matching files + every 10 minutes (and skip the run if nothing under the path + prefix has been committed since the last poll). + +
+ + connectorIndexingStatuses={indexingStatuses} + liveCredential={githubCredential} + getCredential={(credential) => + credential.credential_json.github_access_token + } + onCredentialLink={async (connectorId) => { + if (githubCredential) { + await linkCredential(connectorId, githubCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + specialColumns={[ + { + header: "Repository", + key: "repository", + getValue: (ccPairStatus) => { + const c = ccPairStatus.connector.connector_specific_config; + return `${c.repo_owner}/${c.repo_name}`; + }, + }, + { + header: "Path", + key: "path_prefix", + getValue: (ccPairStatus) => { + const c = ccPairStatus.connector.connector_specific_config; + const ext = c.file_extension || ".json"; + const branch = c.branch ? `@${c.branch}` : ""; + return `${c.path_prefix}//*${ext}${branch}`; + }, + }, + ]} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + /> +
+ + + )} + + {githubCredential ? ( + + + Indexes files matching{" "} + <path_prefix>/<dir>/<file><ext>{" "} + — i.e. exactly one folder under the prefix, file directly inside. + Defaults target a{" "} + service-catalog/products/<product>/*.json layout. + + + + nameBuilder={(values) => + `GithubFiles-${values.repo_owner}/${values.repo_name}` + } + ccPairNameBuilder={(values) => + `${values.repo_owner}/${values.repo_name}` + } + source="github_files" + inputType="poll" + formBody={ + <> + + + + The folder containing per-product subfolders. Files are + indexed at exactly one level deeper. + + } + /> + + + + } + validationSchema={Yup.object().shape({ + repo_owner: Yup.string().required( + "Please enter the owner of the repository" + ), + repo_name: Yup.string().required( + "Please enter the name of the repository" + ), + path_prefix: Yup.string().required( + "Please enter the path prefix to scan" + ), + file_extension: Yup.string().required( + "Please enter the file extension to filter on" + ), + branch: Yup.string(), + })} + initialValues={{ + repo_owner: "", + repo_name: "", + path_prefix: "service-catalog/products", + file_extension: ".json", + branch: "", + }} + refreshFreq={10 * 60} + credentialId={githubCredential.id} + /> + + ) : ( + Provide your access token in Step 1 first. + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+ + } title="GitHub-Files" /> + +
+
+ ); +} diff --git a/web/src/app/admin/connectors/github/page.tsx b/web/src/app/admin/connectors/github/page.tsx index 94fe2001439..bbc0abf607e 100644 --- a/web/src/app/admin/connectors/github/page.tsx +++ b/web/src/app/admin/connectors/github/page.tsx @@ -1,7 +1,8 @@ "use client"; +import { useState } from "react"; import * as Yup from "yup"; -import { GithubIcon, TrashIcon } from "@/components/icons/icons"; +import { EditIcon, GithubIcon, TrashIcon } from "@/components/icons/icons"; import { BooleanFormField, TextFormField, @@ -22,11 +23,12 @@ import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; import { adminDeleteCredential, linkCredential } from "@/lib/credential"; import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; import { usePublicCredentials } from "@/lib/hooks"; -import { Card, Divider, Text, Title } from "@tremor/react"; +import { Button, Card, Divider, Text, Title } from "@tremor/react"; import { AdminPageTitle } from "@/components/admin/Title"; const Main = () => { const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); const { data: connectorIndexingStatuses, isLoading: isConnectorIndexingStatusesLoading, @@ -87,22 +89,73 @@ const Main = () => { {githubCredential ? ( <> - {" "} -
+

Existing Access Token:

{githubCredential.credential_json.github_access_token} -

{" "} +

+
+ {isEditingCredential && ( + + + Update the GitHub personal access token. Both the GitHub and + GitHub-Files connectors share this credential. + + + existingCredentialId={githubCredential.id} + formBody={ + + } + validationSchema={Yup.object().shape({ + github_access_token: Yup.string().required( + "Please enter the access token for Github" + ), + })} + initialValues={{ + github_access_token: + githubCredential.credential_json.github_access_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} ) : ( <> @@ -176,7 +229,18 @@ const Main = () => { getValue: (ccPairStatus) => { const connectorConfig = ccPairStatus.connector.connector_specific_config; - return `${connectorConfig.repo_owner}/${connectorConfig.repo_name}`; + const name = (connectorConfig.repo_name || "").trim(); + if (!name) { + return `${connectorConfig.repo_owner} (all repos)`; + } + if (name.includes(",")) { + const repos = name + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + return `${connectorConfig.repo_owner} (${repos.length} repos)`; + } + return `${connectorConfig.repo_owner}/${name}`; }, }, ]} @@ -198,27 +262,46 @@ const Main = () => { - nameBuilder={(values) => - `GithubConnector-${values.repo_owner}/${values.repo_name}` - } - ccPairNameBuilder={(values) => - `${values.repo_owner}/${values.repo_name}` - } + nameBuilder={(values) => { + const trimmed = (values.repo_name || "").trim(); + if (!trimmed || trimmed.includes(",")) { + return `GithubConnector-${values.repo_owner}`; + } + return `GithubConnector-${values.repo_owner}/${trimmed}`; + }} + ccPairNameBuilder={(values) => { + const trimmed = (values.repo_name || "").trim(); + if (!trimmed || trimmed.includes(",")) { + return values.repo_owner; + } + return `${values.repo_owner}/${trimmed}`; + }} source="github" inputType="poll" formBody={ <> - + + Single repo (e.g. darwin), comma-separated + list (e.g. darwin,onyx,api), or leave blank + to index every repo the access token can see under this + owner. + + } + /> } @@ -226,9 +309,7 @@ const Main = () => { repo_owner: Yup.string().required( "Please enter the owner of the repository to index e.g. darwin-ai" ), - repo_name: Yup.string().required( - "Please enter the name of the repository to index e.g. darwin " - ), + repo_name: Yup.string(), include_prs: Yup.boolean().required(), include_issues: Yup.boolean().required(), })} diff --git a/web/src/app/admin/connectors/jira/page.tsx b/web/src/app/admin/connectors/jira/page.tsx index 588a8db5ab5..596d599deaf 100644 --- a/web/src/app/admin/connectors/jira/page.tsx +++ b/web/src/app/admin/connectors/jira/page.tsx @@ -1,7 +1,8 @@ "use client"; +import { useState } from "react"; import * as Yup from "yup"; -import { JiraIcon, TrashIcon } from "@/components/icons/icons"; +import { EditIcon, JiraIcon, TrashIcon } from "@/components/icons/icons"; import { TextFormField, TextArrayFieldBuilder, @@ -24,10 +25,11 @@ import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsT import { usePopup } from "@/components/admin/connectors/Popup"; import { usePublicCredentials } from "@/lib/hooks"; import { AdminPageTitle } from "@/components/admin/Title"; -import { Card, Divider, Text, Title } from "@tremor/react"; +import { Button, Card, Divider, Text, Title } from "@tremor/react"; const Main = () => { const { popup, setPopup } = usePopup(); + const [isEditingCredential, setIsEditingCredential] = useState(false); const { mutate } = useSWRConfig(); const { @@ -92,13 +94,21 @@ const Main = () => { {jiraCredential ? ( <> -
+

Existing Access Token:

{jiraCredential.credential_json?.jira_api_token}

+
+ {isEditingCredential && ( + + + Update the Jira credential below. The form shape matches your + existing credential type ( + {jiraCredential.credential_json?.jira_user_email + ? "Cloud" + : "Server"} + ). + + {jiraCredential.credential_json?.jira_user_email !== undefined ? ( + + existingCredentialId={jiraCredential.id} + formBody={ + <> + + + + } + validationSchema={Yup.object().shape({ + jira_user_email: Yup.string().required( + "Please enter your username on Jira" + ), + jira_api_token: Yup.string().required( + "Please enter your Jira access token" + ), + })} + initialValues={{ + jira_user_email: + jiraCredential.credential_json?.jira_user_email || "", + jira_api_token: + jiraCredential.credential_json?.jira_api_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + ) : ( + + existingCredentialId={jiraCredential.id} + formBody={ + + } + validationSchema={Yup.object().shape({ + jira_api_token: Yup.string().required( + "Please enter your Jira personal access token" + ), + })} + initialValues={{ + jira_api_token: + jiraCredential.credential_json?.jira_api_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + )} + + )} ) : ( <> diff --git a/web/src/app/admin/connectors/salesforce/page.tsx b/web/src/app/admin/connectors/salesforce/page.tsx deleted file mode 100644 index e699f514ff6..00000000000 --- a/web/src/app/admin/connectors/salesforce/page.tsx +++ /dev/null @@ -1,290 +0,0 @@ -"use client"; - -import * as Yup from "yup"; -import { TrashIcon, SalesforceIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon -import { errorHandlingFetcher as fetcher } from "@/lib/fetcher"; -import useSWR, { useSWRConfig } from "swr"; -import { LoadingAnimation } from "@/components/Loading"; -import { HealthCheckBanner } from "@/components/health/healthcheck"; -import { - SalesforceConfig, - SalesforceCredentialJson, - ConnectorIndexingStatus, - Credential, -} from "@/lib/types"; // Modify or create these types as required -import { adminDeleteCredential, linkCredential } from "@/lib/credential"; -import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; -import { - TextFormField, - TextArrayFieldBuilder, -} from "@/components/admin/connectors/Field"; -import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; -import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; -import { usePublicCredentials } from "@/lib/hooks"; -import { AdminPageTitle } from "@/components/admin/Title"; -import { Card, Text, Title } from "@tremor/react"; - -const MainSection = () => { - const { mutate } = useSWRConfig(); - const { - data: connectorIndexingStatuses, - isLoading: isConnectorIndexingStatusesLoading, - error: isConnectorIndexingStatusesError, - } = useSWR[]>( - "/api/manage/admin/connector/indexing-status", - fetcher - ); - - const { - data: credentialsData, - isLoading: isCredentialsLoading, - error: isCredentialsError, - refreshCredentials, - } = usePublicCredentials(); - - if ( - (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || - (!credentialsData && isCredentialsLoading) - ) { - return ; - } - - if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { - return
Failed to load connectors
; - } - - if (isCredentialsError || !credentialsData) { - return
Failed to load credentials
; - } - - const SalesforceConnectorIndexingStatuses: ConnectorIndexingStatus< - SalesforceConfig, - SalesforceCredentialJson - >[] = connectorIndexingStatuses.filter( - (connectorIndexingStatus) => - connectorIndexingStatus.connector.source === "salesforce" - ); - - const SalesforceCredential: Credential | undefined = - credentialsData.find( - (credential) => credential.credential_json?.sf_username - ); - - return ( - <> - - The Salesforce connector allows you to index and search through your - Salesforce data. Once setup, all indicated Salesforce data will be will - be queryable within Darwin. - - - - Step 1: Provide Salesforce credentials - - {SalesforceCredential ? ( - <> -
- Existing SalesForce Username: - - {SalesforceCredential.credential_json.sf_username} - - -
- - ) : ( - <> - - As a first step, please provide the Salesforce admin account's - username, password, and Salesforce security token. You can follow - the guide{" "} - - here - {" "} - to create get your Salesforce Security Token. - - - - formBody={ - <> - - - - - } - validationSchema={Yup.object().shape({ - sf_username: Yup.string().required( - "Please enter your Salesforce username" - ), - sf_password: Yup.string().required( - "Please enter your Salesforce password" - ), - sf_security_token: Yup.string().required( - "Please enter your Salesforce security token" - ), - })} - initialValues={{ - sf_username: "", - sf_password: "", - sf_security_token: "", - }} - onSubmit={(isSuccess) => { - if (isSuccess) { - refreshCredentials(); - } - }} - /> - - - )} - - - Step 2: Manage Salesforce Connector - - - {SalesforceConnectorIndexingStatuses.length > 0 && ( - <> - - The latest state of your Salesforce objects are fetched every 10 - minutes. - -
- - connectorIndexingStatuses={SalesforceConnectorIndexingStatuses} - liveCredential={SalesforceCredential} - getCredential={(credential) => - credential.credential_json.sf_security_token - } - onUpdate={() => - mutate("/api/manage/admin/connector/indexing-status") - } - onCredentialLink={async (connectorId) => { - if (SalesforceCredential) { - await linkCredential(connectorId, SalesforceCredential.id); - mutate("/api/manage/admin/connector/indexing-status"); - } - }} - specialColumns={[ - { - header: "Connectors", - key: "connectors", - getValue: (ccPairStatus) => { - const connectorConfig = - ccPairStatus.connector.connector_specific_config; - return `${connectorConfig.requested_objects}`; - }, - }, - ]} - includeName - /> -
- - )} - - {SalesforceCredential ? ( - - - nameBuilder={(values) => - values.requested_objects && values.requested_objects.length > 0 - ? `Salesforce-${values.requested_objects.join("-")}` - : "Salesforce" - } - ccPairNameBuilder={(values) => - values.requested_objects && values.requested_objects.length > 0 - ? `Salesforce-${values.requested_objects.join("-")}` - : "Salesforce" - } - source="salesforce" - inputType="poll" - // formBody={<>} - formBodyBuilder={TextArrayFieldBuilder({ - name: "requested_objects", - label: "Specify Salesforce objects to organize by:", - subtext: ( - <> -
- Specify the Salesforce object types you want us to index.{" "} -
-
- Click - - {" "} - here{" "} - - for an example of how Darwin uses the objects.
-
- If unsure, don't specify any objects and Darwin will - default to indexing by 'Account'. -
-
- Hint: Use the singular form of the object name (e.g., - 'Opportunity' instead of 'Opportunities'). - - ), - })} - validationSchema={Yup.object().shape({ - requested_objects: Yup.array() - .of( - Yup.string().required( - "Salesforce object names must be strings" - ) - ) - .required(), - })} - initialValues={{ - requested_objects: [], - }} - credentialId={SalesforceCredential.id} - refreshFreq={10 * 60} // 10 minutes - /> - - ) : ( - - Please provide all Salesforce info in Step 1 first! Once you're - done with that, you can then specify which Salesforce objects you want - to make searchable. - - )} - - ); -}; - -export default function Page() { - return ( -
-
- -
- - } title="Salesforce" /> - - -
- ); -} diff --git a/web/src/app/admin/connectors/sf-account/page.tsx b/web/src/app/admin/connectors/sf-account/page.tsx new file mode 100644 index 00000000000..f2ca7d1f1f5 --- /dev/null +++ b/web/src/app/admin/connectors/sf-account/page.tsx @@ -0,0 +1,357 @@ +"use client"; + +import { useState } from "react"; +import * as Yup from "yup"; +import { EditIcon, TrashIcon, SalesforceIcon } from "@/components/icons/icons"; +import { errorHandlingFetcher as fetcher } from "@/lib/fetcher"; +import useSWR, { useSWRConfig } from "swr"; +import { LoadingAnimation } from "@/components/Loading"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { Button } from "@tremor/react"; +import { + SalesforceConfig, + SalesforceCredentialJson, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; // Modify or create these types as required +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { usePublicCredentials } from "@/lib/hooks"; +import { AdminPageTitle } from "@/components/admin/Title"; +import { Card, Text, Title } from "@tremor/react"; + +const MainSection = () => { + const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const SalesforceConnectorIndexingStatuses: ConnectorIndexingStatus< + SalesforceConfig, + SalesforceCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "salesforce" + ); + + // Match credentials tagged for the Account connector. Untagged credentials + // (created before sf_credential_kind existed) are accepted as legacy + // "account" credentials so existing setups keep working. + const SalesforceCredential: Credential | undefined = + credentialsData.find( + (credential) => + credential.credential_json?.sf_username && + (credential.credential_json?.sf_credential_kind === "account" || + credential.credential_json?.sf_credential_kind === undefined) + ); + + return ( + <> + + The Salesforce connector indexes Account records — including standard + and custom fields like Name, Owner, RecordType, CSM, TAM, CSD, + Maintenance Flag, Vertical, and Annual Revenue — making them queryable + within Darwin. + + + + Step 1: Provide Salesforce credentials + + {SalesforceCredential ? ( + <> +
+ Existing SalesForce Username: + + {SalesforceCredential.credential_json.sf_username} + + + +
+ {isEditingCredential && ( + + + Update the Salesforce Connected App credentials below. All + connectors using this credential will pick up the change on + their next run. + + + existingCredentialId={SalesforceCredential.id} + formBody={ + <> + + + + + + } + validationSchema={Yup.object().shape({ + sf_client_id: Yup.string().required( + "Please enter your Salesforce Client Id" + ), + sf_client_secret: Yup.string().required( + "Please enter your Salesforce Client Secret" + ), + sf_username: Yup.string().required( + "Please enter your Salesforce username" + ), + sf_password: Yup.string().required( + "Please enter your Salesforce password" + ), + // Hidden discriminator (set in initialValues below); + // declared here so the schema's inferred Shape matches + // SalesforceCredentialJson. + sf_credential_kind: Yup.string() + .oneOf(["account", "kbarticles"]) + .optional(), + })} + initialValues={{ + sf_client_id: + SalesforceCredential.credential_json.sf_client_id || "", + sf_client_secret: + SalesforceCredential.credential_json.sf_client_secret || "", + sf_username: + SalesforceCredential.credential_json.sf_username || "", + sf_password: + SalesforceCredential.credential_json.sf_password || "", + sf_credential_kind: "account", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} + + ) : ( + <> + + As a first step, please provide the Salesforce Connected App's + client_id and client_secret along with the Salesforce account's + username and password. + + + + formBody={ + <> + + + + + + } + validationSchema={Yup.object().shape({ + sf_client_id: Yup.string().required( + "Please enter your Salesforce Client Id" + ), + sf_client_secret: Yup.string().required( + "Please enter your Salesforce Client Secret" + ), + sf_username: Yup.string().required( + "Please enter your Salesforce username" + ), + sf_password: Yup.string().required( + "Please enter your Salesforce password" + ), + sf_credential_kind: Yup.string() + .oneOf(["account", "kbarticles"]) + .optional(), + })} + initialValues={{ + sf_client_id: "", + sf_client_secret: "", + sf_username: "", + sf_password: "", + sf_credential_kind: "account", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> + + + )} + + + Step 2: Manage Salesforce Connector + + + {SalesforceConnectorIndexingStatuses.length > 0 && ( + <> + + The latest state of your Salesforce Account records is fetched every + 10 minutes. + +
+ + connectorIndexingStatuses={SalesforceConnectorIndexingStatuses} + liveCredential={SalesforceCredential} + getCredential={(credential) => + credential.credential_json.sf_client_secret + } + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + onCredentialLink={async (connectorId) => { + if (SalesforceCredential) { + await linkCredential(connectorId, SalesforceCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + includeName + /> +
+ + )} + + {SalesforceCredential ? ( + + + The Salesforce connector indexes the Account object using a + curated set of fields. Filtering is configured at the indexer + process via environment variables — restart the indexer worker after + changing them: + +
    +
  • + SF_ACCOUNT_NAME_FILTER — substring match on + Account.Name (set SF_ACCOUNT_NAME_EXACT=1 for an + exact match). +
    • +
  • + SF_MAINTENANCE_FLAG_FILTER — exact match on + Maintenance_Flag__c; comma-separate for multiple values. +
    • +
+ + nameBuilder={() => "SF-Account"} + ccPairNameBuilder={() => "SF-Account"} + source="salesforce" + inputType="poll" + formBody={<>} + // SalesforceConfig has `requested_objects?: string[]`; the + // SF-Account flow doesn't render an input for it (the + // backend hard-codes the Account object set), but yup's + // Shape inference still requires the field be declared. + validationSchema={Yup.object().shape({ + requested_objects: Yup.array() + .of(Yup.string().required()) + .optional(), + })} + initialValues={{}} + credentialId={SalesforceCredential.id} + refreshFreq={10 * 60} // 10 minutes + /> + + ) : ( + + Please provide all Salesforce info in Step 1 first! Once you're + done with that, you can create the Account connector below. + + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+ + } title="SF-Account" /> + + +
+ ); +} diff --git a/web/src/app/admin/connectors/sfkbarticles/page.tsx b/web/src/app/admin/connectors/sf-kbarticles/page.tsx similarity index 67% rename from web/src/app/admin/connectors/sfkbarticles/page.tsx rename to web/src/app/admin/connectors/sf-kbarticles/page.tsx index d333638e7a0..7672751b36e 100644 --- a/web/src/app/admin/connectors/sfkbarticles/page.tsx +++ b/web/src/app/admin/connectors/sf-kbarticles/page.tsx @@ -1,7 +1,8 @@ "use client"; +import { useState } from "react"; import * as Yup from "yup"; -import { TrashIcon, SalesforceIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon +import { EditIcon, TrashIcon, SalesforceIcon } from "@/components/icons/icons"; import { errorHandlingFetcher as fetcher } from "@/lib/fetcher"; import useSWR, { useSWRConfig } from "swr"; import { LoadingAnimation } from "@/components/Loading"; @@ -22,10 +23,11 @@ import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsT import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; import { usePublicCredentials } from "@/lib/hooks"; import { AdminPageTitle } from "@/components/admin/Title"; -import { Card, Text, Title } from "@tremor/react"; +import { Button, Card, Text, Title } from "@tremor/react"; const MainSection = () => { const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); const { data: connectorIndexingStatuses, isLoading: isConnectorIndexingStatusesLoading, @@ -62,13 +64,18 @@ const MainSection = () => { SfKbArticlesCredentialJson >[] = connectorIndexingStatuses.filter( (connectorIndexingStatus) => - connectorIndexingStatus.connector.source === "salesforce" + connectorIndexingStatus.connector.source === "sfkbarticles" ); + // Strict match — legacy untagged credentials belong to the Account + // (salesforce/page.tsx) connector, so we only pick up credentials that + // were explicitly created via this page. const SfKbArticlesCredential: | Credential | undefined = credentialsData.find( - (credential) => credential.credential_json?.sf_username + (credential) => + credential.credential_json?.sf_username && + credential.credential_json?.sf_credential_kind === "kbarticles" ); return ( @@ -84,21 +91,107 @@ const MainSection = () => { {SfKbArticlesCredential ? ( <> -
+
Existing SalesForce Username: {SfKbArticlesCredential.credential_json.sf_username} +
+ {isEditingCredential && ( + + + Update the Salesforce Connected App credentials below. + + + existingCredentialId={SfKbArticlesCredential.id} + formBody={ + <> + + + + + + } + validationSchema={Yup.object().shape({ + sf_client_id: Yup.string().required( + "Please enter your Salesforce Client Id" + ), + sf_client_secret: Yup.string().required( + "Please enter your Salesforce Client Secret" + ), + sf_username: Yup.string().required( + "Please enter your Salesforce username" + ), + sf_password: Yup.string().required( + "Please enter your Salesforce password" + ), + sf_credential_kind: Yup.string() + .oneOf(["account", "kbarticles"]) + .optional(), + })} + initialValues={{ + sf_client_id: + SfKbArticlesCredential.credential_json.sf_client_id || "", + sf_client_secret: + SfKbArticlesCredential.credential_json.sf_client_secret || + "", + sf_username: + SfKbArticlesCredential.credential_json.sf_username || "", + sf_password: + SfKbArticlesCredential.credential_json.sf_password || "", + sf_credential_kind: "kbarticles", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} ) : ( <> @@ -143,12 +236,16 @@ const MainSection = () => { sf_password: Yup.string().required( "Please enter your Salesforce password" ), + sf_credential_kind: Yup.string() + .oneOf(["account", "kbarticles"]) + .optional(), })} initialValues={{ sf_client_id: "", sf_client_secret: "", sf_username: "", sf_password: "", + sf_credential_kind: "kbarticles", }} onSubmit={(isSuccess) => { if (isSuccess) { @@ -208,13 +305,13 @@ const MainSection = () => { nameBuilder={(values) => values.requested_objects && values.requested_objects.length > 0 - ? `SfKbArticles-${values.requested_objects.join("-")}` - : "SfKbArticles" + ? `SF-KBArticles-${values.requested_objects.join("-")}` + : "SF-KBArticles" } ccPairNameBuilder={(values) => values.requested_objects && values.requested_objects.length > 0 - ? `SfKbArticles-${values.requested_objects.join("-")}` - : "SfKbArticles" + ? `SF-KBArticles-${values.requested_objects.join("-")}` + : "SF-KBArticles" } source="sfkbarticles" inputType="poll" @@ -281,7 +378,7 @@ export default function Page() { } - title="Salesforce KB Articles" + title="SF-KBArticles" /> diff --git a/web/src/app/admin/connectors/sharepoint/page.tsx b/web/src/app/admin/connectors/sharepoint/page.tsx index 4aea858a3d0..8de3e66148e 100644 --- a/web/src/app/admin/connectors/sharepoint/page.tsx +++ b/web/src/app/admin/connectors/sharepoint/page.tsx @@ -1,7 +1,8 @@ "use client"; import * as Yup from "yup"; -import { TrashIcon, SharepointIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon +import { useState } from "react"; +import { EditIcon, TrashIcon, SharepointIcon } from "@/components/icons/icons"; import { errorHandlingFetcher } from "@/lib/fetcher"; import { ErrorCallout } from "@/components/ErrorCallout"; import useSWR, { useSWRConfig } from "swr"; @@ -23,10 +24,11 @@ import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsT import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; import { usePublicCredentials } from "@/lib/hooks"; import { AdminPageTitle } from "@/components/admin/Title"; -import { Card, Text, Title } from "@tremor/react"; +import { Button, Card, Text, Title } from "@tremor/react"; const MainSection = () => { const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); const { data: connectorIndexingStatuses, isLoading: isConnectorIndexingStatusesLoading, @@ -95,21 +97,93 @@ const MainSection = () => { {sharepointCredential ? ( <> -
+
Existing Azure AD Client ID: {sharepointCredential.credential_json.sp_client_id} +
+ {isEditingCredential && ( + + + Update the Azure AD application credential below. All connectors + using this credential pick up the change on their next poll. + + + existingCredentialId={sharepointCredential.id} + formBody={ + <> + + + + + } + validationSchema={Yup.object().shape({ + sp_client_id: Yup.string().required( + "Please enter your Application (client) ID" + ), + sp_directory_id: Yup.string().required( + "Please enter your Directory (tenant) ID" + ), + sp_client_secret: Yup.string().required( + "Please enter your Client Secret" + ), + })} + initialValues={{ + sp_client_id: + sharepointCredential.credential_json.sp_client_id || "", + sp_directory_id: + sharepointCredential.credential_json.sp_directory_id || "", + sp_client_secret: + sharepointCredential.credential_json.sp_client_secret || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} ) : ( <> diff --git a/web/src/app/admin/connectors/slack/page.tsx b/web/src/app/admin/connectors/slack/page.tsx index e8d6f82306a..b82e1194aae 100644 --- a/web/src/app/admin/connectors/slack/page.tsx +++ b/web/src/app/admin/connectors/slack/page.tsx @@ -1,7 +1,8 @@ "use client"; +import { useState } from "react"; import * as Yup from "yup"; -import { SlackIcon, TrashIcon } from "@/components/icons/icons"; +import { EditIcon, SlackIcon, TrashIcon } from "@/components/icons/icons"; import { errorHandlingFetcher } from "@/lib/fetcher"; import { ErrorCallout } from "@/components/ErrorCallout"; import useSWR, { useSWRConfig } from "swr"; @@ -29,6 +30,7 @@ import { AdminPageTitle } from "@/components/admin/Title"; const MainSection = () => { const { mutate } = useSWRConfig(); + const [isEditingCredential, setIsEditingCredential] = useState(false); const { data: connectorIndexingStatuses, isLoading: isConnectorIndexingStatusesLoading, @@ -89,23 +91,75 @@ const MainSection = () => { {slackCredential ? ( <> -
+
Existing Slack Bot Token: {slackCredential.credential_json.slack_bot_token} +
+ {isEditingCredential && ( + + + Update the Slack bot token. The change is saved against the + existing credential, so all linked Slack connectors pick it up + on their next poll. + + + existingCredentialId={slackCredential.id} + formBody={ + + } + validationSchema={Yup.object().shape({ + slack_bot_token: Yup.string().required( + "Please enter your Slack bot token" + ), + })} + initialValues={{ + slack_bot_token: + slackCredential.credential_json.slack_bot_token || "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + setIsEditingCredential(false); + refreshCredentials(); + } + }} + extraActions={ + + } + /> + + )} ) : ( <> diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx index 86ca489b5ee..dd6b4b4c8a0 100644 --- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx +++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx @@ -1,25 +1,30 @@ "use client"; import { + Button, Table, TableHead, TableRow, TableHeaderCell, TableBody, TableCell, + Text, } from "@tremor/react"; import { CCPairStatus, IndexAttemptStatus } from "@/components/Status"; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { PageSelector } from "@/components/PageSelector"; import { timeAgo } from "@/lib/time"; -import { ConnectorIndexingStatus } from "@/lib/types"; +import { ConnectorIndexingStatus, ValidSources } from "@/lib/types"; import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle"; import { getDocsProcessedPerMinute } from "@/lib/indexAttempt"; import { useRouter } from "next/navigation"; import { isCurrentlyDeleting } from "@/lib/documentDeletion"; -import { FiCheck, FiEdit2, FiXCircle } from "react-icons/fi"; +import { FiCheck, FiEdit2, FiSearch, FiX, FiXCircle } from "react-icons/fi"; +import { getSourceMetadata } from "@/lib/sources"; +import { updateConnector } from "@/lib/connector"; +import { usePopup } from "@/components/admin/connectors/Popup"; -const NUM_IN_PAGE = 20; +const NUM_IN_PAGE = 10; function CCPairIndexingStatusDisplay({ ccPairsIndexingStatus, @@ -96,20 +101,275 @@ function ClickableTableRow({ export function CCPairIndexingStatusTable({ ccPairsIndexingStatuses, + onRefresh, }: { ccPairsIndexingStatuses: ConnectorIndexingStatus[]; + onRefresh?: () => void; }) { const [page, setPage] = useState(1); - const ccPairsIndexingStatusesForPage = ccPairsIndexingStatuses.slice( - NUM_IN_PAGE * (page - 1), - NUM_IN_PAGE * page + const [sourceFilter, setSourceFilter] = useState("all"); + const [statusFilter, setStatusFilter] = useState("all"); + const [nameSearch, setNameSearch] = useState(""); + const [selectedCcPairIds, setSelectedCcPairIds] = useState>( + new Set() ); + const [isMutating, setIsMutating] = useState(false); + const { popup, setPopup } = usePopup(); + + const uniqueSources = useMemo(() => { + const set = new Set(); + for (const s of ccPairsIndexingStatuses) { + set.add(s.connector.source); + } + return Array.from(set).sort(); + }, [ccPairsIndexingStatuses]); + + // The status as the row actually presents in the table: a disabled + // connector reads as "paused" (we render "not_started" with disabled + // styling), otherwise it's the latest IndexAttempt status. Keeping + // this in sync with `CCPairIndexingStatusDisplay` ensures the filter + // matches what the user sees on screen. + const effectiveStatus = (s: ConnectorIndexingStatus): string => { + if (s.connector.disabled) return "paused"; + return s.last_status || "not_started"; + }; + + const filteredRows = useMemo(() => { + let rows = ccPairsIndexingStatuses; + if (sourceFilter !== "all") { + rows = rows.filter((s) => s.connector.source === sourceFilter); + } + if (statusFilter !== "all") { + rows = rows.filter((s) => effectiveStatus(s) === statusFilter); + } + const q = nameSearch.trim().toLowerCase(); + if (q) { + rows = rows.filter((s) => s.name?.toLowerCase().includes(q)); + } + return rows; + }, [ccPairsIndexingStatuses, sourceFilter, statusFilter, nameSearch]); + + // Reset page + selection when any filter changes so we never act on hidden rows. + useEffect(() => { + setPage(1); + setSelectedCcPairIds(new Set()); + }, [sourceFilter, statusFilter, nameSearch]); + + const anyFilterActive = + sourceFilter !== "all" || + statusFilter !== "all" || + nameSearch.trim() !== ""; + + const clearAllFilters = () => { + setSourceFilter("all"); + setStatusFilter("all"); + setNameSearch(""); + }; + + const totalPages = Math.max(1, Math.ceil(filteredRows.length / NUM_IN_PAGE)); + const safePage = Math.min(page, totalPages); + const rowsForPage = filteredRows.slice( + NUM_IN_PAGE * (safePage - 1), + NUM_IN_PAGE * safePage + ); + + const idsOnPage = rowsForPage.map((r) => r.cc_pair_id); + const allOnPageSelected = + idsOnPage.length > 0 && idsOnPage.every((id) => selectedCcPairIds.has(id)); + const anyOnPageSelected = idsOnPage.some((id) => selectedCcPairIds.has(id)); + + const togglePageSelection = () => { + setSelectedCcPairIds((prev) => { + const next = new Set(prev); + if (allOnPageSelected) { + idsOnPage.forEach((id) => next.delete(id)); + } else { + idsOnPage.forEach((id) => next.add(id)); + } + return next; + }); + }; + + const toggleRowSelection = (id: number) => { + setSelectedCcPairIds((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }; + + const selectedRows = ccPairsIndexingStatuses.filter((s) => + selectedCcPairIds.has(s.cc_pair_id) + ); + const selectedCount = selectedRows.length; + const allSelectedDisabled = + selectedCount > 0 && selectedRows.every((s) => s.connector.disabled); + const allSelectedEnabled = + selectedCount > 0 && selectedRows.every((s) => !s.connector.disabled); + + const bulkSetDisabled = async (disabled: boolean) => { + if (selectedRows.length === 0 || isMutating) return; + setIsMutating(true); + + const results = await Promise.allSettled( + selectedRows.map((s) => updateConnector({ ...s.connector, disabled })) + ); + const failures = results.filter((r) => r.status === "rejected").length; + const successes = results.length - failures; + + setPopup({ + message: failures + ? `${successes} updated, ${failures} failed` + : `${disabled ? "Paused" : "Re-enabled"} ${successes} connector(s)`, + type: failures ? "error" : "success", + }); + setTimeout(() => setPopup(null), 4000); + + setSelectedCcPairIds(new Set()); + setIsMutating(false); + onRefresh?.(); + }; return (
+ {popup} +
+ + + + +
+ + setNameSearch(e.target.value)} + /> +
+ + {anyFilterActive && ( + + )} + + + {filteredRows.length} of{" "} + {ccPairsIndexingStatuses.length} connectors + {selectedCount > 0 && ( + <> + {" · "} + {selectedCount} selected + + )} + +
+ + {selectedCount > 0 && ( +
+ + {selectedCount} selected + +
+ + + +
+
+ )} +
+ + { + if (el) { + el.indeterminate = !allOnPageSelected && anyOnPageSelected; + } + }} + onChange={togglePageSelection} + onClick={(e) => e.stopPropagation()} + /> + Connector Status Is Public @@ -118,15 +378,29 @@ export function CCPairIndexingStatusTable({ - {ccPairsIndexingStatusesForPage.map((ccPairsIndexingStatus) => { + {rowsForPage.map((ccPairsIndexingStatus) => { + const id = ccPairsIndexingStatus.cc_pair_id; + const isSelected = selectedCcPairIds.has(id); return ( + e.stopPropagation()} + > + toggleRowSelection(id)} + onClick={(e) => e.stopPropagation()} + /> +
@@ -160,14 +434,12 @@ export function CCPairIndexingStatusTable({ })}
- {ccPairsIndexingStatuses.length > NUM_IN_PAGE && ( + {filteredRows.length > NUM_IN_PAGE && (
{ setPage(newPage); window.scrollTo({ diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx index 8cebea2349a..f5b8170dbcd 100644 --- a/web/src/app/admin/indexing/status/page.tsx +++ b/web/src/app/admin/indexing/status/page.tsx @@ -1,5 +1,6 @@ "use client"; +import { useState } from "react"; import useSWR from "swr"; import { LoadingAnimation } from "@/components/Loading"; @@ -10,18 +11,63 @@ import { CCPairIndexingStatusTable } from "./CCPairIndexingStatusTable"; import { AdminPageTitle } from "@/components/admin/Title"; import Link from "next/link"; import { Button, Text } from "@tremor/react"; +import { FiRefreshCw } from "react-icons/fi"; + +const INDEXING_STATUS_URL_BASE = "/api/manage/admin/connector/indexing-status"; + +// "Show" filter: drives the server-side `disabled` query param so we +// don't ship paused cc-pairs over the wire by default. Environments +// with hundreds of historical (paused) connectors paid for them in +// every 30s poll before this. +type ShowFilter = "enabled" | "disabled" | "all"; + +function buildIndexingStatusUrl(show: ShowFilter): string { + if (show === "enabled") return `${INDEXING_STATUS_URL_BASE}?disabled=false`; + if (show === "disabled") return `${INDEXING_STATUS_URL_BASE}?disabled=true`; + return INDEXING_STATUS_URL_BASE; +} function Main() { + // Default to "enabled" so the initial load is small. Switching the + // dropdown changes the SWR key (different URL) so SWR re-fetches + // and caches each variant separately. + const [show, setShow] = useState("enabled"); + // Tracks whether the user just clicked Refresh, so the button only + // spins on user-initiated refresh — not on every 30s background + // poll (which would otherwise leave the button perpetually loading). + const [isManualRefreshing, setIsManualRefreshing] = useState(false); + const { data: indexAttemptData, isLoading: indexAttemptIsLoading, error: indexAttemptError, + mutate: refetchIndexAttempt, } = useSWR[]>( - "/api/manage/admin/connector/indexing-status", + buildIndexingStatusUrl(show), errorHandlingFetcher, - { refreshInterval: 10000 } // 10 seconds + { + // Background poll cadence. 10s was unnecessarily aggressive for + // an admin overview page and kept all open admin tabs hammering + // the endpoint. + refreshInterval: 30000, + // Don't poll while the tab is hidden — admins routinely leave + // the page open in a background tab. + refreshWhenHidden: false, + // Re-fetch when the tab regains focus so a stale view doesn't + // linger after a long absence. + revalidateOnFocus: true, + } ); + const handleManualRefresh = async () => { + setIsManualRefreshing(true); + try { + await refetchIndexAttempt(); + } finally { + setIsManualRefreshing(false); + } + }; + if (indexAttemptIsLoading) { return ; } @@ -34,7 +80,7 @@ function Main() { ); } - if (indexAttemptData.length === 0) { + if (indexAttemptData.length === 0 && show === "all") { return ( It looks like you don't have any connectors setup yet. Visit the{" "} @@ -58,7 +104,42 @@ function Main() { }); return ( - + <> +
+ + +
+ {indexAttemptData.length === 0 ? ( + + No {show === "enabled" ? "enabled" : "disabled"} connectors. + + ) : ( + refetchIndexAttempt()} + /> + )} + ); } diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx index 7a0acb8d731..70d1fc59b84 100644 --- a/web/src/app/chat/lib.tsx +++ b/web/src/app/chat/lib.tsx @@ -589,7 +589,7 @@ export async function uploadFilesForChat( return [responseJson.files as FileDescriptor[], null]; } -export async function useScrollonStream({ +export function useScrollonStream({ isStreaming, scrollableDivRef, scrollDist, diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx index 27e4464833a..c7b5826e1db 100644 --- a/web/src/components/admin/Layout.tsx +++ b/web/src/components/admin/Layout.tsx @@ -227,6 +227,20 @@ export async function Layout({ children }: { children: React.ReactNode }) { }, ], }, + { + name: "Analytics", + items: [ + { + name: ( +
+ +
Usage Analytics
+
+ ), + link: "/admin/analytics", + }, + ], + }, ...(SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED ? [ { diff --git a/web/src/components/admin/connectors/ConnectorTitle.tsx b/web/src/components/admin/connectors/ConnectorTitle.tsx index 01f357c4118..246253f487f 100644 --- a/web/src/components/admin/connectors/ConnectorTitle.tsx +++ b/web/src/components/admin/connectors/ConnectorTitle.tsx @@ -35,9 +35,12 @@ export const ConnectorTitle = ({ let additionalMetadata = new Map(); if (connector.source === "github") { const typedConnector = connector as Connector; + const repoName = typedConnector.connector_specific_config.repo_name; additionalMetadata.set( "Repo", - `${typedConnector.connector_specific_config.repo_owner}/${typedConnector.connector_specific_config.repo_name}` + repoName + ? `${typedConnector.connector_specific_config.repo_owner}/${repoName}` + : typedConnector.connector_specific_config.repo_owner ); } else if (connector.source === "gitlab") { const typedConnector = connector as Connector; diff --git a/web/src/components/admin/connectors/CredentialForm.tsx b/web/src/components/admin/connectors/CredentialForm.tsx index db081e1043d..e746e696649 100644 --- a/web/src/components/admin/connectors/CredentialForm.tsx +++ b/web/src/components/admin/connectors/CredentialForm.tsx @@ -3,22 +3,23 @@ import { Formik, Form } from "formik"; import * as Yup from "yup"; import { Popup } from "./Popup"; import { CredentialBase } from "@/lib/types"; -import { createCredential } from "@/lib/credential"; +import { createCredential, updateCredential } from "@/lib/credential"; import { Button } from "@tremor/react"; export async function submitCredential( - credential: CredentialBase + credential: CredentialBase, + existingCredentialId?: number ): Promise<{ message: string; isSuccess: boolean }> { - let isSuccess = false; try { - const response = await createCredential(credential); + const response = + existingCredentialId !== undefined + ? await updateCredential(existingCredentialId, credential) + : await createCredential(credential); if (response.ok) { - isSuccess = true; return { message: "Success!", isSuccess: true }; - } else { - const errorData = await response.json(); - return { message: `Error: ${errorData.detail}`, isSuccess: false }; } + const errorData = await response.json(); + return { message: `Error: ${errorData.detail}`, isSuccess: false }; } catch (error) { return { message: `Error: ${error}`, isSuccess: false }; } @@ -29,6 +30,13 @@ interface Props { validationSchema: Yup.ObjectSchema; initialValues: YupObjectType; onSubmit: (isSuccess: boolean) => void; + // When set, the form PATCHes the existing credential instead of creating + // a new one. Provide initialValues = existing credential_json so the user + // sees their current settings prefilled. + existingCredentialId?: number; + // Optional: rendered alongside the submit button (e.g. a Cancel that + // closes the edit panel without saving). + extraActions?: JSX.Element; } export function CredentialForm({ @@ -36,24 +44,32 @@ export function CredentialForm({ validationSchema, initialValues, onSubmit, + existingCredentialId, + extraActions, }: Props): JSX.Element { const [popup, setPopup] = useState<{ message: string; type: "success" | "error"; } | null>(null); + const isEditing = existingCredentialId !== undefined; + return ( <> {popup && } { formikHelpers.setSubmitting(true); - submitCredential({ - credential_json: values, - admin_public: true, - }).then(({ message, isSuccess }) => { + submitCredential( + { + credential_json: values, + admin_public: true, + }, + existingCredentialId + ).then(({ message, isSuccess }) => { setPopup({ message, type: isSuccess ? "success" : "error" }); formikHelpers.setSubmitting(false); setTimeout(() => { @@ -66,16 +82,17 @@ export function CredentialForm({ {({ isSubmitting }) => (
{formBody} -
+
+ {extraActions}
)} diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx index 8ef2a7d7a4f..d020abd9647 100644 --- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx +++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx @@ -54,7 +54,7 @@ function UsageTypeSection({ ); } -export function _WelcomeModal({ user }: { user: User | null }) { +export function WelcomeModalContent({ user }: { user: User | null }) { const router = useRouter(); const [selectedFlow, setSelectedFlow] = useState( null diff --git a/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx index 7355990f22e..8501e2773ae 100644 --- a/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx +++ b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx @@ -1,7 +1,7 @@ import { cookies } from "next/headers"; import { _CompletedWelcomeFlowDummyComponent, - _WelcomeModal, + WelcomeModalContent, } from "./WelcomeModal"; import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants"; import { User } from "@/lib/types"; @@ -20,5 +20,5 @@ export function WelcomeModal({ user }: { user: User | null }) { return <_CompletedWelcomeFlowDummyComponent />; } - return <_WelcomeModal user={user} />; + return ; } diff --git a/web/src/lib/connector.ts b/web/src/lib/connector.ts index 5f812d5d6fa..16adc5ba517 100644 --- a/web/src/lib/connector.ts +++ b/web/src/lib/connector.ts @@ -75,15 +75,17 @@ export async function deleteConnector( export async function runConnector( connectorId: number, credentialIds: number[], - fromBeginning: boolean = false + fromBeginning: boolean = false, + indexingPriority: number = 0 ): Promise { const response = await fetch("/api/manage/admin/connector/run-once", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ connector_id: connectorId, - credentialIds, + credential_ids: credentialIds, from_beginning: fromBeginning, + indexing_priority: indexingPriority, }), }); if (!response.ok) { @@ -92,6 +94,24 @@ export async function runConnector( return null; } +export async function updateIndexAttemptPriority( + indexAttemptId: number, + indexingPriority: number +): Promise { + const response = await fetch( + `/api/manage/admin/index-attempt/${indexAttemptId}/priority`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ indexing_priority: indexingPriority }), + } + ); + if (!response.ok) { + return (await response.json()).detail; + } + return null; +} + export async function deleteConnectorIfExistsAndIsUnlinked({ source, name, diff --git a/web/src/lib/credential.ts b/web/src/lib/credential.ts index 8fb80f02507..106920b8593 100644 --- a/web/src/lib/credential.ts +++ b/web/src/lib/credential.ts @@ -10,6 +10,19 @@ export async function createCredential(credential: CredentialBase) { }); } +export async function updateCredential( + credentialId: number, + credential: CredentialBase +) { + return await fetch(`/api/manage/credential/${credentialId}`, { + method: "PATCH", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(credential), + }); +} + export async function adminDeleteCredential(credentialId: number) { return await fetch(`/api/manage/admin/credential/${credentialId}`, { method: "DELETE", diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index 6c93d412a69..62ff0b707ee 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -45,6 +45,10 @@ interface PartialSourceMetadata { icon: React.FC<{ size?: number; className?: string }>; displayName: string; category: SourceCategory; + // Optional explicit admin URL. If unset, the URL is derived from + // displayName. Use this when you want the URL to differ from the + // displayName-derived default (e.g. when retaining a legacy route). + adminUrl?: string; } type SourceMap = { @@ -82,6 +86,11 @@ const SOURCE_METADATA_MAP: SourceMap = { displayName: "Github", category: SourceCategory.AppConnection, }, + github_files: { + icon: GithubIcon, + displayName: "GitHub-Files", + category: SourceCategory.AppConnection, + }, gitlab: { icon: GitlabIcon, displayName: "Gitlab", @@ -169,12 +178,12 @@ const SOURCE_METADATA_MAP: SourceMap = { }, salesforce: { icon: SalesforceIcon, - displayName: "Salesforce", + displayName: "SF-Account", category: SourceCategory.AppConnection, }, sfkbarticles: { icon: SalesforceIcon, - displayName: "SfKbArticles", + displayName: "SF-KBArticles", category: SourceCategory.AppConnection, }, sharepoint: { @@ -246,9 +255,11 @@ function fillSourceMetadata( return { internalName: internalName, ...partialMetadata, - adminUrl: `/admin/connectors/${partialMetadata.displayName - .toLowerCase() - .replaceAll(" ", "-")}`, + adminUrl: + partialMetadata.adminUrl ?? + `/admin/connectors/${partialMetadata.displayName + .toLowerCase() + .replaceAll(" ", "-")}`, }; } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index d1dae473573..dd7f78ae88a 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -29,6 +29,7 @@ export interface MinimalUserSnapshot { export type ValidSources = | "web" | "github" + | "github_files" | "gitlab" | "slack" | "google_drive" @@ -108,11 +109,23 @@ export interface WebConfig { export interface GithubConfig { repo_owner: string; - repo_name: string; + // Optional: blank means index every repo the access token can see + // under this owner. The form's helper text ("leave blank to index + // every repo ...") and the page's `repo_name || ""` consumers all + // expect this; the yup schema doesn't `.required()` it either. + repo_name?: string; include_prs: boolean; include_issues: boolean; } +export interface GithubFilesConfig { + repo_owner: string; + repo_name: string; + path_prefix: string; + file_extension: string; + branch?: string; +} + export interface GitlabConfig { project_owner: string; project_name: string; @@ -278,6 +291,8 @@ export interface IndexAttemptSnapshot { full_exception_trace: string | null; time_started: string | null; time_updated: string; + // 0 = normal (auto-scheduled). Higher values jump the indexing queue. + indexing_priority?: number; } export interface ConnectorIndexingStatus< @@ -459,9 +474,15 @@ export interface OCICredentialJson { } export interface SalesforceCredentialJson { + sf_client_id: string; + sf_client_secret: string; sf_username: string; sf_password: string; - sf_security_token: string; + // Discriminator so the salesforce + sfkbarticles pages don't pick up each + // other's credentials. "account" for the salesforce/Account connector, + // "kbarticles" for the sfkbarticles connector. Optional for backward + // compat with credentials created before this field existed. + sf_credential_kind?: "account" | "kbarticles"; } export interface SfKbArticlesCredentialJson { @@ -469,6 +490,7 @@ export interface SfKbArticlesCredentialJson { sf_client_secret: string; sf_username: string; sf_password: string; + sf_credential_kind?: "account" | "kbarticles"; } export interface SharepointCredentialJson { diff --git a/web/src/middleware.ts b/web/src/middleware.ts index 714e70b4323..c8e353abde8 100644 --- a/web/src/middleware.ts +++ b/web/src/middleware.ts @@ -10,7 +10,6 @@ const eePaths = [ "/admin/whitelabeling", "/admin/performance/custom-analytics", ]; -const eePathsForMatcher = eePaths.map((path) => `${path}/:path*`); export async function middleware(request: NextRequest) { if (SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED) { @@ -33,7 +32,18 @@ export async function middleware(request: NextRequest) { return NextResponse.next(); } -// Specify the paths that the middleware should run for +// Next.js statically analyses this `config` object at build time and +// rejects computed values, so the matcher list has to be inlined as +// literals. Keep this in sync with `eePaths` above — adding a path to +// one without the other will cause middleware to either skip the route +// (if missing here) or run on every route (if a non-literal sneaks back). export const config = { - matcher: eePathsForMatcher, + matcher: [ + "/admin/groups/:path*", + "/admin/api-key/:path*", + "/admin/performance/usage/:path*", + "/admin/performance/query-history/:path*", + "/admin/whitelabeling/:path*", + "/admin/performance/custom-analytics/:path*", + ], };